Skip to content

Commit

Permalink
APPS-1366 Support Secret Agent keys for S3 credentials configuration (#…
Browse files Browse the repository at this point in the history 
…280)

* add cache to secret agent

* use s3 auth

* go mod

* storage toModel

* extract s3  storage to file

* secret agent in s3

* dto to model

* add convert test

* add convert test

* update storage to model test

* resolve secret agent

* no sa validation

* update doc

* merge v2

* check nil agent

* assign directly

* remove secretKeyPassword

* remove port from error message

* combine secret agents

* add secret agent test

* regenerate api

* return only SecretAgentCOnfig

* use %w
  • Loading branch information
@korotkov-aerospike
korotkov-aerospike authored Dec 5, 2024
1 parent ebbb018 commit e02d6c5
Show file tree
Hide file tree
Showing 26 changed files with 675 additions and 280 deletions.
34 changes: 25 additions & 9 deletions docs/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1885,29 +1885,25 @@ const docTemplate = `{
]
},
"password": {
"description": "The password for the cluster authentication.",
"description": "The password for the cluster authentication.\nIt can be either plain text or path into the secret agent.",
"type": "string",
"example": "testPswd"
},
"password-key-secret": {
"description": "The secret keyword in Aerospike Secret Agent containing password.\nOnly applicable when SecretAgent is specified.",
"type": "string"
},
"password-path": {
"description": "The file path with the password string, will take precedence over the password field.",
"description": "The file path with the password string.",
"type": "string",
"example": "/path/to/pass.txt"
},
"secret-agent": {
"description": "Secret Agent configuration (optional).",
"description": "Secret Agent configuration (optional).\nMutually exclusive with secret-agent-name.",
"allOf": [
{
"$ref": "#/definitions/dto.SecretAgent"
}
]
},
"secret-agent-name": {
"description": "Secret Agent configuration (optional). Link to one of preconfigured agents.",
"description": "Secret Agent configuration (optional). Link to one of preconfigured agents.\nMutually exclusive with secret-agent.",
"type": "string"
},
"user": {
Expand Down Expand Up @@ -2518,6 +2514,10 @@ const docTemplate = `{
"s3-region"
],
"properties": {
"access-key-id": {
"description": "Access Key ID for authentication with S3 StaticCredentialsProvider.\nCan be a path in secret agent or an actual value.",
"type": "string"
},
"bucket": {
"description": "The S3 bucket name.",
"type": "string"
Expand Down Expand Up @@ -2557,11 +2557,27 @@ const docTemplate = `{
"description": "The S3 region string.",
"type": "string",
"example": "eu-central-1"
},
"secret-access-key": {
"description": "Secret Access Key for authentication with S3 StaticCredentialsProvider.\nCan be a path in secret agent or an actual value.",
"type": "string"
},
"secret-agent": {
"description": "Secret Agent configuration (optional).\nMutually exclusive with secret-agent-name.",
"allOf": [
{
"$ref": "#/definitions/dto.SecretAgent"
}
]
},
"secret-agent-name": {
"description": "Secret Agent configuration (optional). Link to one of preconfigured agents.\nMutually exclusive with secret-agent.",
"type": "string"
}
}
},
"dto.SecretAgent": {
"description": "SecretAgent represents the configuration of an Aerospike Secret Agent for a backup/restore operation.",
"description": "SecretAgent represents the configuration of an Aerospike Secret Agent.",
"type": "object",
"properties": {
"address": {
Expand Down
33 changes: 24 additions & 9 deletions docs/openapi.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2032,28 +2032,24 @@
"type" : "string"
},
"password" : {
"description" : "The password for the cluster authentication.",
"description" : "The password for the cluster authentication.\nIt can be either plain text or path into the secret agent.",
"example" : "testPswd",
"type" : "string"
},
"password-key-secret" : {
"description" : "The secret keyword in Aerospike Secret Agent containing password.\nOnly applicable when SecretAgent is specified.",
"type" : "string"
},
"password-path" : {
"description" : "The file path with the password string, will take precedence over the password field.",
"description" : "The file path with the password string.",
"example" : "/path/to/pass.txt",
"type" : "string"
},
"secret-agent" : {
"allOf" : [ {
"$ref" : "#/components/schemas/dto.SecretAgent"
} ],
"description" : "Secret Agent configuration (optional).",
"description" : "Secret Agent configuration (optional).\nMutually exclusive with secret-agent-name.",
"type" : "object"
},
"secret-agent-name" : {
"description" : "Secret Agent configuration (optional). Link to one of preconfigured agents.",
"description" : "Secret Agent configuration (optional). Link to one of preconfigured agents.\nMutually exclusive with secret-agent.",
"type" : "string"
},
"user" : {
Expand Down Expand Up @@ -2583,6 +2579,10 @@
},
"dto.S3Storage" : {
"properties" : {
"access-key-id" : {
"description" : "Access Key ID for authentication with S3 StaticCredentialsProvider.\nCan be a path in secret agent or an actual value.",
"type" : "string"
},
"bucket" : {
"description" : "The S3 bucket name.",
"type" : "string"
Expand Down Expand Up @@ -2621,13 +2621,28 @@
"description" : "The S3 region string.",
"example" : "eu-central-1",
"type" : "string"
},
"secret-access-key" : {
"description" : "Secret Access Key for authentication with S3 StaticCredentialsProvider.\nCan be a path in secret agent or an actual value.",
"type" : "string"
},
"secret-agent" : {
"allOf" : [ {
"$ref" : "#/components/schemas/dto.SecretAgent"
} ],
"description" : "Secret Agent configuration (optional).\nMutually exclusive with secret-agent-name.",
"type" : "object"
},
"secret-agent-name" : {
"description" : "Secret Agent configuration (optional). Link to one of preconfigured agents.\nMutually exclusive with secret-agent.",
"type" : "string"
}
},
"required" : [ "bucket", "s3-region" ],
"type" : "object"
},
"dto.SecretAgent" : {
"description" : "SecretAgent represents the configuration of an Aerospike Secret Agent for a backup/restore operation.",
"description" : "SecretAgent represents the configuration of an Aerospike Secret Agent.",
"properties" : {
"address" : {
"description" : "Address of the Secret Agent.",
Expand Down
45 changes: 33 additions & 12 deletions docs/openapi.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1615,27 +1615,26 @@ components:
- PKI
type: string
password:
description: The password for the cluster authentication.
example: testPswd
type: string
password-key-secret:
description: |-
The secret keyword in Aerospike Secret Agent containing password.
Only applicable when SecretAgent is specified.
The password for the cluster authentication.
It can be either plain text or path into the secret agent.
example: testPswd
type: string
password-path:
description: "The file path with the password string, will take precedence\
\ over the password field."
description: The file path with the password string.
example: /path/to/pass.txt
type: string
secret-agent:
allOf:
- $ref: '#/components/schemas/dto.SecretAgent'
description: Secret Agent configuration (optional).
description: |-
Secret Agent configuration (optional).
Mutually exclusive with secret-agent-name.
type: object
secret-agent-name:
description: Secret Agent configuration (optional). Link to one of preconfigured
agents.
description: |-
Secret Agent configuration (optional). Link to one of preconfigured agents.
Mutually exclusive with secret-agent.
type: string
user:
description: The username for the cluster authentication.
Expand Down Expand Up @@ -2160,6 +2159,11 @@ components:
type: object
dto.S3Storage:
properties:
access-key-id:
description: |-
Access Key ID for authentication with S3 StaticCredentialsProvider.
Can be a path in secret agent or an actual value.
type: string
bucket:
description: The S3 bucket name.
type: string
Expand Down Expand Up @@ -2194,13 +2198,30 @@ components:
description: The S3 region string.
example: eu-central-1
type: string
secret-access-key:
description: |-
Secret Access Key for authentication with S3 StaticCredentialsProvider.
Can be a path in secret agent or an actual value.
type: string
secret-agent:
allOf:
- $ref: '#/components/schemas/dto.SecretAgent'
description: |-
Secret Agent configuration (optional).
Mutually exclusive with secret-agent-name.
type: object
secret-agent-name:
description: |-
Secret Agent configuration (optional). Link to one of preconfigured agents.
Mutually exclusive with secret-agent.
type: string
required:
- bucket
- s3-region
type: object
dto.SecretAgent:
description: SecretAgent represents the configuration of an Aerospike Secret
Agent for a backup/restore operation.
Agent.
example:
tls-ca-file: /path/to/ca.pem
address: localhost
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ require (
github.com/aerospike/aerospike-client-go/v7 v7.7.3
github.com/aerospike/aerospike-management-lib v1.5.0
github.com/aerospike/backup-go v0.2.1-0.20241201083923-4f493baae0f0
github.com/aws/aws-sdk-go-v2 v1.32.5
github.com/aws/aws-sdk-go-v2/config v1.28.5
github.com/aws/aws-sdk-go-v2/credentials v1.17.46
github.com/aws/aws-sdk-go-v2/service/s3 v1.69.0
github.com/aws/smithy-go v1.22.1
github.com/go-logr/logr v1.4.2
Expand Down Expand Up @@ -43,9 +45,7 @@ require (
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect
Expand Down
2 changes: 1 addition & 1 deletion internal/server/configuration/configuration_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func readStorage(configURI string) (model.Storage, error) {
return nil, fmt.Errorf("validate storage configuration error: %w", err)
}

return configStorage.ToModel(), nil
return configStorage.ToModel(model.NewConfig())
}

func loadFileContent(configFile string) ([]byte, error) {
Expand Down
6 changes: 4 additions & 2 deletions internal/server/handlers/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (s *Service) readAllBackups(w http.ResponseWriter, r *http.Request, isFullB
return
}

response, err := dto.Serialize(dto.ConvertBackupDetailsMap(backups), dto.JSON)
response, err := dto.Serialize(dto.ConvertBackupDetailsMap(backups, s.config), dto.JSON)
if err != nil {
hLogger.Error("failed to marshal backup list",
slog.Any("error", err),
Expand Down Expand Up @@ -169,7 +169,9 @@ func (s *Service) readBackupsForRoutine(w http.ResponseWriter, r *http.Request,
http.Error(w, "failed to retrieve backup list: "+err.Error(), http.StatusInternalServerError)
return
}
backupDetails := dto.ConvertModelsToDTO(backups, dto.NewBackupDetailsFromModel)
backupDetails := dto.ConvertModelsToDTO(backups, func(m *model.BackupDetails) *dto.BackupDetails {
return dto.NewBackupDetailsFromModel(m, s.config)
})
response, err := dto.Serialize(backupDetails, dto.JSON)
if err != nil {
hLogger.Error("failed to marshal backup list",
Expand Down
2 changes: 2 additions & 0 deletions internal/server/handlers/config_routine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ func (s *Service) ReadRoutines(w http.ResponseWriter, _ *http.Request) {
// @Success 200 {object} dto.BackupRoutine
// @Response 400 {string} string
// @Failure 404 {string} string "The specified cluster could not be found"
//
//nolint:dupl
func (s *Service) readRoutine(w http.ResponseWriter, r *http.Request) {
hLogger := s.logger.With(slog.String("handler", "readRoutine"))

Expand Down
19 changes: 15 additions & 4 deletions internal/server/handlers/config_storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,13 @@ func (s *Service) addStorage(w http.ResponseWriter, r *http.Request) {
}

err = s.changeConfig(r.Context(), func(config *model.Config) error {
return config.AddStorage(name, newStorage.ToModel())
storage, err := newStorage.ToModel(config)
if err != nil {
return fmt.Errorf("failed to convert storage: %w", err)
}
return config.AddStorage(name, storage)
})

if err != nil {
hLogger.Error("failed to add storage",
slog.String("name", name),
Expand All @@ -77,7 +82,7 @@ func (s *Service) addStorage(w http.ResponseWriter, r *http.Request) {
func (s *Service) ReadAllStorage(w http.ResponseWriter, _ *http.Request) {
hLogger := s.logger.With(slog.String("handler", "ReadAllStorage"))

toDTO := dto.ConvertStorageMapToDTO(s.config.Storage)
toDTO := dto.ConvertStorageMapToDTO(s.config.Storage, s.config)
jsonResponse, err := dto.Serialize(toDTO, dto.JSON)
if err != nil {
hLogger.Error("failed to marshal storage",
Expand Down Expand Up @@ -108,6 +113,8 @@ func (s *Service) ReadAllStorage(w http.ResponseWriter, _ *http.Request) {
// @Response 400 {string} string
// @Failure 404 {string} string "The specified storage could not be found"
// @Failure 500 {string} string
//
//nolint:dupl
func (s *Service) readStorage(w http.ResponseWriter, r *http.Request) {
hLogger := s.logger.With(slog.String("handler", "readStorage"))

Expand All @@ -123,7 +130,7 @@ func (s *Service) readStorage(w http.ResponseWriter, r *http.Request) {
return
}

jsonResponse, err := dto.Serialize(dto.NewStorageFromModel(storage), dto.JSON)
jsonResponse, err := dto.Serialize(dto.NewStorageFromModel(storage, s.config), dto.JSON)
if err != nil {
hLogger.Error("failed to marshal storage",
slog.Any("error", err),
Expand Down Expand Up @@ -170,7 +177,11 @@ func (s *Service) updateStorage(w http.ResponseWriter, r *http.Request) {
}

err = s.changeConfig(r.Context(), func(config *model.Config) error {
return config.UpdateStorage(storageName, updatedStorage.ToModel())
storage, err := updatedStorage.ToModel(config)
if err != nil {
return fmt.Errorf("failed to convert storage: %w", err)
}
return config.UpdateStorage(storageName, storage)
})
if err != nil {
hLogger.Error("failed to update storage",
Expand Down
Loading

0 comments on commit e02d6c5

Please sign in to comment.