EVEREST-400 encrypt secrets in secrets storage (#245)

Co-authored-by: Andrew Minkin <[email protected]>
Co-authored-by: Maxim Kondratenko <[email protected]>

.github/workflows/ci.yml

@@ -196,7 +196,13 @@ jobs:
           go env
           pwd
           git status
+
   integration_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version: [1.21.x]
+        may-fail: [false]
     name: API Integration Tests
     runs-on: ubuntu-20.04
     env:
@@ -243,7 +249,7 @@ jobs:
         run: |
           make local-env-up
           make build-debug
-          make run-debug &> everest-backend.log &
+          SECRETS_ROOT_KEY=$(openssl rand -base64 32) make run-debug &> everest-backend.log &
 
       - name: Checkout CLI repo
         uses: actions/checkout@v4

README.md

@@ -20,9 +20,17 @@ The Percona Everest has two primary components that assist you in creating the e
 
 To start using Everest, use the following commands:
 
+```sh
+export SECRETS_ROOT_KEY=$(openssl rand -base64 32)
+echo "$SECRETS_ROOT_KEY"
+```
+This generates a base64-encoded 256-bit key used for secrets encryption. Make
+sure to securely store this key, without it everest won't be able to access the
+secrets stored in its internal secrets storage.
+
 ```sh
 wget https://raw.githubusercontent.com/percona/percona-everest-backend/main/deploy/quickstart-compose.yml
-docker compose -f quickstart.yml up -d
+docker compose -f quickstart-compose.yml up -d
 ```
 This will spin up the backend/frontend, accessible at http://127.0.0.1:8080.
 

api/backup_storage.go

@@ -197,11 +197,11 @@ func (e *EverestServer) deleteBackupStorage(c context.Context, bs *model.BackupS
 			e.l.Error(err)
 			return errors.New("could not delete backup storage")
 		}
-		if _, err := e.secretsStorage.DeleteSecret(c, bs.AccessKeyID); err != nil {
+		if err := e.secretsStorage.DeleteSecret(c, bs.AccessKeyID); err != nil {
 			return errors.Join(err, errors.New("could not delete access key from secrets storage"))
 		}
 
-		if _, err := e.secretsStorage.DeleteSecret(c, bs.SecretKeyID); err != nil {
+		if err := e.secretsStorage.DeleteSecret(c, bs.SecretKeyID); err != nil {
 			return errors.Join(err, errors.New("could not delete secret key from secrets storage"))
 		}
 
@@ -342,7 +342,7 @@ func (e *EverestServer) createSecrets(
 		newAccessKeyID = &newID
 
 		// create new AccessKey
-		err := e.secretsStorage.CreateSecret(ctx, newID, *accessKey)
+		err := e.secretsStorage.PutSecret(ctx, newID, *accessKey)
 		if err != nil {
 			e.l.Error(err)
 			return newAccessKeyID, newSecretKeyID, errors.New("could not store access key in secrets storage")
@@ -354,7 +354,7 @@ func (e *EverestServer) createSecrets(
 		newSecretKeyID = &newID
 
 		// create new SecretKey
-		err := e.secretsStorage.CreateSecret(ctx, newID, *secretKey)
+		err := e.secretsStorage.PutSecret(ctx, newID, *secretKey)
 		if err != nil {
 			e.l.Error(err)
 			return newAccessKeyID, newSecretKeyID, errors.New("could not store secret key in secrets storage")
@@ -367,15 +367,15 @@ func (e *EverestServer) createSecrets(
 func (e *EverestServer) deleteOldSecretsAfterUpdate(ctx context.Context, params *UpdateBackupStorageParams, s *model.BackupStorage) {
 	// delete old AccessKey
 	if params.AccessKey != nil {
-		_, cErr := e.secretsStorage.DeleteSecret(ctx, s.AccessKeyID)
+		cErr := e.secretsStorage.DeleteSecret(ctx, s.AccessKeyID)
 		if cErr != nil {
 			e.l.Errorf("Failed to delete unused secret, please delete it manually. id = %s", s.AccessKeyID)
 		}
 	}
 
 	// delete old SecretKey
 	if params.SecretKey != nil {
-		_, cErr := e.secretsStorage.DeleteSecret(ctx, s.SecretKeyID)
+		cErr := e.secretsStorage.DeleteSecret(ctx, s.SecretKeyID)
 		if cErr != nil {
 			e.l.Errorf("Failed to delete unused secret, please delete it manually. id = %s", s.SecretKeyID)
 		}
@@ -391,14 +391,14 @@ func (e *EverestServer) cleanUpNewSecretsOnUpdateError(err error, newAccessKeyID
 
 	// if an error appeared - cleanup the created secrets
 	if newAccessKeyID != nil {
-		_, err = e.secretsStorage.DeleteSecret(ctx, *newAccessKeyID)
+		err = e.secretsStorage.DeleteSecret(ctx, *newAccessKeyID)
 		if err != nil {
 			e.l.Errorf("Failed to delete unused secret, please delete it manually. id = %s", *newAccessKeyID)
 		}
 	}
 
 	if newSecretKeyID != nil {
-		_, err = e.secretsStorage.DeleteSecret(ctx, *newSecretKeyID)
+		err = e.secretsStorage.DeleteSecret(ctx, *newSecretKeyID)
 		if err != nil {
 			e.l.Errorf("Failed to delete unused secret, please delete it manually. id = %s", *newSecretKeyID)
 		}

api/deps.go

@@ -27,12 +27,9 @@ import (
 const pgErrUniqueViolation = "unique_violation"
 
 type secretsStorage interface {
-	CreateSecret(ctx context.Context, id, value string) error
+	PutSecret(ctx context.Context, id, value string) error
 	GetSecret(ctx context.Context, id string) (string, error)
-	UpdateSecret(ctx context.Context, id, value string) error
-	DeleteSecret(ctx context.Context, id string) (string, error)
-
-	Close() error
+	DeleteSecret(ctx context.Context, id string) error
 }
 
 type storage interface {

api/everest.go

@@ -20,6 +20,7 @@ package api
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -64,7 +65,7 @@ func NewEverestServer(c *config.EverestConfig, l *zap.SugaredLogger) (*EverestSe
 	if err := e.initHTTPServer(); err != nil {
 		return e, err
 	}
-	err := e.initEverest()
+	err := e.initEverest(context.Background())
 	if err != nil {
 		e.l.Error(err)
 		return e, err
@@ -79,15 +80,29 @@ func NewEverestServer(c *config.EverestConfig, l *zap.SugaredLogger) (*EverestSe
 	return e, err
 }
 
-func (e *EverestServer) initEverest() error {
+func (e *EverestServer) initEverest(ctx context.Context) error {
 	db, err := model.NewDatabase(pgStorageName, e.config.DSN, pgMigrationsDir)
 	if err != nil {
 		return err
 	}
-	e.storage = db
-	e.secretsStorage = db // so far the db implements both interfaces - the regular storage and the secrets storage
+
 	_, err = db.Migrate()
-	return err
+	if err != nil {
+		return err
+	}
+
+	e.storage = db
+
+	secretsRootKey, err := base64.StdEncoding.DecodeString(e.config.SecretsRootKey)
+	if err != nil {
+		return err
+	}
+	e.secretsStorage, err = model.NewSecretsStorage(ctx, e.config.DSN, secretsRootKey)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (e *EverestServer) initKubeClient(ctx context.Context, kubernetesID string) (*model.KubernetesCluster, *kubernetes.Kubernetes, int, error) {
@@ -179,17 +194,6 @@ func (e *EverestServer) Shutdown(ctx context.Context) error {
 		}
 	}()
 
-	e.waitGroup.Add(1)
-	go func() {
-		defer e.waitGroup.Done()
-		e.l.Info("Shutting down secrets storage")
-		if err := e.secretsStorage.Close(); err != nil {
-			e.l.Error(errors.Join(err, errors.New("could not shut down secret storage")))
-		} else {
-			e.l.Info("Secret storage shut down")
-		}
-	}()
-
 	done := make(chan struct{}, 1)
 	go func() {
 		e.waitGroup.Wait()

api/kubernetes.go

@@ -106,7 +106,7 @@ func (e *EverestServer) RegisterKubernetesCluster(ctx echo.Context) error {
 		return ctx.JSON(http.StatusBadRequest, Error{Message: pointer.ToString("Could not create Kubernetes cluster")})
 	}
 
-	err = e.secretsStorage.CreateSecret(c, k.ID, params.Kubeconfig)
+	err = e.secretsStorage.PutSecret(c, k.ID, params.Kubeconfig)
 	if err != nil {
 		e.l.Error(err)
 		return ctx.JSON(http.StatusBadRequest, Error{Message: pointer.ToString("Could not store kubeconfig in secrets storage")})
@@ -179,7 +179,7 @@ func (e *EverestServer) UnregisterKubernetesCluster(ctx echo.Context, kubernetes
 }
 
 func (e *EverestServer) removeK8sCluster(ctx context.Context, kubernetesID string) error {
-	if _, err := e.secretsStorage.DeleteSecret(ctx, kubernetesID); err != nil {
+	if err := e.secretsStorage.DeleteSecret(ctx, kubernetesID); err != nil {
 		return errors.Join(err, errors.New("could not delete kubeconfig from secrets storage"))
 	}
 

api/monitoring_instance.go

@@ -71,7 +71,7 @@ func (e *EverestServer) CreateMonitoringInstance(ctx echo.Context) error {
 	if err != nil {
 		e.l.Error(err)
 
-		_, err := e.secretsStorage.DeleteSecret(ctx.Request().Context(), apiKeyID)
+		err := e.secretsStorage.DeleteSecret(ctx.Request().Context(), apiKeyID)
 		if err != nil {
 			e.l.Warnf("Could not delete secret %s from secret storage due to error: %s", apiKeyID, err)
 		}
@@ -203,7 +203,7 @@ func (e *EverestServer) deleteMonitoringConfig(c context.Context, i *model.Monit
 			return errors.New("could not delete monitoring instance")
 		}
 
-		_, err := e.secretsStorage.DeleteSecret(c, i.APIKeySecretID)
+		err := e.secretsStorage.DeleteSecret(c, i.APIKeySecretID)
 		if err != nil {
 			return errors.Join(err, fmt.Errorf("could not delete monitoring instance API key secret %s", i.APIKeySecretID))
 		}
@@ -236,7 +236,7 @@ func (e *EverestServer) createAndStorePMMApiKey(ctx context.Context, name, url,
 	}
 
 	apiKeyID := uuid.NewString()
-	if err := e.secretsStorage.CreateSecret(ctx, apiKeyID, apiKey); err != nil {
+	if err := e.secretsStorage.PutSecret(ctx, apiKeyID, apiKey); err != nil {
 		e.l.Error(err)
 		return "", errors.New("could not save API key to secrets storage")
 	}
@@ -263,7 +263,7 @@ func (e *EverestServer) performMonitoringInstanceUpdate(
 			APIKeySecretID: apiKeyID,
 		})
 		if err != nil {
-			if _, err := e.secretsStorage.DeleteSecret(ctx.Request().Context(), *apiKeyID); err != nil {
+			if err := e.secretsStorage.DeleteSecret(ctx.Request().Context(), *apiKeyID); err != nil {
 				return errors.Join(err, fmt.Errorf("could not delete secret %s from secret storage", *apiKeyID))
 			}
 
@@ -290,7 +290,7 @@ func (e *EverestServer) performMonitoringInstanceUpdate(
 		}
 
 		if apiKeyID != nil {
-			if _, err := e.secretsStorage.DeleteSecret(context.Background(), previousAPIKeyID); err != nil {
+			if err := e.secretsStorage.DeleteSecret(context.Background(), previousAPIKeyID); err != nil {
 				return errors.Join(err, fmt.Errorf("could not delete monitoring instance api key secret %s", previousAPIKeyID))
 			}
 		}

cmd/config/config.go

@@ -16,7 +16,18 @@
 // Package config ...
 package config
 
-import "github.com/kelseyhightower/envconfig"
+import (
+	"crypto/aes"
+	"encoding/base64"
+	"errors"
+
+	"github.com/kelseyhightower/envconfig"
+)
+
+const (
+	// AES256BitKeySize is the size (bytes) of a 256-bit key.
+	AES256BitKeySize = 2 * aes.BlockSize
+)
 
 //nolint:gochecknoglobals
 var (
@@ -39,18 +50,30 @@ type EverestConfig struct {
 	TelemetryInterval string `envconfig:"TELEMETRY_INTERVAL"`
 	// DisableTelemetry disable Everest and the upstream operators telemetry
 	DisableTelemetry bool `default:"false" envconfig:"DISABLE_TELEMETRY"`
+	// SecretsRootKey is a base64-encoded 256-bit key used for the secrets encryption.
+	SecretsRootKey string `required:"true" envconfig:"SECRETS_ROOT_KEY"`
 }
 
 // ParseConfig parses env vars and fills EverestConfig.
 func ParseConfig() (*EverestConfig, error) {
 	c := &EverestConfig{}
 	err := envconfig.Process("", c)
+	if err != nil {
+		return nil, err
+	}
+
 	if c.TelemetryURL == "" {
 		c.TelemetryURL = TelemetryURL
 	}
 	if c.TelemetryInterval == "" {
 		c.TelemetryInterval = TelemetryInterval
 	}
 
-	return c, err
+	// SecretsRootKey must be a base64-encoded 256-bit key.
+	secretsRootKey, err := base64.StdEncoding.DecodeString(c.SecretsRootKey)
+	if err != nil || len(secretsRootKey) != AES256BitKeySize {
+		return nil, errors.New("secrets root key must be a base64-encoded 256-bit key")
+	}
+
+	return c, nil
 }

deploy/quickstart-compose.yml

@@ -17,5 +17,6 @@ services:
         condition: service_healthy
     environment:
       - DSN=postgres://admin:pwd@pg:5432/postgres?sslmode=disable
+      - SECRETS_ROOT_KEY=${SECRETS_ROOT_KEY}
     ports:
       - ${EVEREST_BIND_ADDR:-127.0.0.1}:8080:8080

deploy/quickstart-k8s.yaml

@@ -29,6 +29,11 @@ spec:
           env:
             - name: DSN
               value: postgres://admin:pwd@localhost:5432/postgres?sslmode=disable
+            - name: SECRETS_ROOT_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: everest-secrets-root-key
+                  key: secrets-root-key
           ports:
             - containerPort: 8080
           readinessProbe: