add fetching awsRole in settings

formancehq · Feb 28, 2024 · 6af9271 · 6af9271
1 parent 0a7d33f
commit 6af9271
Show file tree

Hide file tree

Showing 17 changed files with 250 additions and 58 deletions.
diff --git a/components/operator/README.md b/components/operator/README.md
@@ -170,6 +170,7 @@ Available settings:
 
 | Key                                                                                      | Type   | Example             | Description                                                          |
 |------------------------------------------------------------------------------------------|--------|---------------------|----------------------------------------------------------------------|
+| awsRole                                                                                  | string |                     |  AWS Role |
 | postgres.`<module-name>`.uri                                                             | URI    |                     | Postgres database configuration                                      |
 | elasticsearch.dsn                                                                        | URI    |                     | Elasticsearch connection URI                                         |
 | temporal.dsn                                                                             | URI    |                     | Temporal URI                                                         |

diff --git a/components/operator/internal/resources/auths/controller.go b/components/operator/internal/resources/auths/controller.go
@@ -23,6 +23,7 @@ import (
 	"github.com/formancehq/operator/internal/resources/gatewayhttpapis"
 	"github.com/formancehq/operator/internal/resources/jobs"
 	"github.com/formancehq/operator/internal/resources/registries"
+	"github.com/formancehq/operator/internal/resources/settings"
 	. "github.com/formancehq/stack/libs/go-libs/collectionutils"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
@@ -59,10 +60,20 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, auth *v1beta1.Auth, version st
 			return errors.Wrap(err, "resolving image")
 		}
 
+		serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+		if err != nil {
+			return errors.Wrap(err, "getting service account name")
+		}
+
+		migrateContainer, err := databases.MigrateDatabaseContainer(ctx, stack, image, database)
+		if err != nil {
+			return errors.Wrap(err, "creating migrate container")
+		}
+
 		if IsGreaterOrEqual(version, "v2.0.0-rc.5") && databases.GetSavedModuleVersion(database) != version {
 			if err := jobs.Handle(ctx, auth, "migrate",
-				databases.MigrateDatabaseContainer(image, database),
-				jobs.WithServiceAccount(database.Status.URI.Query().Get("awsRole")),
+				migrateContainer,
+				jobs.WithServiceAccount(serviceAccountName),
 			); err != nil {
 				return err
 			}

diff --git a/components/operator/internal/resources/auths/deployment.go b/components/operator/internal/resources/auths/deployment.go
@@ -30,9 +30,14 @@ func createDeployment(ctx Context, stack *v1beta1.Stack, auth *v1beta1.Auth, dat
 		return nil, err
 	}
 
+	postgresEnvVar, err := databases.GetPostgresEnvVars(ctx, stack, database)
+	if err != nil {
+		return nil, err
+	}
+
 	env = append(env, gatewayEnv...)
 	env = append(env, GetDevEnvVars(stack, auth)...)
-	env = append(env, databases.GetPostgresEnvVars(database)...)
+	env = append(env, postgresEnvVar...)
 	env = append(env, Env("CONFIG", "/config/config.yaml"))
 
 	authUrl, err := getUrl(ctx, stack.Name)
@@ -66,10 +71,15 @@ func createDeployment(ctx Context, stack *v1beta1.Stack, auth *v1beta1.Auth, dat
 		env = append(env, Env("CAOS_OIDC_DEV", "1"))
 	}
 
+	serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return nil, err
+	}
+
 	return deployments.CreateOrUpdate(ctx, auth, "auth",
 		deployments.WithMatchingLabels("auth"),
 		deployments.WithReplicasFromSettings(ctx, stack),
-		deployments.WithServiceAccountName(database.Status.URI.Query().Get("awsRole")),
+		deployments.WithServiceAccountName(serviceAccountName),
 		func(t *appsv1.Deployment) error {
 			t.Spec.Template.Annotations = MergeMaps(t.Spec.Template.Annotations, map[string]string{
 				"config-hash": HashFromConfigMaps(configMap),

diff --git a/components/operator/internal/resources/benthos/controller.go b/components/operator/internal/resources/benthos/controller.go
@@ -19,9 +19,10 @@ package benthos
 import (
 	"embed"
 	"fmt"
-	"github.com/formancehq/operator/internal/resources/resourcereferences"
 	"sort"
 
+	"github.com/formancehq/operator/internal/resources/resourcereferences"
+
 	"github.com/formancehq/operator/internal/resources/services"
 	"github.com/formancehq/operator/internal/resources/settings"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -281,9 +282,15 @@ func createDeployment(ctx Context, stack *v1beta1.Stack, b *v1beta1.Benthos) err
 		return streams[i].Name < streams[j].Name
 	})
 
+	serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return err
+	}
+
 	_, err = deployments.CreateOrUpdate(ctx, b, "benthos",
 		resourcereferences.Annotate[*appsv1.Deployment]("elasticsearch-secret-hash", resourceReference),
 		deployments.WithMatchingLabels("benthos"),
+		deployments.WithServiceAccountName(serviceAccountName),
 		deployments.WithInitContainers(b.Spec.InitContainers...),
 		deployments.WithContainers(corev1.Container{
 			Name:    "benthos",

diff --git a/components/operator/internal/resources/databases/controller.go b/components/operator/internal/resources/databases/controller.go
@@ -18,6 +18,7 @@ package databases
 
 import (
 	"fmt"
+
 	"github.com/formancehq/operator/api/formance.com/v1beta1"
 	"github.com/formancehq/operator/internal/core"
 	"github.com/formancehq/operator/internal/resources/jobs"
@@ -57,7 +58,12 @@ func Reconcile(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Databas
 		return err
 	}
 
-	if awsRole := databaseURL.Query().Get("awsRole"); awsRole != "" {
+	awsRole, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return err
+	}
+
+	if awsRole != "" {
 		_, err = resourcereferences.Create(ctx, database, "database", awsRole, &v1.ServiceAccount{})
 	} else {
 		err = resourcereferences.Delete(ctx, database, "database")
@@ -152,19 +158,28 @@ func handleDatabaseJob(ctx core.Context, stack *v1beta1.Stack, database *v1beta1
 		annotations["secret-hash"] = secretReference.Status.Hash
 	}
 
-	env := GetPostgresEnvVars(database)
+	env, err := GetPostgresEnvVars(ctx, stack, database)
+	if err != nil {
+		return err
+	}
+
 	if database.Spec.Debug {
 		env = append(env, core.Env("DEBUG", "true"))
 	}
 
+	serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return err
+	}
+
 	return jobs.Handle(ctx, database, name, v1.Container{
 		Name:  name,
 		Image: operatorUtilsImage,
 		Args:  args,
 		Env:   env,
 	},
 		jobs.Mutator(core.WithAnnotations[*batchv1.Job](annotations)),
-		jobs.WithServiceAccount(database.Status.URI.Query().Get("awsRole")),
+		jobs.WithServiceAccount(serviceAccountName),
 	)
 }
 

diff --git a/components/operator/internal/resources/databases/env.go b/components/operator/internal/resources/databases/env.go
@@ -2,17 +2,18 @@ package databases
 
 import (
 	"fmt"
+
 	"github.com/formancehq/operator/api/formance.com/v1beta1"
 	"github.com/formancehq/operator/internal/core"
 	"github.com/formancehq/operator/internal/resources/settings"
 	corev1 "k8s.io/api/core/v1"
 )
 
-func GetPostgresEnvVars(db *v1beta1.Database) []corev1.EnvVar {
-	return PostgresEnvVarsWithPrefix(db, "")
+func GetPostgresEnvVars(ctx core.Context, stack *v1beta1.Stack, db *v1beta1.Database) ([]corev1.EnvVar, error) {
+	return PostgresEnvVarsWithPrefix(ctx, stack, db, "")
 }
 
-func PostgresEnvVarsWithPrefix(database *v1beta1.Database, prefix string) []corev1.EnvVar {
+func PostgresEnvVarsWithPrefix(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Database, prefix string) ([]corev1.EnvVar, error) {
 	ret := []corev1.EnvVar{
 		core.Env(fmt.Sprintf("%sPOSTGRES_HOST", prefix), database.Status.URI.Hostname()),
 		core.Env(fmt.Sprintf("%sPOSTGRES_PORT", prefix), database.Status.URI.Port()),
@@ -48,7 +49,13 @@ func PostgresEnvVarsWithPrefix(database *v1beta1.Database, prefix string) []core
 			)),
 		)
 	}
-	if awsRole := database.Status.URI.Query().Get("awsRole"); awsRole != "" {
+
+	awsRole, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	if awsRole != "" {
 		ret = append(ret, core.Env(fmt.Sprintf("%sPOSTGRES_AWS_ENABLE_IAM", prefix), "true"))
 	}
 
@@ -62,5 +69,5 @@ func PostgresEnvVarsWithPrefix(database *v1beta1.Database, prefix string) []core
 			fmt.Sprintf("%sPOSTGRES_DATABASE", prefix))),
 	)
 
-	return ret
+	return ret, nil
 }
diff --git a/components/operator/internal/resources/databases/migrate.go b/components/operator/internal/resources/databases/migrate.go
@@ -2,6 +2,7 @@ package databases
 
 import (
 	"fmt"
+
 	"github.com/formancehq/operator/api/formance.com/v1beta1"
 	"github.com/formancehq/operator/internal/core"
 	"github.com/formancehq/operator/internal/resources/jobs"
@@ -13,7 +14,7 @@ type MigrationConfiguration struct {
 	AdditionalEnv []v1.EnvVar
 }
 
-func MigrateDatabaseContainer(image string, database *v1beta1.Database, options ...func(m *MigrationConfiguration)) v1.Container {
+func MigrateDatabaseContainer(ctx core.Context, stack *v1beta1.Stack, image string, database *v1beta1.Database, options ...func(m *MigrationConfiguration)) (v1.Container, error) {
 	m := &MigrationConfiguration{}
 	for _, option := range options {
 		option(m)
@@ -23,7 +24,11 @@ func MigrateDatabaseContainer(image string, database *v1beta1.Database, options
 		args = []string{"migrate"}
 	}
 
-	env := GetPostgresEnvVars(database)
+	env, err := GetPostgresEnvVars(ctx, stack, database)
+	if err != nil {
+		return v1.Container{}, err
+	}
+
 	if m.AdditionalEnv != nil {
 		env = append(env, m.AdditionalEnv...)
 	}
@@ -33,9 +38,14 @@ func MigrateDatabaseContainer(image string, database *v1beta1.Database, options
 		Image: image,
 		Args:  args,
 		Env:   env,
-	}
+	}, nil
 }
 
-func Migrate(ctx core.Context, image string, database *v1beta1.Database, options ...func(m *MigrationConfiguration)) error {
-	return jobs.Handle(ctx, database, fmt.Sprintf("%s-migration", database.Name), MigrateDatabaseContainer(image, database, options...))
+func Migrate(ctx core.Context, stack *v1beta1.Stack, image string, database *v1beta1.Database, options ...func(m *MigrationConfiguration)) error {
+	container, err := MigrateDatabaseContainer(ctx, stack, image, database, options...)
+	if err != nil {
+		return err
+	}
+
+	return jobs.Handle(ctx, database, fmt.Sprintf("%s-migration", database.Name), container)
 }
diff --git a/components/operator/internal/resources/ledgers/deployments.go b/components/operator/internal/resources/ledgers/deployments.go
@@ -2,19 +2,18 @@ package ledgers
 
 import (
 	"fmt"
-	"github.com/formancehq/operator/internal/resources/jobs"
 	"strconv"
 
-	"github.com/formancehq/operator/internal/resources/settings"
-
 	"github.com/formancehq/operator/api/formance.com/v1beta1"
 	"github.com/formancehq/operator/internal/core"
 	"github.com/formancehq/operator/internal/resources/auths"
 	"github.com/formancehq/operator/internal/resources/brokertopics"
 	"github.com/formancehq/operator/internal/resources/databases"
 	"github.com/formancehq/operator/internal/resources/deployments"
 	"github.com/formancehq/operator/internal/resources/gateways"
+	"github.com/formancehq/operator/internal/resources/jobs"
 	"github.com/formancehq/operator/internal/resources/services"
+	"github.com/formancehq/operator/internal/resources/settings"
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -67,15 +66,15 @@ func installLedgerSingleInstance(ctx core.Context, stack *v1beta1.Stack,
 		}
 	}
 
-	if err := createDeployment(ctx, ledger, database, "ledger", *container, v2, deployments.WithReplicas(1)); err != nil {
+	if err := createDeployment(ctx, stack, ledger, database, "ledger", *container, v2, deployments.WithReplicas(1)); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func getUpgradeContainer(database *v1beta1.Database, image, version string) corev1.Container {
-	return databases.MigrateDatabaseContainer(image, database,
+func getUpgradeContainer(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Database, image, version string) (corev1.Container, error) {
+	return databases.MigrateDatabaseContainer(ctx, stack, image, database,
 		func(m *databases.MigrationConfiguration) {
 			if core.IsLower(version, "v2.0.0-rc.6") {
 				m.Command = []string{"buckets", "upgrade-all"}
@@ -95,7 +94,7 @@ func installLedgerMonoWriterMultipleReader(ctx core.Context, stack *v1beta1.Stac
 			return err
 		}
 
-		if err := createDeployment(ctx, ledger, database, name, container, v2, mutators...); err != nil {
+		if err := createDeployment(ctx, stack, ledger, database, name, container, v2, mutators...); err != nil {
 			return err
 		}
 
@@ -156,12 +155,17 @@ func uninstallLedgerMonoWriterMultipleReader(ctx core.Context, stack *v1beta1.St
 	return nil
 }
 
-func createDeployment(ctx core.Context, ledger *v1beta1.Ledger, database *v1beta1.Database,
+func createDeployment(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, database *v1beta1.Database,
 	name string, container corev1.Container, v2 bool, mutators ...core.ObjectMutator[*v1.Deployment]) error {
+	serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return err
+	}
+
 	mutators = append([]core.ObjectMutator[*v1.Deployment]{
 		deployments.WithContainers(container),
 		deployments.WithMatchingLabels(name),
-		deployments.WithServiceAccountName(database.Status.URI.Query().Get("awsRole")),
+		deployments.WithServiceAccountName(serviceAccountName),
 		func(t *v1.Deployment) error {
 			if !v2 {
 				t.Spec.Template.Spec.Volumes = []corev1.Volume{{
@@ -175,7 +179,7 @@ func createDeployment(ctx core.Context, ledger *v1beta1.Ledger, database *v1beta
 		},
 	}, mutators...)
 
-	_, err := deployments.CreateOrUpdate(ctx, ledger, name, mutators...)
+	_, err = deployments.CreateOrUpdate(ctx, ledger, name, mutators...)
 	return err
 }
 
@@ -205,9 +209,14 @@ func setCommonContainerConfiguration(ctx core.Context, stack *v1beta1.Stack, led
 	}
 	env = append(env, authEnvVars...)
 
+	postgresEnvVar, err := databases.PostgresEnvVarsWithPrefix(ctx, stack, database, prefix)
+	if err != nil {
+		return err
+	}
+	env = append(env, postgresEnvVar...)
+
 	container.Image = image
 	container.Env = append(container.Env, env...)
-	container.Env = append(container.Env, databases.PostgresEnvVarsWithPrefix(database, prefix)...)
 	container.Env = append(container.Env, core.Env(fmt.Sprintf("%sSTORAGE_POSTGRES_CONN_STRING", prefix), fmt.Sprintf("$(%sPOSTGRES_URI)", prefix)))
 	container.Env = append(container.Env, core.Env(fmt.Sprintf("%sSTORAGE_DRIVER", prefix), "postgres"))
 	container.Ports = []corev1.ContainerPort{deployments.StandardHTTPPort()}
@@ -301,7 +310,17 @@ func createGatewayDeployment(ctx core.Context, stack *v1beta1.Stack, ledger *v1b
 }
 
 func migrate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, database *v1beta1.Database, image, version string) error {
-	return jobs.Handle(ctx, ledger, "migrate-v2", getUpgradeContainer(database, image, version),
+	serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+	if err != nil {
+		return err
+	}
+
+	upgradeContainer, err := getUpgradeContainer(ctx, stack, database, image, version)
+	if err != nil {
+		return err
+	}
+
+	return jobs.Handle(ctx, ledger, "migrate-v2", upgradeContainer,
 		jobs.PreCreate(func() error {
 			list := &v1.DeploymentList{}
 			if err := ctx.GetClient().List(ctx, list, client.InNamespace(stack.Name)); err != nil {
@@ -317,5 +336,6 @@ func migrate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, dat
 			}
 			return nil
 		}),
-		jobs.WithServiceAccount(database.Status.URI.Query().Get("awsRole")))
+		jobs.WithServiceAccount(serviceAccountName),
+	)
 }
diff --git a/components/operator/internal/resources/orchestrations/controller.go b/components/operator/internal/resources/orchestrations/controller.go
@@ -25,6 +25,7 @@ import (
 	"github.com/formancehq/operator/internal/resources/gatewayhttpapis"
 	"github.com/formancehq/operator/internal/resources/jobs"
 	"github.com/formancehq/operator/internal/resources/registries"
+	"github.com/formancehq/operator/internal/resources/settings"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -61,10 +62,20 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, o *v1beta1.Orchestration, vers
 			return errors.Wrap(err, "resolving image")
 		}
 
+		serviceAccountName, err := settings.GetAWSRole(ctx, stack.Name)
+		if err != nil {
+			return errors.Wrap(err, "getting service account name")
+		}
+
+		migrateContainer, err := databases.MigrateDatabaseContainer(ctx, stack, image, database)
+		if err != nil {
+			return errors.Wrap(err, "creating migrate container")
+		}
+
 		if IsGreaterOrEqual(version, "v2.0.0-rc.5") && databases.GetSavedModuleVersion(database) != version {
 			if err := jobs.Handle(ctx, o, "migrate",
-				databases.MigrateDatabaseContainer(image, database),
-				jobs.WithServiceAccount(database.Status.URI.Query().Get("awsRole")),
+				migrateContainer,
+				jobs.WithServiceAccount(serviceAccountName),
 			); err != nil {
 				return err
 			}