From c51d724a9d93e87456d8c68231f3c39df170e898 Mon Sep 17 00:00:00 2001 From: Amit Lichtenberg Date: Thu, 7 Mar 2024 09:55:00 +0200 Subject: [PATCH] Support for azure credentials operator (#368) Co-authored-by: omris94 <46892443+omris94@users.noreply.github.com> --- helm-charts | 2 +- src/go.mod | 4 + src/go.sum | 8 ++ .../controllers/iam_pod_reconciler/pods.go | 10 +- .../intents_reconcilers/iam_reconciler.go | 8 +- .../iam_reconciler_test.go | 8 +- .../mocks/mock_policy_agent.go | 15 +-- src/shared/agentutils/agentutils.go | 27 +++++ src/shared/awsagent/agent.go | 13 +-- src/shared/awsagent/credentials.go | 107 ++++++++++++++++++ src/shared/awsagent/policies.go | 3 +- src/shared/awsagent/roles.go | 15 +-- src/shared/awsagent/types.go | 2 - src/shared/azureagent/agent.go | 63 +++++++++-- src/shared/azureagent/credentials.go | 79 +++++++++++++ src/shared/azureagent/identities.go | 104 ++++++++++++++--- src/shared/azureagent/policies.go | 4 - src/shared/gcpagent/agent.go | 5 - src/shared/gcpagent/credentials.go | 82 ++++++++++++++ src/shared/gcpagent/utils.go | 39 ++----- 20 files changed, 489 insertions(+), 109 deletions(-) create mode 100644 src/shared/agentutils/agentutils.go create mode 100644 src/shared/awsagent/credentials.go create mode 100644 src/shared/azureagent/credentials.go create mode 100644 src/shared/gcpagent/credentials.go diff --git a/helm-charts b/helm-charts index 0742b73bf..062bb1c57 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 0742b73bf38f292819befeee0828f3d4e1b6fdcc +Subproject commit 062bb1c5797671e7f9bd910b6e3a9edd726574d9 diff --git a/src/go.mod b/src/go.mod index c68f9fd8e..4ede8fcee 100644 --- a/src/go.mod +++ b/src/go.mod @@ -50,8 +50,12 @@ require ( require ( cloud.google.com/go/compute v1.23.0 // indirect + github.com/Azure/azure-sdk-for-go-extensions v0.1.6 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alexflint/go-arg v1.4.2 // indirect diff --git a/src/go.sum b/src/go.sum index 41d69098f..9cc87c63b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -41,6 +41,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.17.2/go.mod h1:K5fzLKwtph+FFgh9j7nFbRUdBKvTcGnsta51fsMTn3o= +github.com/Azure/azure-sdk-for-go-extensions v0.1.6 h1:EXGvDcj54u98XfaI/Cy65Ds6vNsIJeGKYf0eNLB1y4Q= +github.com/Azure/azure-sdk-for-go-extensions v0.1.6/go.mod h1:27StPiXJp6Xzkq2AQL7gPK7VC0hgmCnUKlco1dO1jaM= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= @@ -49,8 +51,14 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xO github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0 h1:g65N4m1sAjm0BkjIJYtp5qnJlkoFtd6oqfa27KO9fI4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0/go.mod h1:noQIdW75SiQFB3mSFJBr4iRRH83S9skaFiBv4C0uEs0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 h1:z4YeiSXxnUI+PqB46Yj6MZA3nwb1CcJIkEMDrzUd8Cs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0/go.mod h1:rko9SzMxcMk0NJsNAxALEGaTYyy79bNRwxgJfrH0Spw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/src/operator/controllers/iam_pod_reconciler/pods.go b/src/operator/controllers/iam_pod_reconciler/pods.go index 5558e6014..95e8f9822 100644 --- a/src/operator/controllers/iam_pod_reconciler/pods.go +++ b/src/operator/controllers/iam_pod_reconciler/pods.go @@ -74,11 +74,19 @@ func (p *IAMPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, errors.Wrap(err) } + logger.Infof("Found %d intents for service", len(intents.Items)) + for _, intent := range intents.Items { - return p.iamReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ + result, err := p.iamReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ Name: intent.Name, Namespace: intent.Namespace, }}) + if err != nil { + return ctrl.Result{}, errors.Wrap(err) + } + if result.Requeue { + return result, nil + } } return ctrl.Result{}, nil diff --git a/src/operator/controllers/intents_reconcilers/iam_reconciler.go b/src/operator/controllers/intents_reconcilers/iam_reconciler.go index c64a6474b..2c522b5e1 100644 --- a/src/operator/controllers/intents_reconcilers/iam_reconciler.go +++ b/src/operator/controllers/intents_reconcilers/iam_reconciler.go @@ -19,7 +19,7 @@ import ( type IAMPolicyAgent interface { IntentType() otterizev1alpha3.IntentType - ApplyOnPodLabel() string + AppliesOnPod(pod *corev1.Pod) bool AddRolePolicyFromIntents(ctx context.Context, namespace string, accountName string, intentsServiceName string, intents []otterizev1alpha3.Intent) error DeleteRolePolicyFromIntents(ctx context.Context, intents otterizev1alpha3.ClientIntents) error } @@ -104,11 +104,7 @@ func (r *IAMIntentsReconciler) Reconcile(ctx context.Context, req reconcile.Requ } func (r *IAMIntentsReconciler) applyTypedIAMIntents(ctx context.Context, pod corev1.Pod, intents otterizev1alpha3.ClientIntents, agent IAMPolicyAgent) error { - if pod.Labels == nil { - return nil - } - - if pod.Labels[agent.ApplyOnPodLabel()] != "true" { + if !agent.AppliesOnPod(&pod) { return nil } diff --git a/src/operator/controllers/intents_reconcilers/iam_reconciler_test.go b/src/operator/controllers/intents_reconcilers/iam_reconciler_test.go index ad8540698..95a01c5a1 100644 --- a/src/operator/controllers/intents_reconcilers/iam_reconciler_test.go +++ b/src/operator/controllers/intents_reconcilers/iam_reconciler_test.go @@ -147,7 +147,7 @@ func (s *IAMIntentsReconcilerTestSuite) TestCreateIAMIntentCallingTheGCPAgent() ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: testNamespace, - Labels: map[string]string{gcpagent.GCPPodLabel: "true"}, + Labels: map[string]string{gcpagent.GCPApplyOnPodLabel: "true"}, }, Spec: corev1.PodSpec{ ServiceAccountName: clientServiceAccount, @@ -167,7 +167,7 @@ func (s *IAMIntentsReconcilerTestSuite) TestCreateIAMIntentCallingTheGCPAgent() ) s.serviceResolver.EXPECT().ResolveClientIntentToPod(gomock.Any(), gomock.Eq(gcpIntents)).Return(clientPod, nil) - s.gcpAgent.EXPECT().ApplyOnPodLabel().Return(gcpagent.GCPPodLabel) + s.gcpAgent.EXPECT().AppliesOnPod(gomock.AssignableToTypeOf(&clientPod)).Return(true) s.gcpAgent.EXPECT().IntentType().Return(otterizev1alpha3.IntentTypeGCP) s.Client.EXPECT().List( gomock.Any(), @@ -221,7 +221,7 @@ func (s *IAMIntentsReconcilerTestSuite) TestCreateIAMIntentPartialDeleteCallingT ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: testNamespace, - Labels: map[string]string{gcpagent.GCPPodLabel: "true"}, + Labels: map[string]string{gcpagent.GCPApplyOnPodLabel: "true"}, }, Spec: corev1.PodSpec{ ServiceAccountName: clientServiceAccount, @@ -241,7 +241,7 @@ func (s *IAMIntentsReconcilerTestSuite) TestCreateIAMIntentPartialDeleteCallingT ) s.serviceResolver.EXPECT().ResolveClientIntentToPod(gomock.Any(), gomock.Eq(clientIntents)).Return(clientPod, nil) - s.gcpAgent.EXPECT().ApplyOnPodLabel().Return(gcpagent.GCPPodLabel) + s.gcpAgent.EXPECT().AppliesOnPod(gomock.AssignableToTypeOf(&clientPod)).Return(true) s.gcpAgent.EXPECT().IntentType().Return(otterizev1alpha3.IntentTypeGCP) s.Client.EXPECT().List( gomock.Any(), diff --git a/src/operator/controllers/intents_reconcilers/mocks/mock_policy_agent.go b/src/operator/controllers/intents_reconcilers/mocks/mock_policy_agent.go index b5cf46acb..fb19a50ad 100644 --- a/src/operator/controllers/intents_reconcilers/mocks/mock_policy_agent.go +++ b/src/operator/controllers/intents_reconcilers/mocks/mock_policy_agent.go @@ -10,6 +10,7 @@ import ( v1alpha3 "github.com/otterize/intents-operator/src/operator/api/v1alpha3" gomock "go.uber.org/mock/gomock" + v1 "k8s.io/api/core/v1" ) // MockIAMPolicyAgent is a mock of IAMPolicyAgent interface. @@ -49,18 +50,18 @@ func (mr *MockIAMPolicyAgentMockRecorder) AddRolePolicyFromIntents(ctx, namespac return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRolePolicyFromIntents", reflect.TypeOf((*MockIAMPolicyAgent)(nil).AddRolePolicyFromIntents), ctx, namespace, accountName, intentsServiceName, intents) } -// ApplyOnPodLabel mocks base method. -func (m *MockIAMPolicyAgent) ApplyOnPodLabel() string { +// AppliesOnPod mocks base method. +func (m *MockIAMPolicyAgent) AppliesOnPod(pod *v1.Pod) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ApplyOnPodLabel") - ret0, _ := ret[0].(string) + ret := m.ctrl.Call(m, "AppliesOnPod", pod) + ret0, _ := ret[0].(bool) return ret0 } -// ApplyOnPodLabel indicates an expected call of ApplyOnPodLabel. -func (mr *MockIAMPolicyAgentMockRecorder) ApplyOnPodLabel() *gomock.Call { +// AppliesOnPod indicates an expected call of AppliesOnPod. +func (mr *MockIAMPolicyAgentMockRecorder) AppliesOnPod(pod interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyOnPodLabel", reflect.TypeOf((*MockIAMPolicyAgent)(nil).ApplyOnPodLabel)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppliesOnPod", reflect.TypeOf((*MockIAMPolicyAgent)(nil).AppliesOnPod), pod) } // DeleteRolePolicyFromIntents mocks base method. diff --git a/src/shared/agentutils/agentutils.go b/src/shared/agentutils/agentutils.go new file mode 100644 index 000000000..f6b639b58 --- /dev/null +++ b/src/shared/agentutils/agentutils.go @@ -0,0 +1,27 @@ +package agentutils + +import ( + "crypto/sha256" + "fmt" +) + +const ( + truncatedHashLength = 6 +) + +// TruncateHashName truncates the given name to the given max length and appends a hash to it. +func TruncateHashName(fullName string, maxLen int) string { + maxTruncatedLen := maxLen - truncatedHashLength - 1 // add another char for the hyphen + + var truncatedName string + if len(fullName) >= maxTruncatedLen { + truncatedName = fullName[:maxTruncatedLen] + } else { + truncatedName = fullName + } + + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(fullName))) + truncatedHash := hash[:truncatedHashLength] + + return fmt.Sprintf("%s-%s", truncatedName, truncatedHash) +} diff --git a/src/shared/awsagent/agent.go b/src/shared/awsagent/agent.go index dc3a0ff7d..bd9933ecf 100644 --- a/src/shared/awsagent/agent.go +++ b/src/shared/awsagent/agent.go @@ -41,10 +41,11 @@ type IAMClient interface { } type Agent struct { - iamClient IAMClient - accountID string - oidcURL string - clusterName string + iamClient IAMClient + accountID string + oidcURL string + clusterName string + markRolesAsUnusedInsteadOfDelete bool } func NewAWSAgent( @@ -131,7 +132,3 @@ func getCurrentEKSCluster(ctx context.Context, config aws.Config) (*eksTypes.Clu return describeClusterOutput.Cluster, nil } - -func (a *Agent) ApplyOnPodLabel() string { - return "credentials-operator.otterize.com/create-aws-role" -} diff --git a/src/shared/awsagent/credentials.go b/src/shared/awsagent/credentials.go new file mode 100644 index 000000000..bede69180 --- /dev/null +++ b/src/shared/awsagent/credentials.go @@ -0,0 +1,107 @@ +package awsagent + +import ( + "context" + "github.com/otterize/intents-operator/src/shared/errors" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" +) + +const ( + // AWSApplyOnPodLabel is used to mark pods that should be processed by the AWS agent to create an associated AWS role + AWSApplyOnPodLabel = "credentials-operator.otterize.com/create-aws-role" + + // ServiceManagedByAWSAgentLabel is used to mark service accounts that are managed by the AWS agent + ServiceManagedByAWSAgentLabel = "credentials-operator.otterize.com/managed-by-aws-agent" + + // ServiceAccountAWSRoleARNAnnotation is used by EKS (Kubernetes at AWS) to link between service accounts + // and IAM roles + ServiceAccountAWSRoleARNAnnotation = "eks.amazonaws.com/role-arn" + + // OtterizeServiceAccountAWSRoleARNAnnotation is used to update a Pod in the mutating webhook with the role ARN + // so that reinvocation is triggered for the EKS pod identity mutating webhook. + OtterizeServiceAccountAWSRoleARNAnnotation = "credentials-operator.otterize.com/eks-role-arn" + + // OtterizeAWSUseSoftDeleteKey is used to mark workloads that should not have their corresponding roles deleted, + // but should be tagged as deleted instead (aka soft delete strategy). + OtterizeAWSUseSoftDeleteKey = "credentials-operator.otterize.com/aws-use-soft-delete" +) + +func (a *Agent) SetSoftDeleteStrategy(markRolesAsUnusedInsteadOfDelete bool) { + a.markRolesAsUnusedInsteadOfDelete = markRolesAsUnusedInsteadOfDelete +} + +func (a *Agent) AppliesOnPod(pod *corev1.Pod) bool { + return pod.Labels != nil && pod.Labels[AWSApplyOnPodLabel] == "true" +} + +func (a *Agent) OnPodAdmission(ctx context.Context, pod *corev1.Pod, serviceAccount *corev1.ServiceAccount) (updated bool, err error) { + if !a.AppliesOnPod(pod) { + return false, nil + } + + serviceAccount.Labels[ServiceManagedByAWSAgentLabel] = "true" + + roleArn := a.GenerateRoleARN(serviceAccount.Namespace, serviceAccount.Name) + serviceAccount.Annotations[ServiceAccountAWSRoleARNAnnotation] = roleArn + pod.Annotations[OtterizeServiceAccountAWSRoleARNAnnotation] = roleArn + + podUseSoftDeleteLabelValue, podUseSoftDeleteLabelExists := pod.Labels[OtterizeAWSUseSoftDeleteKey] + shouldMarkForSoftDelete := podUseSoftDeleteLabelExists && podUseSoftDeleteLabelValue == "true" + if shouldMarkForSoftDelete { + serviceAccount.Labels[OtterizeAWSUseSoftDeleteKey] = "true" + } else { + delete(serviceAccount.Labels, OtterizeAWSUseSoftDeleteKey) + } + + return true, nil +} + +func (a *Agent) OnServiceAccountUpdate(ctx context.Context, serviceAccount *corev1.ServiceAccount) (updated bool, requeue bool, err error) { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if serviceAccount.Labels == nil || serviceAccount.Labels[ServiceManagedByAWSAgentLabel] != "true" { + logger.Debug("ServiceAccount is not managed by the AWS agent, skipping") + return false, false, nil + } + + useSoftDeleteStrategy := a.shouldUseSoftDeleteStrategy(serviceAccount) + + // calling create in any case because this way we validate it is not soft-deleted and it is configured with the correct soft-delete strategy + role, err := a.CreateOtterizeIAMRole(ctx, serviceAccount.Namespace, serviceAccount.Name, useSoftDeleteStrategy) + if err != nil { + return false, false, errors.Errorf("failed creating AWS role for service account: %w", err) + } + logger.WithField("arn", *role.Arn).Info("created AWS role for ServiceAccount") + + roleARN, ok := serviceAccount.Annotations[ServiceAccountAWSRoleARNAnnotation] + + // update annotation if it doesn't exist or if it is misconfigured + shouldUpdate := !ok || roleARN != *role.Arn + + serviceAccount.Annotations[ServiceAccountAWSRoleARNAnnotation] = *role.Arn + return shouldUpdate, false, nil +} + +func (a *Agent) shouldUseSoftDeleteStrategy(serviceAccount *corev1.ServiceAccount) bool { + if a.markRolesAsUnusedInsteadOfDelete { + return true + } + if serviceAccount.Labels == nil { + return false + } + + softDeleteValue, shouldSoftDelete := serviceAccount.Labels[OtterizeAWSUseSoftDeleteKey] + return shouldSoftDelete && softDeleteValue == "true" +} + +func (a *Agent) OnServiceAccountTermination(ctx context.Context, serviceAccount *corev1.ServiceAccount) error { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if serviceAccount.Labels == nil || serviceAccount.Labels[ServiceManagedByAWSAgentLabel] != "true" { + logger.Debug("ServiceAccount is not managed by the Azure agent, skipping") + return nil + } + + return a.DeleteOtterizeIAMRole(ctx, serviceAccount.Namespace, serviceAccount.Name) +} diff --git a/src/shared/awsagent/policies.go b/src/shared/awsagent/policies.go index afdd313ea..94b7da1ac 100644 --- a/src/shared/awsagent/policies.go +++ b/src/shared/awsagent/policies.go @@ -202,8 +202,7 @@ func (a *Agent) SetRolePolicy(ctx context.Context, namespace, accountName string } if !exists { - errorMessage := fmt.Sprintf("role not found: %s", roleName) - return errors.New(errorMessage) + return errors.Errorf("role not found: %s", roleName) } policyDoc, _, err := generatePolicyDocument(statements) diff --git a/src/shared/awsagent/roles.go b/src/shared/awsagent/roles.go index 702e4af82..614d96ca1 100644 --- a/src/shared/awsagent/roles.go +++ b/src/shared/awsagent/roles.go @@ -2,12 +2,12 @@ package awsagent import ( "context" - "crypto/sha256" "encoding/json" "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/aws/smithy-go" + "github.com/otterize/intents-operator/src/shared/agentutils" "github.com/otterize/intents-operator/src/shared/errors" "github.com/samber/lo" "github.com/sirupsen/logrus" @@ -323,18 +323,7 @@ func (a *Agent) generateTrustPolicy(namespaceName, accountName string) (string, func (a *Agent) generateRoleName(namespace string, accountName string) string { fullName := fmt.Sprintf("otr-%s.%s@%s", namespace, accountName, a.clusterName) - - var truncatedName string - if len(fullName) >= (maxTruncatedLength) { - truncatedName = fullName[:maxTruncatedLength] - } else { - truncatedName = fullName - } - - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(fullName))) - hash = hash[:truncatedHashLength] - - return fmt.Sprintf("%s-%s", truncatedName, hash) + return agentutils.TruncateHashName(fullName, maxAWSNameLength) } func (a *Agent) GenerateRoleARN(namespace string, accountName string) string { diff --git a/src/shared/awsagent/types.go b/src/shared/awsagent/types.go index 75f98148a..eed76cb66 100644 --- a/src/shared/awsagent/types.go +++ b/src/shared/awsagent/types.go @@ -33,5 +33,3 @@ type StatementEntry struct { } const maxAWSNameLength = 64 -const truncatedHashLength = 6 -const maxTruncatedLength = maxAWSNameLength - truncatedHashLength - 1 // add another char for the hyphen diff --git a/src/shared/azureagent/agent.go b/src/shared/azureagent/agent.go index 21069e9c0..84ef5e05a 100644 --- a/src/shared/azureagent/agent.go +++ b/src/shared/azureagent/agent.go @@ -4,23 +4,30 @@ import ( "context" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/otterize/intents-operator/src/shared/errors" "github.com/sirupsen/logrus" ) type Config struct { - SubscriptionID string - ResourceGroup string - AKSClusterName string + SubscriptionID string + ResourceGroup string + AKSClusterName string + Location string // optional, detected from ResourceGroup if not provided + AKSClusterOIDCIssuerURL string // optional, detected from AKS cluster if not provided } type Agent struct { conf Config credentials *azidentity.DefaultAzureCredential + resourceGroupsClient *armresources.ResourceGroupsClient userAssignedIdentitiesClient *armmsi.UserAssignedIdentitiesClient federatedIdentityCredentialsClient *armmsi.FederatedIdentityCredentialsClient roleDefinitionsClient *armauthorization.RoleDefinitionsClient roleAssignmentsClient *armauthorization.RoleAssignmentsClient + managedClustersClient *armcontainerservice.ManagedClustersClient } func NewAzureAgent(ctx context.Context, conf Config) (*Agent, error) { @@ -28,34 +35,70 @@ func NewAzureAgent(ctx context.Context, conf Config) (*Agent, error) { credentials, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { - return nil, err + return nil, errors.Wrap(err) } armmsiClientFactory, err := armmsi.NewClientFactory(conf.SubscriptionID, credentials, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err) } armauthorizationClientFactory, err := armauthorization.NewClientFactory(conf.SubscriptionID, credentials, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err) + } + + resourceGroupsClient, err := armresources.NewResourceGroupsClient(conf.SubscriptionID, credentials, nil) + if err != nil { + return nil, errors.Wrap(err) + } + + armcontainerserviceClientFactory, err := armcontainerservice.NewClientFactory(conf.SubscriptionID, credentials, nil) + if err != nil { + return nil, errors.Wrap(err) } userAssignedIdentitiesClient := armmsiClientFactory.NewUserAssignedIdentitiesClient() federatedIdentityCredentialsClient := armmsiClientFactory.NewFederatedIdentityCredentialsClient() roleDefinitionsClient := armauthorizationClientFactory.NewRoleDefinitionsClient() roleAssignmentsClient := armauthorizationClientFactory.NewRoleAssignmentsClient() + managedClustersClient := armcontainerserviceClientFactory.NewManagedClustersClient() - return &Agent{ + agent := &Agent{ conf: conf, credentials: credentials, + resourceGroupsClient: resourceGroupsClient, userAssignedIdentitiesClient: userAssignedIdentitiesClient, federatedIdentityCredentialsClient: federatedIdentityCredentialsClient, roleDefinitionsClient: roleDefinitionsClient, roleAssignmentsClient: roleAssignmentsClient, - }, nil + managedClustersClient: managedClustersClient, + } + + if err := agent.loadConfDefaults(ctx); err != nil { + return nil, errors.Wrap(err) + } + + return agent, nil } -func (a *Agent) ApplyOnPodLabel() string { - return "credentials-operator.otterize.com/create-azure-role-assignment" +func (a *Agent) loadConfDefaults(ctx context.Context) error { + if a.conf.Location == "" { + resourceGroup, err := a.resourceGroupsClient.Get(ctx, a.conf.ResourceGroup, nil) + if err != nil { + return errors.Errorf("error querying for resource group: %w", err) + } + + a.conf.Location = *resourceGroup.Location + } + if a.conf.AKSClusterOIDCIssuerURL == "" { + cluster, err := a.managedClustersClient.Get(ctx, a.conf.ResourceGroup, a.conf.AKSClusterName, nil) + if err != nil { + return errors.Errorf("error querying for managed cluster: %w", err) + } + + a.conf.AKSClusterOIDCIssuerURL = *cluster.Properties.OidcIssuerProfile.IssuerURL + } + + return nil } diff --git a/src/shared/azureagent/credentials.go b/src/shared/azureagent/credentials.go new file mode 100644 index 000000000..59bab6b10 --- /dev/null +++ b/src/shared/azureagent/credentials.go @@ -0,0 +1,79 @@ +package azureagent + +import ( + "context" + "github.com/otterize/intents-operator/src/shared/errors" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" +) + +const ( + // AzureApplyOnPodLabel is used to mark pods that should be processed by the Azure agent to create an associated Azure identity & role assignment + AzureApplyOnPodLabel = "credentials-operator.otterize.com/create-azure-role-assignment" + + // ServiceManagedByAzureAgentLabel is used to mark service accounts that are managed by the Azure agent + ServiceManagedByAzureAgentLabel = "credentials-operator.otterize.com/managed-by-azure-agent" + + // AzureUseWorkloadIdentityLabel is used by the azure workload identity mechanism to mark pods that should use workload identity + AzureUseWorkloadIdentityLabel = "azure.workload.identity/use" + + // AzureWorkloadIdentityClientIdAnnotation is used by the azure workload identity mechanism to link between service accounts and user assigned identities + AzureWorkloadIdentityClientIdAnnotation = "azure.workload.identity/client-id" +) + +func (a *Agent) AppliesOnPod(pod *corev1.Pod) bool { + return pod.Labels != nil && pod.Labels[AzureApplyOnPodLabel] == "true" +} + +func (a *Agent) OnPodAdmission(ctx context.Context, pod *corev1.Pod, serviceAccount *corev1.ServiceAccount) (updated bool, err error) { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if !a.AppliesOnPod(pod) { + return false, nil + } + + serviceAccount.Labels[ServiceManagedByAzureAgentLabel] = "true" + + pod.Labels[AzureUseWorkloadIdentityLabel] = "true" + + // get or create the user assigned identity, ensuring the identity & federated credentials are in-place + identity, err := a.getOrCreateUserAssignedIdentity(ctx, serviceAccount.Namespace, serviceAccount.Name) + if err != nil { + return false, errors.Errorf("failed to create user assigned identity: %w", err) + } + + clientId := *identity.Properties.ClientID + + if serviceAccount.Annotations[AzureWorkloadIdentityClientIdAnnotation] == clientId { + // existing identity matches the annotated identity + return false, nil + } + + logger.WithField("identity", *identity.Name).WithField("clientId", clientId). + Info("Annotating service account with managed identity client ID") + + serviceAccount.Annotations[AzureWorkloadIdentityClientIdAnnotation] = *identity.Properties.ClientID + return true, nil +} + +func (a *Agent) OnServiceAccountUpdate(ctx context.Context, serviceAccount *corev1.ServiceAccount) (updated bool, requeue bool, err error) { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if serviceAccount.Labels == nil || serviceAccount.Labels[ServiceManagedByAzureAgentLabel] != "true" { + logger.Debug("ServiceAccount is not managed by the Azure agent, skipping") + return false, false, nil + } + + return false, false, nil +} + +func (a *Agent) OnServiceAccountTermination(ctx context.Context, serviceAccount *corev1.ServiceAccount) error { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if serviceAccount.Labels == nil || serviceAccount.Labels[ServiceManagedByAzureAgentLabel] != "true" { + logger.Debug("ServiceAccount is not managed by the Azure agent, skipping") + return nil + } + + return a.deleteUserAssignedIdentity(ctx, serviceAccount.Namespace, serviceAccount.Name) +} diff --git a/src/shared/azureagent/identities.go b/src/shared/azureagent/identities.go index 73553a26e..f33806406 100644 --- a/src/shared/azureagent/identities.go +++ b/src/shared/azureagent/identities.go @@ -2,14 +2,41 @@ package azureagent import ( "context" - "crypto/sha256" "fmt" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + "github.com/otterize/intents-operator/src/shared/agentutils" "github.com/otterize/intents-operator/src/shared/errors" + "github.com/samber/lo" + "github.com/sirupsen/logrus" ) -func (a *Agent) findUserAssignedIdentity(ctx context.Context, namespaceName, accountName string) (armmsi.Identity, error) { - userAssignedIdentityName := a.generateUserAssignedIdentityName(namespaceName, accountName) +const ( + federatedIdentityAudience = "api://AzureADTokenExchange" + + // UserAssignedIdentities rules: + // 3-128 characters + // Alphanumerics, hyphens, and underscores + // Start with letter or number. + maxManagedIdentityLength = 128 + + // 3-120 characters + // Alphanumeric, dash, or underscore characters are supported + // the first character must be alphanumeric only + maxFederatedIdentityLength = 120 +) + +func (a *Agent) generateUserAssignedIdentityName(namespace string, accountName string) string { + fullName := fmt.Sprintf("ottr-uai-%s-%s-%s", namespace, accountName, a.conf.AKSClusterName) + return agentutils.TruncateHashName(fullName, maxManagedIdentityLength) +} + +func (a *Agent) generateFederatedIdentityCredentialsName(namespace string, accountName string) string { + fullName := fmt.Sprintf("ottr-fic-%s-%s-%s", namespace, accountName, a.conf.AKSClusterName) + return agentutils.TruncateHashName(fullName, maxFederatedIdentityLength) +} + +func (a *Agent) findUserAssignedIdentity(ctx context.Context, namespace string, accountName string) (armmsi.Identity, error) { + userAssignedIdentityName := a.generateUserAssignedIdentityName(namespace, accountName) userAssignedIdentity, err := a.userAssignedIdentitiesClient.Get(ctx, a.conf.ResourceGroup, userAssignedIdentityName, nil) if err != nil { return armmsi.Identity{}, errors.Wrap(err) @@ -18,22 +45,65 @@ func (a *Agent) findUserAssignedIdentity(ctx context.Context, namespaceName, acc return userAssignedIdentity.Identity, nil } -func (a *Agent) generateUserAssignedIdentityName(namespace string, accountName string) string { - // UserAssignedIdentities rules: - // 3-128 characters - // Alphanumerics, hyphens, and underscores - // Start with letter or number. - fullName := fmt.Sprintf("otr-%s-%s-%s", namespace, accountName, a.conf.AKSClusterName) +func (a *Agent) getOrCreateUserAssignedIdentity(ctx context.Context, namespace string, accountName string) (armmsi.Identity, error) { + logger := logrus.WithField("namespace", namespace).WithField("account", accountName) + userAssignedIdentityName := a.generateUserAssignedIdentityName(namespace, accountName) + logger.WithField("identity", userAssignedIdentityName).Debug("getting or creating user assigned identity") + userAssignedIdentity, err := a.userAssignedIdentitiesClient.CreateOrUpdate( + ctx, + a.conf.ResourceGroup, + userAssignedIdentityName, + armmsi.Identity{ + Location: &a.conf.Location, + }, + nil, + ) + if err != nil { + return armmsi.Identity{}, errors.Wrap(err) + } + + federatedIdentityCredentialsName := a.generateFederatedIdentityCredentialsName(namespace, accountName) + logger.WithField("federatedIdentity", federatedIdentityCredentialsName).Debug("getting or creating federated identity credentials") + _, err = a.federatedIdentityCredentialsClient.CreateOrUpdate( + ctx, + a.conf.ResourceGroup, + userAssignedIdentityName, + federatedIdentityCredentialsName, + armmsi.FederatedIdentityCredential{ + Properties: &armmsi.FederatedIdentityCredentialProperties{ + Issuer: lo.ToPtr(a.conf.AKSClusterOIDCIssuerURL), + Subject: lo.ToPtr( + fmt.Sprintf("system:serviceaccount:%s:%s", + namespace, + accountName)), + Audiences: []*string{lo.ToPtr(federatedIdentityAudience)}, + }, + }, + nil, + ) + if err != nil { + return armmsi.Identity{}, errors.Wrap(err) + } + + return userAssignedIdentity.Identity, nil +} - var truncatedName string - if len(fullName) >= (maxManagedIdentityTruncatedLength) { - truncatedName = fullName[:maxManagedIdentityTruncatedLength] - } else { - truncatedName = fullName +func (a *Agent) deleteUserAssignedIdentity(ctx context.Context, namespace string, accountName string) error { + logger := logrus.WithField("namespace", namespace).WithField("account", accountName) + userAssignedIdentityName := a.generateUserAssignedIdentityName(namespace, accountName) + federatedIdentityCredentialsName := a.generateFederatedIdentityCredentialsName(namespace, accountName) + + logger.WithField("federatedIdentity", federatedIdentityCredentialsName).Debug("deleting federated identity credentials") + _, err := a.federatedIdentityCredentialsClient.Delete(ctx, a.conf.ResourceGroup, userAssignedIdentityName, federatedIdentityCredentialsName, nil) + if err != nil { + return errors.Wrap(err) } - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(fullName))) - hash = hash[:truncatedHashLength] + logger.WithField("identity", userAssignedIdentityName).Debug("deleting user assigned identity") + _, err = a.userAssignedIdentitiesClient.Delete(ctx, a.conf.ResourceGroup, userAssignedIdentityName, nil) + if err != nil { + return errors.Wrap(err) + } - return fmt.Sprintf("%s-%s", truncatedName, hash) + return nil } diff --git a/src/shared/azureagent/policies.go b/src/shared/azureagent/policies.go index cc6406163..212dfd9e1 100644 --- a/src/shared/azureagent/policies.go +++ b/src/shared/azureagent/policies.go @@ -10,10 +10,6 @@ import ( "github.com/samber/lo" ) -const maxManagedIdentityLength = 128 -const truncatedHashLength = 6 -const maxManagedIdentityTruncatedLength = maxManagedIdentityLength - truncatedHashLength - 1 // add another char for the hyphen - func (a *Agent) IntentType() otterizev1alpha3.IntentType { return otterizev1alpha3.IntentTypeAzure } diff --git a/src/shared/gcpagent/agent.go b/src/shared/gcpagent/agent.go index 48e27548d..01f6815ff 100644 --- a/src/shared/gcpagent/agent.go +++ b/src/shared/gcpagent/agent.go @@ -12,7 +12,6 @@ import ( const ( EnvGcpProjectId = "gcp-project-id" EnvGcpGkeName = "gcp-eks-name" - GCPPodLabel = "credentials-operator.otterize.com/create-gcp-sa" ) type Agent struct { @@ -63,7 +62,3 @@ func getGCPAttribute(attribute string) (res string, err error) { } return res, nil } - -func (a *Agent) ApplyOnPodLabel() string { - return GCPPodLabel -} diff --git a/src/shared/gcpagent/credentials.go b/src/shared/gcpagent/credentials.go new file mode 100644 index 000000000..dff077acd --- /dev/null +++ b/src/shared/gcpagent/credentials.go @@ -0,0 +1,82 @@ +package gcpagent + +import ( + "context" + "github.com/otterize/intents-operator/src/shared/errors" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" +) + +const ( + // GCPApplyOnPodLabel is used to mark pods that should be processed by the GCP agent to create an associated GCP service account + GCPApplyOnPodLabel = "credentials-operator.otterize.com/create-gcp-sa" + + // ServiceManagedByGCPAgentLabel is used to mark service accounts that are managed by the GCP agent + ServiceManagedByGCPAgentLabel = "credentials-operator.otterize.com/managed-by-gcp-agent" + + // GCPWorkloadIdentityAnnotation is used by GCP workload identity to link between service accounts + GCPWorkloadIdentityAnnotation = "iam.gke.io/gcp-service-account" + GCPWorkloadIdentityNotSet = "false" +) + +func (a *Agent) AppliesOnPod(pod *corev1.Pod) bool { + return pod.Labels != nil && pod.Labels[GCPApplyOnPodLabel] == "true" +} + +func (a *Agent) OnPodAdmission(ctx context.Context, pod *corev1.Pod, serviceAccount *corev1.ServiceAccount) (updated bool, err error) { + if !a.AppliesOnPod(pod) { + return false, nil + } + + serviceAccount.Labels[ServiceManagedByGCPAgentLabel] = "true" + serviceAccount.Annotations[GCPWorkloadIdentityAnnotation] = GCPWorkloadIdentityNotSet + + return true, nil +} + +func (a *Agent) OnServiceAccountUpdate(ctx context.Context, serviceAccount *corev1.ServiceAccount) (updated bool, requeue bool, err error) { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if serviceAccount.Labels == nil || serviceAccount.Labels[ServiceManagedByGCPAgentLabel] != "true" { + logger.Debug("ServiceAccount is not managed by the GCP agent, skipping") + return false, false, nil + } + + // Check if we should update the service account - if the annotation is not set + if value, ok := serviceAccount.Annotations[GCPWorkloadIdentityAnnotation]; !ok || value != GCPWorkloadIdentityNotSet { + logger.Debug("ServiceAccount GCP workload identity annotation is already set, skipping") + return false, false, nil + } + + // Annotate the namespace to connect workload identity + requeue, err = a.AnnotateGKENamespace(ctx, serviceAccount.Namespace) + if err != nil { + return false, false, errors.Errorf("failed to annotate namespace: %w", err) + } + if requeue { + // TODO: maybe do apierrors.IsConflict(err) check instead? + return false, true, nil + } + + // Create IAMServiceAccount (Creates a GCP service account) + err = a.CreateAndConnectGSA(ctx, serviceAccount.Namespace, serviceAccount.Name) + if err != nil { + return false, false, errors.Errorf("failed to create and connect GSA: %w", err) + } + + // Annotate the service account with the GCP IAM role + gsaFullName := a.GetGSAFullName(serviceAccount.Namespace, serviceAccount.Name) + serviceAccount.Annotations[GCPWorkloadIdentityAnnotation] = gsaFullName + return true, false, nil +} + +func (a *Agent) OnServiceAccountTermination(ctx context.Context, serviceAccount *corev1.ServiceAccount) error { + logger := logrus.WithFields(logrus.Fields{"serviceAccount": serviceAccount.Name, "namespace": serviceAccount.Namespace}) + + if serviceAccount.Labels == nil || serviceAccount.Labels[ServiceManagedByGCPAgentLabel] != "true" { + logger.Debug("ServiceAccount is not managed by the GCP agent, skipping") + return nil + } + + return a.DeleteGSA(ctx, serviceAccount.Namespace, serviceAccount.Name) +} diff --git a/src/shared/gcpagent/utils.go b/src/shared/gcpagent/utils.go index 13427d5f3..f7abc4ef9 100644 --- a/src/shared/gcpagent/utils.go +++ b/src/shared/gcpagent/utils.go @@ -1,58 +1,39 @@ package gcpagent import ( - "crypto/sha256" "fmt" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/iam/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1" otterizev1alpha3 "github.com/otterize/intents-operator/src/operator/api/v1alpha3" + "github.com/otterize/intents-operator/src/shared/agentutils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const truncatedHashLength = 6 - -const maxK8SNameLength = 250 -const maxK8STruncatedLength = maxK8SNameLength - truncatedHashLength - 1 // add another char for the hyphen - -const maxDisplayNameLength = 100 -const maxDisplayNameTruncatedLength = maxDisplayNameLength - truncatedHashLength - 1 // add another char for the hyphen - -const maxGCPNameLength = 30 -const maxGCPTruncatedLength = maxGCPNameLength - truncatedHashLength - 1 // add another char for the hyphen - -func (a *Agent) limitResourceName(name string, maxLength int) string { - var truncatedName string - if len(name) >= maxLength { - truncatedName = name[:maxLength] - } else { - truncatedName = name - } - - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(name))) - hash = hash[:truncatedHashLength] - - return fmt.Sprintf("%s-%s", truncatedName, hash) -} +const ( + maxK8SNameLength = 250 + maxDisplayNameLength = 100 + maxGCPNameLength = 30 +) func (a *Agent) generateKSAPolicyName(namespace string, intentsServiceName string) string { name := fmt.Sprintf("otr-%s-%s-intent-policy", namespace, intentsServiceName) - return a.limitResourceName(name, maxK8STruncatedLength) + return agentutils.TruncateHashName(name, maxK8SNameLength) } func (a *Agent) generateGSAToKSAPolicyName(ksaName string) string { name := fmt.Sprintf("otr-%s-gcp-identity", ksaName) - return a.limitResourceName(name, maxK8STruncatedLength) + return agentutils.TruncateHashName(name, maxK8SNameLength) } func (a *Agent) generateGSADisplayName(namespace string, accountName string) string { name := fmt.Sprintf("otr-%s-%s-%s", a.clusterName, namespace, accountName) - return a.limitResourceName(name, maxDisplayNameTruncatedLength) + return agentutils.TruncateHashName(name, maxDisplayNameLength) } func (a *Agent) generateGSAName(namespace string, accountName string) string { fullName := a.generateGSADisplayName(namespace, accountName) - return a.limitResourceName(fullName, maxGCPTruncatedLength) + return agentutils.TruncateHashName(fullName, maxGCPNameLength) } func (a *Agent) GetGSAFullName(namespace string, accountName string) string {