Merge pull request #665 from somtochiama/test-workload-id

Add integration tests for Workload Identity

.github/workflows/integration-azure.yaml

@@ -14,6 +14,10 @@ permissions:
 jobs:
   oci-test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        enable-workload-id: [ "true", "false" ]
+      fail-fast: false
     defaults:
       run:
         working-directory: ./oci/tests/integration
@@ -50,3 +54,6 @@ jobs:
           ARM_SUBSCRIPTION_ID: ${{ secrets.OCI_E2E_AZ_ARM_SUBSCRIPTION_ID }}
           ARM_TENANT_ID: ${{ secrets.OCI_E2E_AZ_ARM_TENANT_ID }}
           TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }}
+          TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
+          TF_VAR_wi_k8s_sa_name: test-workload-id
+          TF_VAR_wi_k8s_sa_ns: default

.github/workflows/integration-gcp.yaml

@@ -14,6 +14,10 @@ permissions:
 jobs:
   oci-test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        enable-workload-id: ["true", "false"]
+      fail-fast: false
     defaults:
       run:
         working-directory: ./oci/tests/integration
@@ -64,3 +68,6 @@ jobs:
           TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
           TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
           TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}
+          TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
+          TF_VAR_wi_k8s_sa_name: test-workload-id
+          TF_VAR_wi_k8s_sa_ns: default

oci/tests/integration/.env.sample

@@ -42,3 +42,9 @@
 ## apply. Overriding it with a static value helps avoid modifying the resource
 ## tags during development when the configurations are applied frequently.
 # export TF_VAR_tags='{"environment"="dev", "createdat"='"\"$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)\""'}'
+
+## Workload Identity Terraform Variables
+## These variables only get used if the `TF_VAR_enable_wi` variable is set to "true".
+# export TF_VAR_wi_k8s_sa_name=test-workload-id
+# export TF_VAR_wi_k8s_sa_ns=default
+# export TF_VAR_enable_wi=true

oci/tests/integration/README.md

@@ -62,10 +62,15 @@ Following permissions are needed for provisioning the infrastructure and running
 the tests:
 - `Microsoft.Kubernetes/*`
 - `Microsoft.Resources/*`
-- `Microsoft.Authorization/roleAssignments/{Read,Write,Delete}`
+- `Microsoft.Authorization/roleAssignments/{read,write,delete}`
 - `Microsoft.ContainerRegistry/*`
 - `Microsoft.ContainerService/*`
 
+Additional permissions needed when Workload Identity is enabled:
+
+- `Microsoft.ManagedIdentity/userAssignedIdentities/{read,write,delete}`
+- `Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/{read,write,delete}`
+
 #### IAM and CI setup
 
 To create the necessary IAM role with all the permissions, set up CI secrets and
@@ -88,11 +93,17 @@ module "azure_gh_actions" {
   azure_permissions = [
     "Microsoft.Kubernetes/*",
     "Microsoft.Resources/*",
-    "Microsoft.Authorization/roleAssignments/Read",
-    "Microsoft.Authorization/roleAssignments/Write",
-    "Microsoft.Authorization/roleAssignments/Delete",
+    "Microsoft.Authorization/roleAssignments/read",
+    "Microsoft.Authorization/roleAssignments/write",
+    "Microsoft.Authorization/roleAssignments/delete",
     "Microsoft.ContainerRegistry/*",
-    "Microsoft.ContainerService/*"
+    "Microsoft.ContainerService/*",
+    "Microsoft.ManagedIdentity/userAssignedIdentities/read",
+    "Microsoft.ManagedIdentity/userAssignedIdentities/write",
+    "Microsoft.ManagedIdentity/userAssignedIdentities/delete",
+    "Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/read",
+    "Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/write",
+    "Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/delete"
   ]
   azure_location = "eastus"
 
@@ -166,6 +177,10 @@ tests:
 - Service Account User - `roles/iam.serviceAccountUser`
 - Storage Admin - `roles/storage.admin`
 
+If workload identity is enabled, the following role is also needed:
+
+- Project IAM Admin - `roles/resourcemanager.projectIamAdmin`
+
 #### IAM and CI setup
 
 To create the necessary IAM role with all the permissions, set up CI secrets and
@@ -194,7 +209,8 @@ module "gcp_gh_actions" {
     "roles/iam.serviceAccountAdmin",
     "roles/iam.serviceAccountTokenCreator",
     "roles/iam.serviceAccountUser",
-    "roles/storage.admin"
+    "roles/storage.admin",
+    "roles/resourcemanager.projectIamAdmin"
   ]
 
   github_project = "pkg"
@@ -228,6 +244,19 @@ TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -verbose -retain -provid
 ...
 ```
 
+## Workload Identity
+
+By default, the tests use node identity for authentication. To run the integration tests on clusters with workload identity
+enabled for any of the providers. The following terraform variables need to be set.
+
+```shell
+export TF_VAR_wi_k8s_sa_name=
+export TF_VAR_wi_k8s_sa_ns=
+export TF_VAR_enable_wi=
+```
+
+They have been included in the `.env.sample` and you can simply uncomment it.
+
 ## Debugging the tests
 
 For debugging environment provisioning, enable verbose output with `-verbose`

oci/tests/integration/aws_test.go

@@ -21,12 +21,17 @@ package integration
 
 import (
 	"context"
+	"fmt"
 
 	tfjson "github.com/hashicorp/terraform-json"
 
 	"github.com/fluxcd/test-infra/tftestenv"
 )
 
+const (
+	eksRoleArnAnnotation = "eks.amazonaws.com/role-arn"
+)
+
 // createKubeconfigEKS constructs kubeconfig from the terraform state output at
 // the given kubeconfig path.
 func createKubeconfigEKS(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
@@ -80,3 +85,16 @@ func pushAppTestImagesECR(ctx context.Context, localImgs map[string]string, outp
 	remoteImage := repo + ":test"
 	return tftestenv.PushTestAppImagesECR(ctx, localImgs, remoteImage)
 }
+
+// getWISAAnnotationsAWS returns annotations for a kubernetes service account required to configure IRSA / workload
+// identity on AWS. It gets the role ARN from the terraform output and returns the map[eks.amazonaws.com/role-arn=<arn>]
+func getWISAAnnotationsAWS(output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	iamARN := output["aws_wi_iam_arn"].Value.(string)
+	if iamARN == "" {
+		return nil, fmt.Errorf("no AWS iam role arn in terraform output")
+	}
+
+	return map[string]string{
+		eksRoleArnAnnotation: iamARN,
+	}, nil
+}

oci/tests/integration/azure_test.go

@@ -28,6 +28,11 @@ import (
 	"github.com/fluxcd/test-infra/tftestenv"
 )
 
+const (
+	// azureWIClientIdAnnotation is the key for the annotation on the kubernetes serviceaccount
+	azureWIClientIdAnnotation = "azure.workload.identity/client-id"
+)
+
 // createKubeConfigAKS constructs kubeconfig for an AKS cluster from the
 // terraform state output at the given kubeconfig path.
 func createKubeConfigAKS(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
@@ -62,3 +67,17 @@ func pushAppTestImagesACR(ctx context.Context, localImgs map[string]string, outp
 	registryURL := output["acr_registry_url"].Value.(string)
 	return tftestenv.PushTestAppImagesACR(ctx, localImgs, registryURL)
 }
+
+// getWISAAnnotationsAzure returns azure workload identity's annotations for
+// kubernetes service account using output from terraform.
+// https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview?tabs=dotnet#pod-annotations
+func getWISAAnnotationsAzure(output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	clientID := output["workload_identity_client_id"].Value.(string)
+	if clientID == "" {
+		return nil, fmt.Errorf("no Azure client id in terraform output")
+	}
+
+	return map[string]string{
+		azureWIClientIdAnnotation: clientID,
+	}, nil
+}

oci/tests/integration/gcp_test.go

@@ -28,6 +28,12 @@ import (
 	"github.com/fluxcd/test-infra/tftestenv"
 )
 
+const (
+	// gcpIAMAnnotation is the key for the annotation on the kubernetes serviceaccount
+	// with the email address of the IAM service account on GCP.
+	gcpIAMAnnotation = "iam.gke.io/gcp-service-account"
+)
+
 // createKubeconfigGKE constructs kubeconfig from the terraform state output at
 // the given kubeconfig path.
 func createKubeconfigGKE(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
@@ -72,3 +78,15 @@ func pushAppTestImagesGCR(ctx context.Context, localImgs map[string]string, outp
 	repositoryID := output["gcp_artifact_repository"].Value.(string)
 	return tftestenv.PushTestAppImagesGCR(ctx, localImgs, project, region, repositoryID)
 }
+
+// getWISAAnnotationsGCP returns workload identity annotations for a kubernetes ServiceAccount
+func getWISAAnnotationsGCP(output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	saEmail := output["wi_iam_serviceaccount_email"].Value.(string)
+	if saEmail == "" {
+		return nil, fmt.Errorf("no GCP serviceaccount email in terraform output")
+	}
+
+	return map[string]string{
+		gcpIAMAnnotation: saEmail,
+	}, nil
+}

oci/tests/integration/repo_list_test.go

@@ -93,6 +93,17 @@ func testImageRepositoryListTags(t *testing.T, args []string) {
 	}
 	job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever
 
+	if enableWI {
+		job.Spec.Template.Spec.ServiceAccountName = wiServiceAccount
+
+		// azure requires this label on the pod for workload identity to work.
+		if *targetProvider == "azure" {
+			job.Spec.Template.Labels = map[string]string{
+				"azure.workload.identity/use": "true",
+			}
+		}
+	}
+
 	key := client.ObjectKeyFromObject(job)
 
 	g.Expect(testEnv.Client.Create(ctx, job)).To(Succeed())