ci: add e2e test for self-managed infrastructure

.github/actions/constellation_create/action.yml

@@ -1,5 +1,5 @@
 name: Constellation create
-description: Create a new Constellation cluster using latest OS image.
+description: Create a new Constellation cluster using the latest OS image.
 
 inputs:
   workerNodesCount:
@@ -50,6 +50,9 @@ inputs:
   internalLoadBalancer:
     description: "Whether to use an internal load balancer for the control plane"
     required: false
+  selfManagedInfra:
+    description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
+    required: true
 
 outputs:
   kubeconfig:
@@ -124,14 +127,25 @@ runs:
       run: |
         yq eval -i '(.internalLoadBalancer) = true' constellation-conf.yaml
 
-    - name: Constellation create
+    - name: Show Cluster Configuration
       shell: bash
       run: |
         echo "Creating cluster using config:"
         cat constellation-conf.yaml
         sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' || true
+
+    - name: Constellation create (CLI)
+      if : inputs.selfManagedInfra != 'true'
+      shell: bash
+      run: |
         constellation create -y --debug --tf-log=DEBUG
 
+    - name: Constellation create (self-managed)
+      if : inputs.selfManagedInfra == 'true'
+      uses: ./.github/actions/self_managed_create
+      with:
+        cloudProvider: ${{ inputs.cloudProvider }}
+
     - name: Cdbg deploy
       if: inputs.isDebugImage == 'true'
       uses: ./.github/actions/cdbg_deploy

.github/actions/constellation_destroy/action.yml

@@ -5,6 +5,9 @@ inputs:
   kubeconfig:
     description: "The kubeconfig for the cluster."
     required: true
+  selfManagedInfra:
+    description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
+    required: true
 
 runs:
   using: "composite"
@@ -39,6 +42,18 @@ runs:
         echo "::endgroup::"
 
     - name: Constellation terminate
+      if: inputs.selfManagedInfra != 'true'
       shell: bash
       run: |
         constellation terminate --yes --tf-log=DEBUG
+
+    - name: Constellation terminate (self-managed)
+      if: inputs.selfManagedInfra == 'true'
+      shell: bash
+      working-directory: ${{ github.workspace }}/e2e-infra
+      run: |
+        terraform init
+        terraform destroy -auto-approve
+        # Explicitly delete the state file
+        rm ${{ github.workspace }}/constellation-state.yaml
+        rm ${{ github.workspace }}/constellation-admin.conf

.github/actions/e2e_test/action.yml

@@ -76,6 +76,9 @@ inputs:
     description: "Enable security policy for the cluster."
   internalLoadBalancer:
     description: "Enable internal load balancer for the cluster."
+  selfManagedInfra:
+    description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
+    default: "false"
 
 outputs:
   kubeconfig:
@@ -260,6 +263,8 @@ runs:
         kubernetesVersion: ${{ inputs.kubernetesVersion }}
         refStream: ${{ inputs.refStream }}
         internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
+        test: ${{ inputs.test }}
+        selfManagedInfra: ${{ inputs.selfManagedInfra }}
 
     - name: Deploy log- and metrics-collection (Kubernetes)
       id: deploy-logcollection

.github/actions/self_managed_create/action.yml

@@ -0,0 +1,111 @@
+name: Self-managed infrastructure creation
+description: "Create the required infrastructure for a Constellation cluster manually."
+
+inputs:
+  cloudProvider:
+    description: "The cloud provider the test runs on."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Copy Terraform configuration and Constellation config
+      shell: bash
+      working-directory:
+      run: |
+        cp -r ${{ github.workspace }}/cli/internal/terraform/terraform/${{ inputs.cloudProvider }} ${{ github.workspace }}/e2e-infra
+        cp ${{ github.workspace }}/constellation-conf.yaml ${{ github.workspace }}/e2e-infra
+
+    - name: Get CSP image reference
+      id: get_image
+      shell: bash
+      working-directory: ${{ github.workspace }}/e2e-infra
+      run: |
+        echo "image_ref=$(bazel run //hack/image-fetch:image-fetch)" >> $GITHUB_OUTPUT
+
+    - name: Write Terraform variables
+      shell: bash
+      working-directory: ${{ github.workspace }}/e2e-infra
+      run: |
+        echo "name = \"$(yq '.name' constellation-conf.yaml)\"" >> terraform.tfvars
+        echo "debug = $(yq '.debugCluster' constellation-conf.yaml)" >> terraform.tfvars
+        echo "custom_endpoint = \"$(yq '.customEndpoint' constellation-conf.yaml)\"" >> terraform.tfvars
+        echo "image_id = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars
+        echo "node_groups = {
+          control_plane_default = {
+            role = \"$(yq '.nodeGroups.control_plane_default.role' constellation-conf.yaml)\"
+            zone = \"$(yq '.nodeGroups.control_plane_default.zone' constellation-conf.yaml)\"
+            instance_type = \"$(yq '.nodeGroups.control_plane_default.instanceType' constellation-conf.yaml)\"
+            disk_size = \"$(yq '.nodeGroups.control_plane_default.stateDiskSizeGB' constellation-conf.yaml)\"
+            disk_type = \"$(yq '.nodeGroups.control_plane_default.stateDiskType' constellation-conf.yaml)\"
+            initial_count = \"$(yq '.nodeGroups.control_plane_default.initialCount' constellation-conf.yaml)\"
+          }
+          worker_default = {
+            role = \"$(yq '.nodeGroups.worker_default.role' constellation-conf.yaml)\"
+            zone = \"$(yq '.nodeGroups.worker_default.zone' constellation-conf.yaml)\"
+            instance_type = \"$(yq '.nodeGroups.worker_default.instanceType' constellation-conf.yaml)\"
+            disk_size = \"$(yq '.nodeGroups.worker_default.stateDiskSizeGB' constellation-conf.yaml)\"
+            disk_type = \"$(yq '.nodeGroups.worker_default.stateDiskType' constellation-conf.yaml)\"
+            initial_count = \"$(yq '.nodeGroups.worker_default.initialCount' constellation-conf.yaml)\"
+          }
+        }" >> terraform.tfvars
+        if [[ "${{ inputs.cloudProvider }}" == 'aws' ]]; then
+          echo "iam_instance_profile_control_plane = \"$(yq '.provider.aws.iamProfileControlPlane' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "iam_instance_profile_worker_nodes = \"$(yq '.provider.aws.iamProfileWorkerNodes' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "region = \"$(yq '.provider.aws.region' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "zone = \"$(yq '.provider.aws.zone' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "ami = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars
+          echo "enable_snp = $(yq '.attestation | has("awsSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
+        elif [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then
+          echo "location = \"$(yq '.provider.azure.location' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "create_maa = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
+          echo "confidential_vm = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
+          echo "secure_boot = $(yq '.provider.azure.secureBoot' constellation-conf.yaml)" >> terraform.tfvars
+          echo "resource_group = \"$(yq '.provider.azure.resourceGroup' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "user_assigned_identity = \"$(yq '.provider.azure.userAssignedIdentity' constellation-conf.yaml)\"" >> terraform.tfvars
+        elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then
+          echo "project = \"$(yq '.provider.gcp.project' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "region = \"$(yq '.provider.gcp.region' constellation-conf.yaml)\"" >> terraform.tfvars
+          echo "zone = \"$(yq '.provider.gcp.zone' constellation-conf.yaml)\"" >> terraform.tfvars
+        fi
+        terraform fmt terraform.tfvars
+        echo "Using Terraform variables:"
+        cat terraform.tfvars
+
+    - name: Apply Terraform configuration
+      shell: bash
+      working-directory: ${{ github.workspace }}/e2e-infra
+      run: |
+        terraform init
+        terraform apply -auto-approve
+
+    - name: Patch MAA Policy
+      shell: bash
+      working-directory: ${{ github.workspace }}/e2e-infra
+      if: ${{ inputs.cloudProvider }} == 'azure'
+      run: |
+        bazel run //hack/maa-patch:maa-patch $(terraform output attestationURL | jq -r)
+
+    - name: Write outputs to state file
+      shell: bash
+      working-directory: ${{ github.workspace }}/e2e-infra
+      run: |
+        yq eval '.version ="v1"' --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.initSecret =\"$(terraform output initSecret | jq -r | tr -d '\n' | hexdump -ve '/1 "%02x"' && echo '')\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.clusterEndpoint =\"$(terraform output out_of_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.inClusterEndpoint =\"$(terraform output in_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.ipCidrNode =\"$(terraform output ip_cidr_nodes | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.uid =\"$(terraform output uid | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.name =\"$(terraform output name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        yq eval ".infrastructure.apiServerCertSANs =$(terraform output -json api_server_cert_sans)" --inplace ${{ github.workspace }}/constellation-state.yaml
+        if [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then
+          yq eval ".infrastructure.azure.resourceGroup =\"$(terraform output resource_group | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+          yq eval ".infrastructure.azure.subscriptionID =\"$(terraform output subscription_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+          yq eval ".infrastructure.azure.networkSecurityGroupName =\"$(terraform output network_security_group_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+          yq eval ".infrastructure.azure.loadBalancerName =\"$(terraform output loadbalancer_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+          yq eval ".infrastructure.azure.userAssignedIdentity =\"$(terraform output user_assigned_identity_client_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+          yq eval ".infrastructure.azure.attestationURL =\"$(terraform output attestationURL | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then
+          yq eval ".infrastructure.gcp.projectID =\"$(terraform output project | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+          yq eval ".infrastructure.gcp.ipCidrPod =\"$(terraform output ip_cidr_pods | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
+        fi

.github/workflows/e2e-test-daily.yml

@@ -91,12 +91,14 @@ jobs:
           awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
+          selfManagedInfra: "false"
 
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+          selfManagedInfra: "false"
 
       - name: Always delete IAM configuration
         if: always()

.github/workflows/e2e-test-manual-internal.yml

@@ -205,12 +205,14 @@ jobs:
           cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
           internalLoadBalancer: true
+          selfManagedInfra: "false"
 
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+          selfManagedInfra: "false"
 
       - name: Always delete IAM configuration
         if: always()

.github/workflows/e2e-test-manual.yml

@@ -260,12 +260,14 @@ jobs:
           cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
           cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
+          selfManagedInfra: "false"
 
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+          selfManagedInfra: "false"
 
       - name: Always delete IAM configuration
         if: always()

.github/workflows/e2e-test-release.yml

@@ -151,6 +151,24 @@ jobs:
             kubernetes-version: "v1.28"
             runner: "ubuntu-22.04"
 
+          # self-managed infra test on latest k8s version
+          # runs Sonobuoy full test
+          - test: "sonobuoy full"
+            provider: "gcp"
+            kubernetes-version: "v1.28"
+            runner: "ubuntu-22.04"
+            selfManagedInfra: "true"
+          - test: "sonobuoy full"
+            provider: "azure"
+            kubernetes-version: "v1.28"
+            runner: "ubuntu-22.04"
+            selfManagedInfra: "true"
+          - test: "sonobuoy full"
+            provider: "aws"
+            kubernetes-version: "v1.28"
+            runner: "ubuntu-22.04"
+            selfManagedInfra: "true"
+
           #
           # Tests on macOS runner
           #
@@ -213,12 +231,14 @@ jobs:
           cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
           cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
 
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+          selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
 
       - name: Always delete IAM configuration
         if: always()

.github/workflows/e2e-test-weekly.yml

@@ -171,6 +171,24 @@ jobs:
             provider: "aws"
             kubernetes-version: "v1.28"
 
+          # self-managed infra test on latest k8s version
+          # with Sonobuoy full
+          - test: "sonobuoy full"
+            refStream: "ref/main/stream/debug/?"
+            provider: "gcp"
+            kubernetes-version: "v1.28"
+            selfManagedInfra: "true"
+          - test: "sonobuoy full"
+            refStream: "ref/main/stream/debug/?"
+            provider: "azure"
+            kubernetes-version: "v1.28"
+            selfManagedInfra: "true"
+          - test: "sonobuoy full"
+            provider: "aws"
+            refStream: "ref/main/stream/debug/?"
+            kubernetes-version: "v1.28"
+            selfManagedInfra: "true"
+
           #
           # Tests on release-stable refStream
           #
@@ -188,6 +206,7 @@ jobs:
             refStream: "ref/release/stream/stable/?"
             provider: "aws"
             kubernetes-version: "v1.27"
+
     runs-on: ubuntu-22.04
     permissions:
       id-token: write
@@ -231,12 +250,14 @@ jobs:
           cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
           azureSNPEnforcementPolicy: ${{ matrix.azureSNPEnforcementPolicy }}
+          selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
 
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+          selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
 
       - name: Always delete IAM configuration
         if: always()

.github/workflows/e2e-upgrade.yml

@@ -182,6 +182,7 @@ jobs:
           awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
+          selfManagedInfra: "false"
 
       - name: Build CLI
         uses: ./.github/actions/build_cli
@@ -287,6 +288,7 @@ jobs:
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+          selfManagedInfra: "false"
 
       - name: Always delete IAM configuration
         if: always()

cli/internal/cloudcmd/BUILD.bazel

@@ -10,7 +10,6 @@ go_library(
         "create.go",
         "iam.go",
         "iamupgrade.go",
-        "patch.go",
         "rollback.go",
         "serviceaccount.go",
         "terminate.go",
@@ -36,10 +35,8 @@ go_library(
         "//internal/constants",
         "//internal/file",
         "//internal/imagefetcher",
+        "//internal/maa",
         "//internal/role",
-        "@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
-        "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
-        "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
         "@com_github_spf13_cobra//:cobra",
     ],
 )
@@ -51,7 +48,6 @@ go_test(
         "clusterupgrade_test.go",
         "create_test.go",
         "iam_test.go",
-        "patch_test.go",
         "rollback_test.go",
         "terminate_test.go",
         "tfupgrade_test.go",

cli/internal/cloudcmd/clusterupgrade.go

@@ -18,6 +18,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/file"
+	"github.com/edgelesssys/constellation/v2/internal/maa"
 )
 
 // ClusterUpgrader is responsible for performing Terraform migrations on cluster upgrades.
@@ -43,7 +44,7 @@ func NewClusterUpgrader(ctx context.Context, existingWorkspace, upgradeWorkspace
 
 	return &ClusterUpgrader{
 		tf:                tfClient,
-		policyPatcher:     NewAzurePolicyPatcher(),
+		policyPatcher:     maa.NewAzurePolicyPatcher(),
 		fileHandler:       fileHandler,
 		existingWorkspace: existingWorkspace,
 		upgradeWorkspace:  upgradeWorkspace,