From 1b5c66ebf5ee30258cc584daf0ce3fdcd4b6c6b0 Mon Sep 17 00:00:00 2001 From: Moritz Sanft Date: Wed, 18 Dec 2024 13:03:59 +0100 Subject: [PATCH] ci: enable STACKIT e2e tests --- .../actions/constellation_create/action.yml | 13 +- .../constellation_iam_create/action.yml | 13 ++ .github/actions/e2e_test/action.yml | 21 ++- .github/actions/login_stackit/action.yml | 16 ++ .github/actions/notify_stackit/action.yml | 19 +++ .github/actions/terraform_apply/action.yml | 13 ++ .github/workflows/e2e-test-stackit.yml | 152 ++++++++++++++++++ .github/workflows/e2e-test.yml | 12 +- 8 files changed, 252 insertions(+), 7 deletions(-) create mode 100644 .github/actions/login_stackit/action.yml create mode 100644 .github/actions/notify_stackit/action.yml create mode 100644 .github/workflows/e2e-test-stackit.yml diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml index 63cddf9d85..caec827d64 100644 --- a/.github/actions/constellation_create/action.yml +++ b/.github/actions/constellation_create/action.yml @@ -257,9 +257,9 @@ runs: continue-on-error: true uses: ./.github/actions/artifact_upload with: - name: serial-logs-${{ inputs.artifactNameSuffix }} - path: > - !(terraform).log + name: debug-logs-${{ inputs.artifactNameSuffix }} + path: | + *.log encryptionSecret: ${{ inputs.encryptionSecret }} - name: Prepare terraform state folders @@ -268,9 +268,12 @@ runs: run: | mkdir to-zip cp -r constellation-terraform to-zip - cp -r constellation-iam-terraform to-zip + # constellation-iam-terraform is optional + if [ -d constellation-iam-terraform ]; then + cp -r constellation-iam-terraform to-zip + fi rm -f to-zip/constellation-terraform/plan.zip - rm -rf to-zip/constellation-terraform/.terraform to-zip/constellation-iam-terraform/.terraform + rm -rf to-zip/*/.terraform - name: Upload terraform state if: always() diff --git a/.github/actions/constellation_iam_create/action.yml b/.github/actions/constellation_iam_create/action.yml index a6607d9826..2064644d5a 100644 --- a/.github/actions/constellation_iam_create/action.yml +++ b/.github/actions/constellation_iam_create/action.yml @@ -42,6 +42,12 @@ inputs: gcpZone: description: "The GCP zone to deploy Constellation in." required: false + # + # STACKIT specific inputs + # + stackitZone: + description: "The STACKIT zone to deploy Constellation in." + required: false runs: using: "composite" @@ -104,3 +110,10 @@ runs: --update-config \ --tf-log=DEBUG \ --yes + + - name: Set STACKIT-specific configuration + shell: bash + run: | + yq eval -i "(.provider.openstack.stackitProjectID) = \"6563baab-869d-4e7f-93cc-4087b4165073\"" constellation-conf.yaml + yq eval -i "(.provider.openstack.availabilityZone) = \"${{ inputs.stackitZone }}\"" constellation-conf.yaml + yq eval -i "(.nodeGroups.[].zone) = \"${{ inputs.stackitZone }}\"" constellation-conf.yaml diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index c2cca982d9..eb4fdb49e1 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -93,6 +93,12 @@ inputs: encryptionSecret: description: "The secret to use for decrypting the artifact." required: true + openStackCloudsYaml: + description: "The contents of ~/.config/openstack/clouds.yaml" + required: false + stackitUat: + description: "The UAT for STACKIT" + required: false outputs: kubeconfig: @@ -229,6 +235,18 @@ runs: with: azure_credentials: ${{ inputs.azureIAMCreateCredentials }} + - name: Login to OpenStack + if: inputs.cloudProvider == 'stackit' + uses: ./.github/actions/login_openstack + with: + clouds_yaml: ${{inputs.openStackCloudsYaml }} + + - name: Login to STACKIT + if: inputs.cloudProvider == 'stackit' + uses: ./.github/actions/login_stackit + with: + serviceAccountToken: ${{ inputs.stackitUat }} + - name: Create prefix id: create-prefix shell: bash @@ -244,7 +262,7 @@ runs: with: attestationVariant: ${{ inputs.attestationVariant }} - - name: Create IAM configuration + - name: Create Constellation config and IAM id: constellation-iam-create uses: ./.github/actions/constellation_iam_create with: @@ -256,6 +274,7 @@ runs: azureRegion: ${{ inputs.regionZone || steps.pick-az-region.outputs.region }} gcpProjectID: ${{ inputs.gcpProject }} gcpZone: ${{ inputs.regionZone || 'europe-west3-b' }} + stackitZone: ${{ inputs.regionZone || 'eu01-2' }} kubernetesVersion: ${{ inputs.kubernetesVersion }} additionalTags: "workflow=${{ github.run_id }}" diff --git a/.github/actions/login_stackit/action.yml b/.github/actions/login_stackit/action.yml new file mode 100644 index 0000000000..a7ff58425a --- /dev/null +++ b/.github/actions/login_stackit/action.yml @@ -0,0 +1,16 @@ +name: STACKIT login +description: "Login to STACKIT" +inputs: + serviceAccountToken: + description: "Credentials authorized to create Constellation on STACKIT." + required: true +runs: + using: "composite" + steps: + - name: Login to STACKIT + env: + UAT: ${{ inputs.serviceAccountToken }} + shell: bash + run: | + mkdir -p ~/.stackit + echo "${UAT}" > ~/.stackit/credentials.json diff --git a/.github/actions/notify_stackit/action.yml b/.github/actions/notify_stackit/action.yml new file mode 100644 index 0000000000..2e64fdac5d --- /dev/null +++ b/.github/actions/notify_stackit/action.yml @@ -0,0 +1,19 @@ +name: Notify STACKIT +description: "Notify STACKIT about test failure" +inputs: + slackToken: + description: "Slack access token." + required: true +runs: + using: "composite" + steps: + - name: Notify STACKIT + env: + SLACK_TOKEN: ${{ inputs.slackToken }} + shell: bash + run: | + curl -X POST \ + -H "Authorization: Bearer $SLACK_TOKEN" \ + -H "Content-type: application/json; charset=utf-8" \ + -d "{\"channel\":\"C0827BT59SM\",\"text\":\"E2E test failed: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID\"}" \ + https://slack.com/api/chat.postMessage diff --git a/.github/actions/terraform_apply/action.yml b/.github/actions/terraform_apply/action.yml index ffb893934d..cf106b41aa 100644 --- a/.github/actions/terraform_apply/action.yml +++ b/.github/actions/terraform_apply/action.yml @@ -29,6 +29,9 @@ runs: "gcpSEVSNP") attestationVariant="gcp-sev-snp" ;; + "qemuVTPM") + attestationVariant="qemu-vtpm" + ;; *) echo "Unknown attestation variant: $(yq '.attestation | keys | .[0]' constellation-conf.yaml)" exit 1 @@ -106,6 +109,16 @@ runs: project_id = "$(yq '.infrastructure.gcp.projectID' constellation-state.yaml)" service_account_key = sensitive("$(cat $(yq '.provider.gcp.serviceAccountKeyPath' constellation-conf.yaml) | base64 -w0)") } + openstack = { + cloud = "stackit" + clouds_yaml_path = "~/.config/openstack/clouds.yaml" + floating_ip_pool_id = "970ace5c-458f-484a-a660-0903bcfd91ad" + deploy_yawol_load_balancer = true + yawol_image_id = "bcd6c13e-75d1-4c3f-bf0f-8f83580cc1be" + yawol_flavor_id = "3b11b27e-6c73-470d-b595-1d85b95a8cdf" + network_id = "$(yq '.infrastructure.networkID' constellation-state.yaml)" + subnet_id = "$(yq '.infrastructure.subnetID' constellation-state.yaml)" + } network_config = { ip_cidr_node = "$(yq '.infrastructure.ipCidrNode' constellation-state.yaml)" ip_cidr_service = "$(yq '.serviceCIDR' constellation-conf.yaml)" diff --git a/.github/workflows/e2e-test-stackit.yml b/.github/workflows/e2e-test-stackit.yml new file mode 100644 index 0000000000..b3a779674e --- /dev/null +++ b/.github/workflows/e2e-test-stackit.yml @@ -0,0 +1,152 @@ +name: e2e test STACKIT + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # Every day at midnight. + +jobs: + find-latest-image: + name: Find latest image + runs-on: ubuntu-24.04 + permissions: + id-token: write + contents: read + outputs: + image-release-stable: ${{ steps.relabel-output.outputs.image-release-stable }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Select relevant image + id: select-image-action + uses: ./.github/actions/select_image + with: + osImage: "ref/release/stream/stable/?" + + - name: Relabel output + id: relabel-output + shell: bash + run: | + ref=$(echo 'ref/release/stream/stable/?' | cut -d/ -f2) + stream=$(echo 'ref/release/stream/stable/?' | cut -d/ -f4) + + echo "image-$ref-$stream=${{ steps.select-image-action.outputs.osImage }}" | tee -a "$GITHUB_OUTPUT" + + e2e-stackit: + strategy: + fail-fast: false + max-parallel: 6 + matrix: + kubernetesVersion: [ "1.28", "1.29", "1.30" ] + clusterCreation: [ "cli", "terraform" ] + test: [ "sonobuoy quick" ] + runs-on: ubuntu-24.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + actions: write + needs: [find-latest-image] + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Setup bazel + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: terraform + + - name: Run E2E test + id: e2e_test + uses: ./.github/actions/e2e_test + with: + workerNodesCount: "1" + controlNodesCount: "1" + cloudProvider: stackit + attestationVariant: qemu-vtpm + osImage: ${{ needs.find-latest-image.outputs.image-release-stable }} + isDebugImage: false + cliVersion: ${{ needs.find-latest-image.outputs.image-release-stable || '' }} + kubernetesVersion: ${{ matrix.kubernetesVersion }} + awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }} + awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} + awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} + gcpProject: constellation-e2e + gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" + gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + test: ${{ matrix.test }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} + azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + registry: ghcr.io + githubToken: ${{ secrets.GITHUB_TOKEN }} + cosignPassword: ${{ secrets.COSIGN_PASSWORD }} + cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }} + fetchMeasurements: false + clusterCreation: ${{ matrix.clusterCreation }} + s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }} + s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }} + stackitUat: ${{ secrets.STACKIT_CI_UAT }} + + - name: Always terminate cluster + if: always() + uses: ./.github/actions/constellation_destroy + with: + kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} + clusterCreation: ${{ matrix.clusterCreation }} + cloudProvider: stackit + azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} + gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Always delete IAM configuration + if: always() + uses: ./.github/actions/constellation_iam_destroy + with: + cloudProvider: stackit + azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Notify about failure + if: | + failure() && + github.ref == 'refs/heads/main' && + github.event_name == 'schedule' + continue-on-error: true + uses: ./.github/actions/notify_e2e_failure + with: + projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} + refStream: "ref/release/stream/stable/?" + test: ${{ matrix.test }} + kubernetesVersion: ${{ matrix.kubernetesVersion }} + provider: stackit + attestationVariant: qemu-vtpm + clusterCreation: ${{ matrix.clusterCreation }} + + - name: Notify STACKIT + if: | + failure() && + github.ref == 'refs/heads/main' && + github.event_name == 'schedule' + continue-on-error: true + uses: ./.github/actions/notify_stackit + with: + slackToken: ${{ secrets.SLACK_TOKEN }} diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 6efdd03b5f..33eb41b291 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -16,6 +16,7 @@ on: - "azure-sev-snp" - "azure-tdx" - "aws-sev-snp" + - "stackit-qemu-vtpm" default: "azure-sev-snp" required: true runner: @@ -137,6 +138,7 @@ jobs: workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }} controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }} cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ steps.split-attestationVariant.outputs.attestationVariant }} steps: - name: Split nodeCount id: split-nodeCount @@ -161,6 +163,12 @@ jobs: attestationVariant="${{ inputs.attestationVariant }}" cloudProvider="${attestationVariant%%-*}" + # special case for STACKIT, as there's no special attestation variant for it + if [[ "${cloudProvider}" == "stackit" ]]; then + attestationVariant="qemu-vtpm" + fi + + echo "attestationVariant=${attestationVariant}" | tee -a "$GITHUB_OUTPUT" echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" find-latest-image: @@ -233,7 +241,7 @@ jobs: workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }} controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }} cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} - attestationVariant: ${{ inputs.attestationVariant }} + attestationVariant: ${{ needs.generate-input-parameters.outputs.attestationVariant }} machineType: ${{ inputs.machineType }} regionZone: ${{ inputs.regionZone }} gcpProject: constellation-e2e @@ -262,6 +270,8 @@ jobs: marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }} force: ${{ inputs.force }} encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }} + stackitUat: ${{ secrets.STACKIT_CI_UAT }} - name: Always terminate cluster if: always()