e2e test Terraform provider example #221
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e test Terraform provider example | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| type: string | |
| description: "Git ref to checkout" | |
| regionZone: | |
| description: "Region or zone to create the cluster in. Leave empty for default region/zone." | |
| type: string | |
| image: | |
| description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used." | |
| type: string | |
| providerVersion: | |
| description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." | |
| type: string | |
| toImage: | |
| description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. | |
| type: string | |
| required: false | |
| toKubernetes: | |
| description: Kubernetes version to target for the upgrade, empty for no upgrade. | |
| type: string | |
| required: false | |
| attestationVariant: | |
| description: "Attestation variant to use." | |
| type: choice | |
| options: | |
| - "aws-sev-snp" | |
| - "azure-sev-snp" | |
| - "azure-tdx" | |
| - "gcp-sev-es" | |
| - "gcp-sev-snp" | |
| default: "azure-sev-snp" | |
| required: true | |
| workflow_call: | |
| inputs: | |
| ref: | |
| type: string | |
| description: "Git ref to checkout" | |
| regionZone: | |
| description: "Which zone to use." | |
| type: string | |
| image: | |
| description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used." | |
| type: string | |
| providerVersion: | |
| description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." | |
| type: string | |
| toImage: | |
| description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. | |
| type: string | |
| required: false | |
| toKubernetes: | |
| description: Kubernetes version to target for the upgrade, empty for target's default version. | |
| type: string | |
| required: false | |
| attestationVariant: | |
| description: "Attestation variant to use." | |
| type: string | |
| required: true | |
| jobs: | |
| provider-example-test: | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| ref: ${{ inputs.ref || github.head_ref }} | |
| - name: Get Latest Image | |
| id: find-latest-image | |
| uses: ./.github/actions/find_latest_image | |
| with: | |
| git-ref: ${{ inputs.ref }} | |
| imageVersion: ${{ inputs.image }} | |
| ref: main | |
| stream: nightly | |
| - name: Determine cloudprovider from attestation variant | |
| id: determine | |
| shell: bash | |
| run: | | |
| attestationVariant="${{ inputs.attestationVariant }}" | |
| cloudProvider="${attestationVariant%%-*}" | |
| echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Log in to the Container registry | |
| uses: ./.github/actions/container_registry_login | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download CLI # needed to determine K8s version for release versions | |
| if: inputs.providerVersion != '' | |
| shell: bash | |
| run: | | |
| curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/constellation-linux-amd64 | |
| chmod u+x constellation | |
| ./constellation version | |
| mkdir -p ${{ github.workspace }}/release | |
| cp ./constellation ${{ github.workspace }}/release | |
| - name: Setup bazel | |
| uses: ./.github/actions/setup_bazel_nix | |
| with: | |
| nixTools: terraform | |
| - name: Create prefix | |
| id: create-prefix | |
| shell: bash | |
| run: | | |
| uuid=$(uuidgen | tr "[:upper:]" "[:lower:]") | |
| uuid="${uuid%%-*}" | |
| uuid="${uuid: -3}" # Final resource name must be no longer than 10 characters on AWS | |
| echo "uuid=${uuid}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "prefix=e2e-${uuid}" | tee -a "${GITHUB_OUTPUT}" | |
| - name: Build Constellation provider and CLI # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade | |
| working-directory: ${{ github.workspace }} | |
| id: build | |
| shell: bash | |
| run: | | |
| mkdir -p ${{ github.workspace }}/build | |
| cd ${{ github.workspace }}/build | |
| bazel run //:devbuild --cli_edition=enterprise | |
| bazel build //bazel/settings:tag | |
| repository_root=$(git rev-parse --show-toplevel) | |
| out_rel=$(bazel cquery --output=files //bazel/settings:tag) | |
| build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") | |
| echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Remove local Terraform registry # otherwise the local registry would be used instead of the public registry | |
| if: inputs.providerVersion != '' | |
| shell: bash | |
| run: | | |
| bazel build //bazel/settings:tag | |
| repository_root=$(git rev-parse --show-toplevel) | |
| out_rel=$(bazel cquery --output=files //bazel/settings:tag) | |
| build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") | |
| terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/" | |
| rm -rf "${terraform_provider_dir}" | |
| - name: Login to AWS (IAM + Cluster role) | |
| if: steps.determine.outputs.cloudProvider == 'aws' | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform | |
| aws-region: eu-central-1 | |
| # extend token expiry to 6 hours to ensure constellation can terminate | |
| role-duration-seconds: 21600 | |
| - name: Login to Azure (IAM + Cluster service principal) | |
| if: steps.determine.outputs.cloudProvider == 'azure' | |
| uses: ./.github/actions/login_azure | |
| with: | |
| azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }} | |
| - name: Login to GCP (IAM + Cluster service account) | |
| if: steps.determine.outputs.cloudProvider == 'gcp' | |
| uses: ./.github/actions/login_gcp | |
| with: | |
| service_account: "[email protected]" | |
| - name: Set Kubernetes version | |
| id: kubernetes | |
| run: | | |
| set -e | |
| # take the middle (2nd) supported Kubernetes version (default) | |
| if [[ "${{ inputs.providerVersion }}" != "" ]]; then | |
| cli_output=$(${{ github.workspace }}/release/constellation config kubernetes-versions) | |
| else | |
| cli_output=$(${{ github.workspace }}/build/constellation config kubernetes-versions) | |
| fi | |
| echo "version=$(echo "${cli_output}" | awk 'NR==3{print $1}')" | tee -a "${GITHUB_OUTPUT}" | |
| - name: Common CSP Terraform overrides | |
| working-directory: ${{ github.workspace }} | |
| shell: bash | |
| run: | | |
| mkdir -p ${{ github.workspace }}/cluster | |
| cd ${{ github.workspace }}/cluster | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| prefixed_version=${{ steps.build.outputs.build_version }} | |
| else | |
| prefixed_version="${{ inputs.providerVersion }}" | |
| fi | |
| version=${prefixed_version#v} # remove v prefix | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| iam_src="${{ github.workspace }}/terraform/infrastructure/iam/${{ steps.determine.outputs.cloudProvider }}" | |
| infra_src="${{ github.workspace }}/terraform/infrastructure/${{ steps.determine.outputs.cloudProvider }}" | |
| else | |
| iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ steps.determine.outputs.cloudProvider }}" | |
| infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ steps.determine.outputs.cloudProvider }}" | |
| fi | |
| # by default use latest nightly image for devbuilds and release image otherwise | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| if [[ "${{ inputs.image }}" == "" ]]; then | |
| image_version="${{ steps.find-latest-image.outputs.image }}" | |
| else | |
| image_version="${{ inputs.image }}" | |
| fi | |
| else | |
| if [[ "${{ inputs.image }}" == "" ]]; then | |
| image_version="${prefixed_version}" | |
| else | |
| image_version="${{ inputs.image }}" | |
| fi | |
| fi | |
| kubernetes_version="${{ steps.kubernetes.outputs.version }}" | |
| cat > _override.tf <<EOF | |
| terraform { | |
| required_providers { | |
| constellation = { | |
| source = "edgelesssys/constellation" | |
| version = "${version}" | |
| } | |
| } | |
| } | |
| locals { | |
| control_plane_count = 1 | |
| worker_count = 1 | |
| } | |
| locals { | |
| name = "${{ steps.create-prefix.outputs.prefix }}" | |
| image_version = "${image_version}" | |
| microservice_version = "${prefixed_version}" | |
| kubernetes_version = "${kubernetes_version}" | |
| attestation_variant = "${{ inputs.attestationVariant }}" | |
| } | |
| module "${{ steps.determine.outputs.cloudProvider }}_iam" { | |
| source = "${iam_src}" | |
| } | |
| module "${{ steps.determine.outputs.cloudProvider }}_infrastructure" { | |
| source = "${infra_src}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create GCP Terraform overrides | |
| if: steps.determine.outputs.cloudProvider == 'gcp' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev) | |
| case "${{ inputs.attestationVariant }}" in | |
| "gcp-sev-snp") | |
| cc_tech="SEV_SNP" | |
| ;; | |
| *) | |
| cc_tech="SEV" | |
| ;; | |
| esac | |
| cat >> _override.tf <<EOF | |
| locals { | |
| project_id = "constellation-e2e" | |
| region = "${region}" | |
| zone = "${{ inputs.regionZone || 'europe-west3-b' }}" | |
| cc_technology = "${cc_tech}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create AWS Terraform overrides | |
| if: steps.determine.outputs.cloudProvider == 'aws' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev) | |
| cat >> _override.tf <<EOF | |
| locals { | |
| region = "${region}" | |
| zone = "${{ inputs.regionZone || 'us-east-2c' }}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create Azure TDX Terraform overrides | |
| if: inputs.attestationVariant == 'azure-tdx' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| cat >> _override.tf <<EOF | |
| locals { | |
| instance_type = "Standard_DC4es_v5" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Copy example Terraform file | |
| working-directory: ${{ github.workspace }} | |
| shell: bash | |
| run: | | |
| cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ steps.determine.outputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf | |
| - name: Apply Terraform Cluster | |
| id: apply_terraform | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' | |
| terraform init | |
| if [[ "${{ inputs.attestationVariant }}" == "azure-sev-snp" ]]; then | |
| terraform apply -target module.azure_iam -auto-approve | |
| terraform apply -target module.azure_infrastructure -auto-approve | |
| ${{ github.workspace }}/build/constellation maa-patch "$(terraform output -raw maa_url)" | |
| terraform apply -target constellation_cluster.azure_example -auto-approve | |
| else | |
| terraform apply -auto-approve | |
| fi | |
| - name: Cleanup Terraform Cluster on failure | |
| # cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later | |
| # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) | |
| if: failure() && steps.apply_terraform.outcome != 'skipped' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform destroy -auto-approve | |
| - name: Add Provider to local Terraform registry # needed if release version was used before | |
| if: inputs.providerVersion != '' | |
| working-directory: ${{ github.workspace }}/build | |
| shell: bash | |
| run: | | |
| bazel run //:devbuild --cli_edition=enterprise | |
| - name: Update cluster configuration # for duplicate variable declaration, the last one is used | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| cat >> _override.tf <<EOF | |
| locals { | |
| image_version = "${{ inputs.toImage || steps.find-latest-image.outputs.image }}" | |
| } | |
| EOF | |
| if [[ "${{ inputs.toKubernetes }}" != "" ]]; then | |
| cat >> _override.tf <<EOF | |
| resource "constellation_cluster" "${{ steps.determine.outputs.cloudProvider }}_example" { | |
| kubernetes_version = "${{ inputs.toKubernetes }}" | |
| } | |
| EOF | |
| fi | |
| prefixed_version=${{ steps.build.outputs.build_version }} | |
| version=${prefixed_version#v} # remove v prefix | |
| # needs to be explicitly set to upgrade | |
| cat >> _override.tf <<EOF | |
| resource "constellation_cluster" "${{ steps.determine.outputs.cloudProvider }}_example" { | |
| constellation_microservice_version = "${prefixed_version}" | |
| } | |
| EOF | |
| cat >> _override.tf <<EOF | |
| terraform { | |
| required_providers { | |
| constellation = { | |
| source = "edgelesssys/constellation" | |
| version = "${version}" | |
| } | |
| } | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Upgrade Terraform Cluster | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init --upgrade | |
| terraform apply -auto-approve | |
| - name: Assert upgrade successful | |
| working-directory: ${{ github.workspace }}/cluster | |
| env: | |
| IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-latest-image.outputs.image }} | |
| KUBERNETES: ${{ inputs.toKubernetes }} | |
| MICROSERVICES: ${{ steps.build.outputs.build_version }} | |
| WORKERNODES: 1 | |
| CONTROLNODES: 1 | |
| run: | | |
| terraform output -raw kubeconfig > constellation-admin.conf | |
| if [[ -n "${MICROSERVICES}" ]]; then | |
| MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}" | |
| fi | |
| if [[ -n "${KUBERNETES}" ]]; then | |
| KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}" | |
| fi | |
| if [[ -n "${IMAGE}" ]]; then | |
| IMAGE_FLAG="--target-image=${IMAGE}" | |
| fi | |
| # cfg must be in same dir as KUBECONFIG | |
| ${{ github.workspace }}/build/constellation config generate "${{ steps.determine.outputs.cloudProvider }}" --attestation ${{ inputs.attestationVariant}} | |
| # make cfg valid with fake data | |
| # IMPORTANT: zone needs to be correct because it is used to resolve the CSP image ref | |
| if [[ "${{ steps.determine.outputs.cloudProvider }}" == "azure" ]]; then | |
| location="${{ inputs.regionZone || 'northeurope' }}" | |
| yq e ".provider.azure.location = \"${location}\"" -i constellation-conf.yaml | |
| yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml | |
| yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml | |
| yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml | |
| yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml | |
| fi | |
| if [[ "${{ steps.determine.outputs.cloudProvider }}" == "gcp" ]]; then | |
| zone="${{ inputs.regionZone || 'europe-west3-b' }}" | |
| region=$(echo "${zone}" | rev | cut -c 2- | rev) | |
| yq e ".provider.gcp.region = \"${region}\"" -i constellation-conf.yaml | |
| yq e ".provider.gcp.zone = \"${zone}\"" -i constellation-conf.yaml | |
| yq e '.provider.gcp.project = "demo-gcp-project"' -i constellation-conf.yaml | |
| yq e '.nodeGroups.control_plane_default.zone = "europe-west3-b"' -i constellation-conf.yaml | |
| # Set the zone for worker_default node group to a fictional value | |
| yq e '.nodeGroups.worker_default.zone = "europe-west3-b"' -i constellation-conf.yaml | |
| yq e '.provider.gcp.serviceAccountKeyPath = "/path/to/your/service-account-key.json"' -i constellation-conf.yaml | |
| fi | |
| if [[ "${{ steps.determine.outputs.cloudProvider }}" == "aws" ]]; then | |
| zone=${{ inputs.regionZone || 'us-east-2c' }} | |
| region=$(echo "${zone}" | rev | cut -c 2- | rev) | |
| yq e ".provider.aws.region = \"${region}\"" -i constellation-conf.yaml | |
| yq e ".provider.aws.zone = \"${zone}\"" -i constellation-conf.yaml | |
| yq e '.provider.aws.iamProfileControlPlane = "demoControlPlaneIAMProfile"' -i constellation-conf.yaml | |
| yq e '.provider.aws.iamProfileWorkerNodes = "demoWorkerNodesIAMProfile"' -i constellation-conf.yaml | |
| yq e '.nodeGroups.control_plane_default.zone = "eu-central-1a"' -i constellation-conf.yaml | |
| yq e '.nodeGroups.worker_default.zone = "eu-central-1a"' -i constellation-conf.yaml | |
| fi | |
| KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run --test_timeout=14400 //e2e/provider-upgrade:provider-upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --cli "${{ github.workspace }}/build/constellation" "$IMAGE_FLAG" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" | |
| - name: Destroy Terraform Cluster | |
| # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) | |
| if: always() && steps.apply_terraform.outcome != 'skipped' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform destroy -auto-approve | |
| - name: Notify about failure | |
| if: | | |
| failure() && | |
| github.ref == 'refs/heads/main' && | |
| github.event_name == 'schedule' | |
| continue-on-error: true | |
| uses: ./.github/actions/notify_e2e_failure | |
| with: | |
| projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} | |
| test: "terraform-provider-example" | |
| refStream: ${{ inputs.ref}} | |
| provider: ${{ steps.determine.outputs.cloudProvider }} | |
| kubernetesVersion: ${{ steps.kubernetes.outputs.version }} | |
| clusterCreation: "terraform" | |
| attestationVariant: ${{ inputs.attestationVariant }} |