e2e test Terraform provider example #136
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e test Terraform provider example | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| type: string | |
| description: "Git ref to checkout" | |
| cloudProvider: | |
| description: "Which cloud provider to use." | |
| type: choice | |
| options: | |
| - "aws" | |
| - "azure" | |
| - "gcp" | |
| required: true | |
| regionZone: | |
| description: "Region or zone to create the cluster in. Leave empty for default region/zone." | |
| type: string | |
| image: | |
| description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used." | |
| type: string | |
| providerVersion: | |
| description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." | |
| type: string | |
| toImage: | |
| description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. | |
| type: string | |
| required: false | |
| toKubernetes: | |
| description: Kubernetes version to target for the upgrade, empty for no upgrade. | |
| type: string | |
| required: false | |
| toProviderVersion: | |
| description: Provider version and microservice version to target for the upgrade (with v prefix). Empty value means use provider from source. | |
| type: string | |
| required: false | |
| workflow_call: | |
| inputs: | |
| ref: | |
| type: string | |
| description: "Git ref to checkout" | |
| cloudProvider: | |
| description: "Which cloud provider to use." | |
| type: string | |
| required: true | |
| regionZone: | |
| description: "Which zone to use." | |
| type: string | |
| image: | |
| description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used." | |
| type: string | |
| providerVersion: | |
| description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." | |
| type: string | |
| toImage: | |
| description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. | |
| type: string | |
| required: false | |
| toKubernetes: | |
| description: Kubernetes version to target for the upgrade, empty for target's default version. | |
| type: string | |
| required: false | |
| toProviderVersion: | |
| description: Provider version and microservice version to target for the upgrade (with v prefix). Empty value means use provider from source. | |
| type: string | |
| required: false | |
| jobs: | |
| provider-example-test: | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| with: | |
| ref: ${{ inputs.ref || github.head_ref }} | |
| - name: Get Latest Image | |
| id: find-latest-image | |
| uses: ./.github/actions/find_latest_image | |
| with: | |
| git-ref: ${{ inputs.ref }} | |
| imageVersion: ${{ inputs.image }} | |
| ref: main | |
| stream: nightly | |
| - name: Upload Terraform module | |
| uses: ./.github/actions/upload_terraform_module | |
| - name: Download Terraform module | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | |
| with: | |
| name: terraform-module | |
| - name: Unzip Terraform module | |
| shell: bash | |
| run: | | |
| unzip terraform-module.zip -d ${{ github.workspace }} | |
| rm terraform-module.zip | |
| - name: Create resource prefix | |
| id: create-prefix | |
| shell: bash | |
| run: | | |
| run_id=${{ github.run_id }} | |
| last_three="${run_id: -3}" | |
| echo "prefix=e2e-${last_three}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Install dependencies (Terraform) | |
| shell: bash | |
| run: | | |
| sudo apt update && sudo apt install gpg | |
| wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg | |
| gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint | |
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list | |
| sudo apt update | |
| sudo apt install terraform=1.4.4-* | |
| - name: Log in to the Container registry | |
| uses: ./.github/actions/container_registry_login | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup bazel | |
| if: inputs.providerVersion == '' || inputs.toProviderVersion == '' | |
| uses: ./.github/actions/setup_bazel_nix | |
| with: | |
| useCache: "true" | |
| buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} | |
| - name: Build Terraform provider binary | |
| if: inputs.providerVersion == '' || inputs.toProviderVersion == '' | |
| uses: ./.github/actions/build_tf_provider | |
| with: | |
| targetOS: 'linux' | |
| targetArch: 'amd64' | |
| outputPath: "build/terraform-provider-constellation" | |
| - name: Add to local Terraform registry | |
| if: inputs.providerVersion == '' # only move if needed, because otherwise the public TF registry will not be used | |
| shell: bash | |
| id: install_tf_provider | |
| run: | | |
| bazel build //bazel/settings:tag | |
| repository_root=$(git rev-parse --show-toplevel) | |
| out_rel=$(bazel cquery --output=files //bazel/settings:tag) | |
| build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") | |
| terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/" | |
| mkdir -p "${terraform_provider_dir}" | |
| mv build/terraform-provider-constellation "${terraform_provider_dir}/terraform-provider-constellation_${build_version}" | |
| echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Login to AWS (IAM + Cluster role) | |
| if: inputs.cloudProvider == 'aws' | |
| uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 | |
| with: | |
| role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform | |
| aws-region: eu-central-1 | |
| # extend token expiry to 6 hours to ensure constellation can terminate | |
| role-duration-seconds: 21600 | |
| - name: Login to Azure (IAM + Cluster service principal) | |
| if: inputs.cloudProvider == 'azure' | |
| uses: ./.github/actions/login_azure | |
| with: | |
| azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }} | |
| - name: Login to GCP (IAM + Cluster service account) | |
| if: inputs.cloudProvider == 'gcp' | |
| uses: ./.github/actions/login_gcp | |
| with: | |
| service_account: "[email protected]" | |
| - name: Common CSP Terraform overrides | |
| working-directory: ${{ github.workspace }} | |
| shell: bash | |
| run: | | |
| mkdir cluster | |
| cd cluster | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| prefixed_version=${{ steps.install_tf_provider.outputs.build_version }} | |
| else | |
| prefixed_version="${{ inputs.providerVersion }}" | |
| fi | |
| version=${prefixed_version#v} # remove v prefix | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| iam_src="../terraform-module/iam/${{ inputs.cloudProvider }}" | |
| infra_src="../terraform-module/${{ inputs.cloudProvider }}" | |
| else | |
| iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ inputs.cloudProvider }}" | |
| infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ inputs.cloudProvider }}" | |
| fi | |
| cat > _override.tf <<EOF | |
| terraform { | |
| required_providers { | |
| constellation = { | |
| source = "edgelesssys/constellation" | |
| version = "${version}" | |
| } | |
| } | |
| } | |
| locals { | |
| name = "${{ steps.create-prefix.outputs.prefix }}" | |
| version = "${{ steps.find-latest-image.outputs.image }}" | |
| } | |
| module "${{ inputs.cloudProvider }}_iam" { | |
| source = "${iam_src}" | |
| } | |
| module "${{ inputs.cloudProvider }}_infrastructure" { | |
| source = "${infra_src}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create GCP Terraform overrides | |
| if: inputs.cloudProvider == 'gcp' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev) | |
| cat >> _override.tf <<EOF | |
| locals { | |
| project_id = "constellation-e2e" | |
| region = "${region}" | |
| zone = "${{ inputs.regionZone || 'europe-west3-b' }}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create AWS Terraform overrides | |
| if: inputs.cloudProvider == 'aws' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev) | |
| cat >> _override.tf <<EOF | |
| locals { | |
| region = "${region}" | |
| zone = "${{ inputs.regionZone || 'us-east-2c' }}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Copy example Terraform file | |
| working-directory: ${{ github.workspace }} | |
| shell: bash | |
| run: | | |
| cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf | |
| - name: Download CLI # needed for Azure MAA patch and the upgrade test | |
| shell: bash | |
| run: | | |
| curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/v2.13.0/constellation-linux-amd64 | |
| chmod u+x constellation | |
| ./constellation version | |
| cp ./constellation ${{ github.workspace }}/cluster | |
| - name: Apply Terraform Cluster | |
| id: apply_terraform | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then | |
| terraform apply -target module.azure_iam -auto-approve | |
| terraform apply -target module.azure_infrastructure -auto-approve | |
| ./constellation maa-patch "$(terraform output -raw maa_url)" | |
| terraform apply -target constellation_cluster.azure_example -auto-approve | |
| else | |
| terraform apply -auto-approve | |
| fi | |
| - name: Cleanup Terraform Cluster on failure | |
| # cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later | |
| # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) | |
| if: failure() && steps.apply_terraform.outcome != 'skipped' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform destroy -auto-approve | |
| - name: Remove or add to local Terraform registry | |
| shell: bash | |
| id: upgrade_tf_provider | |
| run: | | |
| bazel build //bazel/settings:tag | |
| repository_root=$(git rev-parse --show-toplevel) | |
| out_rel=$(bazel cquery --output=files //bazel/settings:tag) | |
| build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") | |
| terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/" | |
| # remove local registry if it was used and public registry should be used now | |
| if [[ "${{ inputs.providerVersion }}" == "" && "${{ inputs.toProviderVersion }}" != "" ]]; then | |
| rm -rf "${terraform_provider_dir}" | |
| fi | |
| # add to local registry if it was not used and local registry should be used now | |
| if [[ "${{ inputs.providerVersion }}" != "" && "${{ inputs.toProviderVersion }}" == "" ]]; then | |
| mkdir -p "${terraform_provider_dir}" | |
| mv build/terraform-provider-constellation "${terraform_provider_dir}/terraform-provider-constellation_${build_version}" | |
| fi | |
| echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Update cluster configuration # for duplicate variable declaration, the last one is used | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.toImage }}" != "" ]]; then | |
| cat >> _override.tf <<EOF | |
| locals { | |
| version = "${{ inputs.toImage }}" | |
| } | |
| EOF | |
| fi | |
| if [[ "${{ inputs.toKubernetes }}" != "" ]]; then | |
| cat >> _override.tf <<EOF | |
| resource "constellation_cluster" "${{ inputs.cloudProvider }}_example" { | |
| kubernetes_version = "${{ inputs.toKubernetes }}" | |
| } | |
| EOF | |
| fi | |
| if [[ "${{ inputs.toProviderVersion }}" == "" ]]; then | |
| prefixed_version=${{ steps.upgrade_tf_provider.outputs.build_version }} | |
| else | |
| prefixed_version="${{ inputs.toProviderVersion }}" | |
| fi | |
| version=${prefixed_version#v} # remove v prefix | |
| if [[ "${{ inputs.toProviderVersion }}" == "" ]]; then | |
| cat >> _override.tf <<EOF | |
| resource "constellation_cluster" "${{ inputs.cloudProvider }}_example" { | |
| constellation_microservice_version = "${prefixed_version}" | |
| } | |
| EOF | |
| fi | |
| if [[ "${{ inputs.toProviderVersion }}" == "" ]]; then | |
| iam_src="../terraform-module/iam/${{ inputs.cloudProvider }}" | |
| infra_src="../terraform-module/${{ inputs.cloudProvider }}" | |
| else | |
| iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.toProviderVersion }}/terraform-module.zip//terraform-module/iam/${{ inputs.cloudProvider }}" | |
| infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.toProviderVersion }}/terraform-module.zip//terraform-module/${{ inputs.cloudProvider }}" | |
| fi | |
| cat >> _override.tf <<EOF | |
| terraform { | |
| required_providers { | |
| constellation = { | |
| source = "edgelesssys/constellation" | |
| version = "${version}" | |
| } | |
| } | |
| } | |
| EOF | |
| cat _override.tf | |
| # TODO rename version to image_version | |
| - name: Upgrade Terraform Cluster | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init --upgrade | |
| terraform apply -auto-approve | |
| - name: Assert upgrade successful | |
| working-directory: ${{ github.workspace }}/cluster | |
| env: | |
| IMAGE: ${{ inputs.toImage }} | |
| KUBERNETES: ${{ inputs.toKubernetes }} | |
| MICROSERVICES: ${{ inputs.toProviderVersion }} | |
| WORKERNODES: 2 | |
| CONTROLNODES: 3 | |
| run: | | |
| terraform output -raw kubeconfig > constellation-admin.conf | |
| if [[ -n "${MICROSERVICES}" ]]; then | |
| MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}" | |
| fi | |
| if [[ -n "${KUBERNETES}" ]]; then | |
| KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}" | |
| fi | |
| if [[ -n "${IMAGE}" ]]; then | |
| IMAGE_FLAG="--target-image=${IMAGE}" | |
| fi | |
| # cfg must be in same dir as KUBECONFIG | |
| ./constellation config generate ${{ inputs.cloudProvider }} | |
| # make cfg valid | |
| # TODO other CSPs | |
| yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml | |
| yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml | |
| yq e '.provider.azure.location = "eastus"' -i constellation-conf.yaml | |
| yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml | |
| yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml | |
| KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker $WORKERNODES --want-control $CONTROLNODES --cli ${{ github.workspace }}/cluster/constellation $IMAGE_FLAG $KUBERNETES_FLAG $MICROSERVICES_FLAG | |
| - name: Destroy Terraform Cluster | |
| # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) | |
| if: always() && steps.apply_terraform.outcome != 'skipped' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform destroy -auto-approve | |
| - name: Notify about failure | |
| if: | | |
| failure() && | |
| github.ref == 'refs/heads/main' && | |
| github.event_name == 'schedule' | |
| continue-on-error: true | |
| uses: ./.github/actions/notify_e2e_failure | |
| with: | |
| projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} | |
| test: "terraform-provider-example" | |
| provider: ${{ inputs.cloudProvider }} |