e2e test Terraform provider example #162
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e test Terraform provider example | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| type: string | |
| description: "Git ref to checkout" | |
| cloudProvider: | |
| description: "Which cloud provider to use." | |
| type: choice | |
| options: | |
| - "aws" | |
| - "azure" | |
| - "gcp" | |
| required: true | |
| regionZone: | |
| description: "Region or zone to create the cluster in. Leave empty for default region/zone." | |
| type: string | |
| image: | |
| description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used." | |
| type: string | |
| providerVersion: | |
| description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." | |
| type: string | |
| toImage: | |
| description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. | |
| type: string | |
| required: false | |
| toKubernetes: | |
| description: Kubernetes version to target for the upgrade, empty for no upgrade. | |
| type: string | |
| required: false | |
| workflow_call: | |
| inputs: | |
| ref: | |
| type: string | |
| description: "Git ref to checkout" | |
| cloudProvider: | |
| description: "Which cloud provider to use." | |
| type: string | |
| required: true | |
| regionZone: | |
| description: "Which zone to use." | |
| type: string | |
| image: | |
| description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used." | |
| type: string | |
| providerVersion: | |
| description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." | |
| type: string | |
| toImage: | |
| description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. | |
| type: string | |
| required: false | |
| toKubernetes: | |
| description: Kubernetes version to target for the upgrade, empty for target's default version. | |
| type: string | |
| required: false | |
| jobs: | |
| provider-example-test: | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| with: | |
| ref: ${{ inputs.ref || github.head_ref }} | |
| - name: Get Latest Image | |
| id: find-latest-image | |
| uses: ./.github/actions/find_latest_image | |
| with: | |
| git-ref: ${{ inputs.ref }} | |
| imageVersion: ${{ inputs.image }} | |
| ref: main | |
| stream: nightly | |
| - name: Upload Terraform module | |
| uses: ./.github/actions/upload_terraform_module | |
| - name: Download Terraform module | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | |
| with: | |
| name: terraform-module | |
| - name: Unzip Terraform module | |
| shell: bash | |
| run: | | |
| unzip terraform-module.zip -d ${{ github.workspace }} | |
| rm terraform-module.zip | |
| - name: Create resource prefix | |
| id: create-prefix | |
| shell: bash | |
| run: | | |
| run_id=${{ github.run_id }} | |
| last_three="${run_id: -3}" | |
| echo "prefix=e2e-${last_three}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Log in to the Container registry | |
| uses: ./.github/actions/container_registry_login | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup bazel | |
| uses: ./.github/actions/setup_bazel_nix | |
| with: | |
| useCache: "true" | |
| buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} | |
| nixTools: terraform | |
| - name: Build Constellation provider # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade | |
| shell: bash | |
| run: | | |
| mkdir build | |
| cd build | |
| bazel run //:devbuild --cli_edition=enterprise | |
| - name: Login to AWS (IAM + Cluster role) | |
| if: inputs.cloudProvider == 'aws' | |
| uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 | |
| with: | |
| role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform | |
| aws-region: eu-central-1 | |
| # extend token expiry to 6 hours to ensure constellation can terminate | |
| role-duration-seconds: 21600 | |
| - name: Login to Azure (IAM + Cluster service principal) | |
| if: inputs.cloudProvider == 'azure' | |
| uses: ./.github/actions/login_azure | |
| with: | |
| azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }} | |
| - name: Login to GCP (IAM + Cluster service account) | |
| if: inputs.cloudProvider == 'gcp' | |
| uses: ./.github/actions/login_gcp | |
| with: | |
| service_account: "[email protected]" | |
| - name: Common CSP Terraform overrides | |
| working-directory: ${{ github.workspace }} | |
| shell: bash | |
| run: | | |
| mkdir cluster | |
| cd cluster | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| prefixed_version=${{ steps.install_tf_provider.outputs.build_version }} | |
| else | |
| prefixed_version="${{ inputs.providerVersion }}" | |
| fi | |
| version=${prefixed_version#v} # remove v prefix | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| iam_src="../terraform-module/iam/${{ inputs.cloudProvider }}" | |
| infra_src="../terraform-module/${{ inputs.cloudProvider }}" | |
| else | |
| iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ inputs.cloudProvider }}" | |
| infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ inputs.cloudProvider }}" | |
| fi | |
| # by default use latest nightly image for devbuilds and release image otherwise | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| if [[ "${{ inputs.image }}" == "" ]]; then | |
| image_version="${{ steps.find-latest-image.outputs.image }}" | |
| else | |
| image_version="${{ inputs.image }}" | |
| fi | |
| else | |
| if [[ "${{ inputs.image }}" == "" ]]; then | |
| image_version="${prefixed_version}" | |
| else | |
| image_version="${{ inputs.image }}" | |
| fi | |
| fi | |
| cat > _override.tf <<EOF | |
| terraform { | |
| required_providers { | |
| constellation = { | |
| source = "edgelesssys/constellation" | |
| version = "${version}" | |
| } | |
| } | |
| } | |
| locals { | |
| control_plane_count = 1 | |
| worker_count = 1 | |
| } | |
| locals { | |
| name = "${{ steps.create-prefix.outputs.prefix }}" | |
| image_version = "${image_version}" | |
| } | |
| module "${{ inputs.cloudProvider }}_iam" { | |
| source = "${iam_src}" | |
| } | |
| module "${{ inputs.cloudProvider }}_infrastructure" { | |
| source = "${infra_src}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create GCP Terraform overrides | |
| if: inputs.cloudProvider == 'gcp' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev) | |
| cat >> _override.tf <<EOF | |
| locals { | |
| project_id = "constellation-e2e" | |
| region = "${region}" | |
| zone = "${{ inputs.regionZone || 'europe-west3-b' }}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Create AWS Terraform overrides | |
| if: inputs.cloudProvider == 'aws' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev) | |
| cat >> _override.tf <<EOF | |
| locals { | |
| region = "${region}" | |
| zone = "${{ inputs.regionZone || 'us-east-2c' }}" | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Copy example Terraform file | |
| working-directory: ${{ github.workspace }} | |
| shell: bash | |
| run: | | |
| cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf | |
| - name: Apply Terraform Cluster | |
| id: apply_terraform | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then | |
| terraform apply -target module.azure_iam -auto-approve | |
| terraform apply -target module.azure_infrastructure -auto-approve | |
| ../build/constellation maa-patch "$(terraform output -raw maa_url)" | |
| terraform apply -target constellation_cluster.azure_example -auto-approve | |
| else | |
| terraform apply -auto-approve | |
| fi | |
| - name: Output kubeconfig # to debug the cluster during the test | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform output -raw kubeconfig | |
| - name: Cleanup Terraform Cluster on failure | |
| # cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later | |
| # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) | |
| if: failure() && steps.apply_terraform.outcome != 'skipped' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform destroy -auto-approve | |
| - name: Remove or add to local Terraform registry | |
| shell: bash | |
| id: upgrade_tf_provider | |
| run: | | |
| bazel build //bazel/settings:tag | |
| repository_root=$(git rev-parse --show-toplevel) | |
| out_rel=$(bazel cquery --output=files //bazel/settings:tag) | |
| build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") | |
| terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/" | |
| # remove local registry if it was used and public registry should be used now | |
| if [[ "${{ inputs.providerVersion }}" == "" ]]; then | |
| rm -rf "${terraform_provider_dir}" | |
| fi | |
| # add to local registry if it was not used and local registry should be used now | |
| if [[ "${{ inputs.providerVersion }}" != "" ]]; then | |
| mkdir -p "${terraform_provider_dir}" | |
| mv build/terraform-provider-constellation "${terraform_provider_dir}/terraform-provider-constellation_${build_version}" | |
| fi | |
| echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Update cluster configuration # for duplicate variable declaration, the last one is used | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.toImage }}" != "" ]]; then | |
| cat >> _override.tf <<EOF | |
| locals { | |
| image_version = "${{ inputs.toImage && inputs.toImage || steps.find-latest-image.outputs.output }}" | |
| } | |
| EOF | |
| fi | |
| if [[ "${{ inputs.toKubernetes }}" != "" ]]; then | |
| cat >> _override.tf <<EOF | |
| resource "constellation_cluster" "${{ inputs.cloudProvider }}_example" { | |
| kubernetes_version = "${{ inputs.toKubernetes }}" | |
| } | |
| EOF | |
| fi | |
| prefixed_version=${{ steps.upgrade_tf_provider.outputs.build_version }} | |
| version=${prefixed_version#v} # remove v prefix | |
| # needs to be explicitly set to upgrade | |
| cat >> _override.tf <<EOF | |
| resource "constellation_cluster" "${{ inputs.cloudProvider }}_example" { | |
| constellation_microservice_version = "${prefixed_version}" | |
| } | |
| EOF | |
| cat >> _override.tf <<EOF | |
| terraform { | |
| required_providers { | |
| constellation = { | |
| source = "edgelesssys/constellation" | |
| version = "${version}" | |
| } | |
| } | |
| } | |
| EOF | |
| cat _override.tf | |
| - name: Upgrade Terraform Cluster | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init --upgrade | |
| terraform apply -auto-approve | |
| - name: Assert upgrade successful | |
| working-directory: ${{ github.workspace }}/cluster | |
| env: | |
| IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-latest-image.outputs.image }} | |
| KUBERNETES: ${{ inputs.toKubernetes }} | |
| MICROSERVICES: ${{ steps.find-latest-image.outputs.image }} | |
| WORKERNODES: 1 | |
| CONTROLNODES: 1 | |
| run: | | |
| terraform output -raw kubeconfig > constellation-admin.conf | |
| if [[ -n "${MICROSERVICES}" ]]; then | |
| MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}" | |
| fi | |
| if [[ -n "${KUBERNETES}" ]]; then | |
| KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}" | |
| fi | |
| if [[ -n "${IMAGE}" ]]; then | |
| IMAGE_FLAG="--target-image=${IMAGE}" | |
| fi | |
| # cfg must be in same dir as KUBECONFIG | |
| ../build/constellation config generate "${{ inputs.cloudProvider }}" | |
| # make cfg valid with fake data | |
| # IMPORTANT: zone needs to be correct because it is used to resolve the CSP image ref | |
| if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then | |
| location="${{ inputs.regionZone || 'northeurope' }}" | |
| yq e ".provider.azure.location = \"${location}\"" -i constellation-conf.yaml | |
| yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml | |
| yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml | |
| yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml | |
| yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml | |
| fi | |
| if [[ "${{ inputs.cloudProvider }}" == "gcp" ]]; then | |
| zone="${{ inputs.regionZone || 'europe-west3-b' }}" | |
| region=$(echo "${zone}" | rev | cut -c 2- | rev) | |
| yq e ".provider.gcp.region = \"${region}\"" -i constellation-conf.yaml | |
| yq e ".provider.gcp.zone = \"${zone}\"" -i constellation-conf.yaml | |
| yq e '.provider.gcp.project = "demo-gcp-project"' -i constellation-conf.yaml | |
| yq e '.nodeGroups.control_plane_default.zone = "europe-west3-b"' -i constellation-conf.yaml | |
| # Set the zone for worker_default node group to a fictional value | |
| yq e '.nodeGroups.worker_default.zone = "europe-west3-b"' -i constellation-conf.yaml | |
| yq e '.provider.gcp.serviceAccountKeyPath = "/path/to/your/service-account-key.json"' -i constellation-conf.yaml | |
| fi | |
| if [[ "${{ inputs.cloudProvider }}" == "aws" ]]; then | |
| zone=${{ inputs.regionZone || 'us-east-2c' }} | |
| region=$(echo "${zone}" | rev | cut -c 2- | rev) | |
| yq e ".provider.aws.region = \"${region}\"" -i constellation-conf.yaml | |
| yq e ".provider.aws.zone = \"${zone}\"" -i constellation-conf.yaml | |
| yq e '.provider.aws.iamProfileControlPlane = "demoControlPlaneIAMProfile"' -i constellation-conf.yaml | |
| yq e '.provider.aws.iamProfileWorkerNodes = "demoWorkerNodesIAMProfile"' -i constellation-conf.yaml | |
| yq e '.nodeGroups.control_plane_default.zone = "eu-central-1a"' -i constellation-conf.yaml | |
| yq e '.nodeGroups.worker_default.zone = "eu-central-1a"' -i constellation-conf.yaml | |
| fi | |
| KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --cli "${{ github.workspace }}/build/constellation" "$IMAGE_FLAG" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" | |
| - name: Destroy Terraform Cluster | |
| # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) | |
| if: always() && steps.apply_terraform.outcome != 'skipped' | |
| working-directory: ${{ github.workspace }}/cluster | |
| shell: bash | |
| run: | | |
| terraform init | |
| terraform destroy -auto-approve | |
| - name: Notify about failure | |
| if: | | |
| failure() && | |
| github.ref == 'refs/heads/main' && | |
| github.event_name == 'schedule' | |
| continue-on-error: true | |
| uses: ./.github/actions/notify_e2e_failure | |
| with: | |
| projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} | |
| test: "terraform-provider-example" | |
| provider: ${{ inputs.cloudProvider }} |