diff --git a/.github/actions/artifact_delete/delete_artifact.sh b/.github/actions/artifact_delete/delete_artifact.sh index 9c94f8c69b..956927116d 100755 --- a/.github/actions/artifact_delete/delete_artifact.sh +++ b/.github/actions/artifact_delete/delete_artifact.sh @@ -7,6 +7,7 @@ function get_artifact_id { artifact_id="$(gh api \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ + --paginate \ "/repos/edgelesssys/constellation/actions/runs/$1/artifacts" --jq ".artifacts |= map(select(.name==\"$2\")) | .artifacts[0].id" || exit 1)" echo "$artifact_id" } diff --git a/.github/actions/artifact_upload/action.yml b/.github/actions/artifact_upload/action.yml index 11dd9a0bdc..bedf3a3158 100644 --- a/.github/actions/artifact_upload/action.yml +++ b/.github/actions/artifact_upload/action.yml @@ -14,6 +14,10 @@ inputs: encryptionSecret: description: 'The secret to use for encrypting the files.' required: true + overwrite: + description: 'Overwrite an artifact with the same name.' + default: false + required: false runs: using: "composite" @@ -69,3 +73,4 @@ runs: path: ${{ steps.tempdir.outputs.directory }}/archive.zip retention-days: ${{ inputs.retention-days }} if-no-files-found: ignore + overwrite: ${{ inputs.overwrite }} diff --git a/.github/actions/e2e_cleanup_timeframe/action.yml b/.github/actions/e2e_cleanup_timeframe/action.yml new file mode 100644 index 0000000000..a9f125ecc2 --- /dev/null +++ b/.github/actions/e2e_cleanup_timeframe/action.yml @@ -0,0 +1,46 @@ +name: E2E cleanup over timeframe +description: Clean up old terraform resources of E2E tests + +inputs: + ghToken: + description: 'The github token that is used with the github CLI.' + required: true + encryptionSecret: + description: 'The secret to use for decrypting the artifacts.' + required: true + azure_credentials: + description: "Credentials authorized to create Constellation on Azure." + required: true + +runs: + using: "composite" + steps: + - name: Authenticate AWS + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + with: + role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EDestroy + aws-region: eu-central-1 + + - name: Authenticate Azure + uses: ./.github/actions/login_azure + with: + azure_credentials: ${{ inputs.azure_credentials }} + + - name: Authenticate GCP + uses: ./.github/actions/login_gcp + with: + service_account: "destroy-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Install unzip + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: | + unzip + - name: Run cleanup + run: ./.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh + shell: bash + env: + GH_TOKEN: ${{ inputs.ghToken }} + ENCRYPTION_SECRET: ${{ inputs.encryptionSecret }} + + diff --git a/.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh b/.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh new file mode 100755 index 0000000000..0da75f25b9 --- /dev/null +++ b/.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# get_e2e_test_ids_on_date gets all workflow IDs of workflows that contain "e2e" on a specific date. +function get_e2e_test_ids_on_date { + ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | contains("e2e") and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)" + echo "$ids" +} + +# download_tfstate_artifact downloads all artifacts matching the pattern terraform-state-* from a given workflow ID. +function download_tfstate_artifact { + gh run download "$1" -p "terraform-state-*" -R edgelesssys/constellation > /dev/null +} + +# delete_resources runs terraform destroy on the constellation-terraform subfolder of a given folder. +function delete_resources { + if [ -d "$1/constellation-terraform" ]; then + cd "$1/constellation-terraform" || exit 1 + terraform init > /dev/null || exit 1 # first, install plugins + terraform destroy -auto-approve || exit 1 + cd ../../ || exit 1 + fi +} + +# delete_iam_config runs terraform destroy on the constellation-iam-terraform subfolder of a given folder. +function delete_iam_config { + if [ -d "$1/constellation-iam-terraform" ]; then + cd "$1/constellation-iam-terraform" || exit 1 + terraform init > /dev/null || exit 1 # first, install plugins + terraform destroy -auto-approve || exit 1 + cd ../../ || exit 1 + fi +} + +# check if the password for artifact decryption was given +if [[ -z $ENCRYPTION_SECRET ]]; then + echo "ENCRYPTION_SECRET is not set. Please set an environment variable with that secret." + exit 1 +fi + +artifact_pwd=$ENCRYPTION_SECRET + +shopt -s nullglob + +start_date=$(date "+%Y-%m-%d") +end_date=$(date --date "-7 day" "+%Y-%m-%d") +dates_to_clean=() + +# get all dates of the last week +while [[ $end_date != "$start_date" ]]; do + dates_to_clean+=("$end_date") + end_date=$(date --date "$end_date +1 day" "+%Y-%m-%d") +done + +echo "[*] retrieving run IDs for cleanup" +database_ids=() +for d in "${dates_to_clean[@]}"; do + echo " retrieving run IDs from $d" + mapfile -td " " tmp < <(get_e2e_test_ids_on_date "$d") + database_ids+=("${tmp[*]}") +done + +# cleanup database_ids +mapfile -t database_ids < <(echo "${database_ids[@]}") +mapfile -td " " database_ids < <(echo "${database_ids[@]}") + +echo "[*] downloading terraform state artifacts" +for id in "${database_ids[@]}"; do + if [[ $id == *[^[:space:]]* ]]; then + echo " downloading from workflow $id" + download_tfstate_artifact "$id" + fi +done + +echo "[*] extracting artifacts" +for directory in ./terraform-state-*; do + echo " extracting $directory" + + # extract and decrypt the artifact + unzip -d "${directory}" -P "$artifact_pwd" "$directory/archive.zip" > /dev/null || exit 1 +done + +# create terraform caching directory +mkdir "$HOME/tf_plugin_cache" +export TF_PLUGIN_CACHE_DIR="$HOME/tf_plugin_cache" +echo "[*] created terraform cache directory $TF_PLUGIN_CACHE_DIR" + +echo "[*] deleting resources" +for directory in ./terraform-state-*; do + echo " deleting resources in $directory" + delete_resources "$directory" + echo " deleting IAM configuration in $directory" + delete_iam_config "$directory" + echo " deleting directory $directory" + rm -rf "$directory" +done + +exit 0 diff --git a/.github/actions/update_tfstate/action.yml b/.github/actions/update_tfstate/action.yml new file mode 100644 index 0000000000..58d9ef11e0 --- /dev/null +++ b/.github/actions/update_tfstate/action.yml @@ -0,0 +1,68 @@ +name: Update TFState +description: "Update the terraform state artifact." + +inputs: + name: + description: "The name of the artifact that contains the tfstate." + required: true + runID: + description: "The ID of your current run (github.run_id)." + required: true + encryptionSecret: + description: "The encryption secret for the artifacts." + required: true + skipDeletion: + description: "Don't try to delete the artifact before updating. You should only use this if you know that no artifact exists." + default: "false" + required: false + +runs: + using: "composite" + steps: + - name: Check if tfstate should be deleted + if: always() && ${{ inputs.skipDeletion }} == "false" + shell: bash + run: | + if [ ! -d constellation-terraform ] && [ ! -d constellation-iam-terraform ]; then + echo "DELETE_TF_STATE=true" >> "$GITHUB_ENV" + else + echo "DELETE_TF_STATE=false" >> "$GITHUB_ENV" + fi + + - name: Delete tfstate artifact if necessary + if: always() && env.DELETE_TF_STATE == 'true' && ${{ inputs.skipDeletion }} == "false" + uses: ./.github/actions/artifact_delete + with: + name: ${{ inputs.name }} + workflowID: ${{ inputs.runID }} + + - name: Prepare terraform state folders + if: always() + shell: bash + run: | + rm -rf to-zip/* + to_upload="" + if [ -d constellation-terraform ]; then + cp -r constellation-terraform to-zip + rm to-zip/constellation-terraform/plan.zip + rm -rf to-zip/constellation-terraform/.terraform + to_upload+="to-zip/constellation-terraform" + fi + if [ -d constellation-iam-terraform ]; then + cp -r constellation-iam-terraform to-zip + rm -rf to-zip/constellation-iam-terraform/.terraform + to_upload+=" to-zip/constellation-iam-terraform" + fi + echo "TO_UPLOAD=$to_upload" >> "$GITHUB_ENV" + + - name: Update tfstate + if: always() + uses: ./.github/actions/artifact_upload + with: + name: ${{ inputs.name }} + path: > + ${{ env.TO_UPLOAD }} + encryptionSecret: ${{ inputs.encryptionSecret }} + overwrite: true + + diff --git a/.github/workflows/e2e-cleanup-weekly.yml b/.github/workflows/e2e-cleanup-weekly.yml new file mode 100644 index 0000000000..aa0fa9e286 --- /dev/null +++ b/.github/workflows/e2e-cleanup-weekly.yml @@ -0,0 +1,24 @@ +name: e2e weekly cleanup + +on: + schedule: + - cron: "0 0 * * 0" # At 00:00 every Sunday UTC + workflow_dispatch: + + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + actions: read + id-token: write + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Cleanup + uses: ./.github/actions/e2e_cleanup_timeframe + with: + ghToken: ${{ secrets.GITHUB_TOKEN }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + azure_credentials: ${{ secrets.AZURE_E2E_DESTROY_CREDENTIALS }} diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index 05072a0997..7618c8e73a 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -56,6 +56,7 @@ jobs: checks: write contents: read packages: write + actions: write needs: [find-latest-image] steps: - name: Check out repository @@ -122,6 +123,16 @@ jobs: azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Notify about failure if: | failure() && diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index 4843d2cb7f..ba9b459479 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -303,6 +303,7 @@ jobs: checks: write contents: read packages: write + actions: write steps: - name: Install the basics tools (macOS) if: runner.os == 'macOS' @@ -378,6 +379,16 @@ jobs: azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + e2e-upgrade: strategy: fail-fast: false @@ -392,6 +403,7 @@ jobs: contents: read checks: write packages: write + actions: write uses: ./.github/workflows/e2e-upgrade.yml with: fromVersion: ${{ matrix.fromVersion }} diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 142ef85290..cf16c8b0ac 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -309,6 +309,7 @@ jobs: checks: write contents: read packages: write + actions: write needs: [find-latest-image] steps: - name: Check out repository @@ -378,6 +379,16 @@ jobs: azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Notify about failure if: | failure() && @@ -408,6 +419,7 @@ jobs: checks: write contents: read packages: write + actions: write uses: ./.github/workflows/e2e-upgrade.yml with: fromVersion: ${{ matrix.fromVersion }} @@ -464,9 +476,9 @@ jobs: name: Run Windows E2E test permissions: id-token: write - checks: write contents: read packages: write + checks: write secrets: inherit uses: ./.github/workflows/e2e-windows.yml with: diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 61c05349a5..71eb9f53a4 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -200,6 +200,7 @@ jobs: checks: write contents: read packages: write + actions: write needs: [find-latest-image, generate-input-parameters] if: always() && !cancelled() steps: @@ -278,3 +279,13 @@ jobs: cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index 34ff34a253..31ac0e1b85 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -189,6 +189,7 @@ jobs: needs: [generate-input-parameters] outputs: kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} + e2e-name-prefix: ${{ steps.e2e_test.outputs.namePrefix }} steps: - name: Checkout if: inputs.gitRef == 'head' @@ -441,6 +442,7 @@ jobs: checks: write contents: read packages: write + actions: write if: always() needs: [generate-input-parameters, create-cluster, e2e-upgrade] steps: @@ -505,6 +507,17 @@ jobs: constellation-version.yaml encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Prepare terraform state artifact upload + if: always() + shell: bash + run: | + mkdir -p to-zip + cp -r constellation-terraform to-zip + rm to-zip/constellation-terraform/plan.zip + rm -rf to-zip/constellation-terraform/.terraform + cp -r constellation-iam-terraform to-zip + rm -rf to-zip/constellation-iam-terraform/.terraform + - name: Always terminate cluster if: always() uses: ./.github/actions/constellation_destroy @@ -523,6 +536,16 @@ jobs: azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ needs.create-cluster.outputs.e2e-name-prefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Notify about failure if: | always() && diff --git a/.github/workflows/e2e-windows.yml b/.github/workflows/e2e-windows.yml index 96906afae9..893ff83067 100644 --- a/.github/workflows/e2e-windows.yml +++ b/.github/workflows/e2e-windows.yml @@ -195,3 +195,26 @@ jobs: test: Windows E2E Test provider: Azure attestationVariant: "azure-sev-snp" + + upload-tfstate: + name: Upload terraform state if it exists + runs-on: ubuntu-22.04 + needs: e2e-test + if: always() + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Upload tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ github.run_id }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + skipDeletion: "true" +