Skip to content

Commit

Permalink
ci: enable STACKIT e2e tests (#3556)
Browse files Browse the repository at this point in the history
  • Loading branch information
msanft authored Dec 19, 2024
1 parent 50ab77e commit c467ff6
Show file tree
Hide file tree
Showing 8 changed files with 263 additions and 7 deletions.
13 changes: 8 additions & 5 deletions .github/actions/constellation_create/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,9 @@ runs:
continue-on-error: true
uses: ./.github/actions/artifact_upload
with:
name: serial-logs-${{ inputs.artifactNameSuffix }}
path: >
!(terraform).log
name: debug-logs-${{ inputs.artifactNameSuffix }}
path: |
*.log
encryptionSecret: ${{ inputs.encryptionSecret }}

- name: Prepare terraform state folders
Expand All @@ -268,9 +268,12 @@ runs:
run: |
mkdir to-zip
cp -r constellation-terraform to-zip
cp -r constellation-iam-terraform to-zip
# constellation-iam-terraform is optional
if [ -d constellation-iam-terraform ]; then
cp -r constellation-iam-terraform to-zip
fi
rm -f to-zip/constellation-terraform/plan.zip
rm -rf to-zip/constellation-terraform/.terraform to-zip/constellation-iam-terraform/.terraform
rm -rf to-zip/*/.terraform
- name: Upload terraform state
if: always()
Expand Down
18 changes: 18 additions & 0 deletions .github/actions/constellation_iam_create/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,15 @@ inputs:
gcpZone:
description: "The GCP zone to deploy Constellation in."
required: false
#
# STACKIT specific inputs
#
stackitZone:
description: "The STACKIT zone to deploy Constellation in."
required: false
stackitProjectID:
description: "The STACKIT project ID to deploy Constellation in."
required: false

runs:
using: "composite"
Expand Down Expand Up @@ -104,3 +113,12 @@ runs:
--update-config \
--tf-log=DEBUG \
--yes
- name: Set STACKIT-specific configuration
shell: bash
env:
STACKIT_PROJECT_ID: ${{ inputs.stackitProjectID }}
run: |
yq eval -i "(.provider.openstack.stackitProjectID) = \"${STACKIT_PROJECT_ID}\"" constellation-conf.yaml
yq eval -i "(.provider.openstack.availabilityZone) = \"${{ inputs.stackitZone }}\"" constellation-conf.yaml
yq eval -i "(.nodeGroups.[].zone) = \"${{ inputs.stackitZone }}\"" constellation-conf.yaml
25 changes: 24 additions & 1 deletion .github/actions/e2e_test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,15 @@ inputs:
encryptionSecret:
description: "The secret to use for decrypting the artifact."
required: true
openStackCloudsYaml:
description: "The contents of ~/.config/openstack/clouds.yaml"
required: false
stackitUat:
description: "The UAT for STACKIT"
required: false
stackitProjectID:
description: "The STACKIT project ID to deploy Constellation in."
required: false

outputs:
kubeconfig:
Expand Down Expand Up @@ -229,6 +238,18 @@ runs:
with:
azure_credentials: ${{ inputs.azureIAMCreateCredentials }}

- name: Login to OpenStack
if: inputs.cloudProvider == 'stackit'
uses: ./.github/actions/login_openstack
with:
clouds_yaml: ${{inputs.openStackCloudsYaml }}

- name: Login to STACKIT
if: inputs.cloudProvider == 'stackit'
uses: ./.github/actions/login_stackit
with:
serviceAccountToken: ${{ inputs.stackitUat }}

- name: Create prefix
id: create-prefix
shell: bash
Expand All @@ -244,7 +265,7 @@ runs:
with:
attestationVariant: ${{ inputs.attestationVariant }}

- name: Create IAM configuration
- name: Create Constellation config and IAM
id: constellation-iam-create
uses: ./.github/actions/constellation_iam_create
with:
Expand All @@ -256,6 +277,8 @@ runs:
azureRegion: ${{ inputs.regionZone || steps.pick-az-region.outputs.region }}
gcpProjectID: ${{ inputs.gcpProject }}
gcpZone: ${{ inputs.regionZone || 'europe-west3-b' }}
stackitZone: ${{ inputs.regionZone || 'eu01-2' }}
stackitProjectID: ${{ inputs.stackitProjectID }}
kubernetesVersion: ${{ inputs.kubernetesVersion }}
additionalTags: "workflow=${{ github.run_id }}"

Expand Down
16 changes: 16 additions & 0 deletions .github/actions/login_stackit/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
name: STACKIT login
description: "Login to STACKIT"
inputs:
serviceAccountToken:
description: "Credentials authorized to create Constellation on STACKIT."
required: true
runs:
using: "composite"
steps:
- name: Login to STACKIT
env:
UAT: ${{ inputs.serviceAccountToken }}
shell: bash
run: |
mkdir -p ~/.stackit
echo "${UAT}" > ~/.stackit/credentials.json
19 changes: 19 additions & 0 deletions .github/actions/notify_stackit/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
name: Notify STACKIT
description: "Notify STACKIT about test failure"
inputs:
slackToken:
description: "Slack access token."
required: true
runs:
using: "composite"
steps:
- name: Notify STACKIT
env:
SLACK_TOKEN: ${{ inputs.slackToken }}
shell: bash
run: |
curl -X POST \
-H "Authorization: Bearer $SLACK_TOKEN" \
-H "Content-type: application/json; charset=utf-8" \
-d "{\"channel\":\"C0827BT59SM\",\"text\":\"E2E test failed: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID\"}" \
https://slack.com/api/chat.postMessage
13 changes: 13 additions & 0 deletions .github/actions/terraform_apply/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ runs:
"gcpSEVSNP")
attestationVariant="gcp-sev-snp"
;;
"qemuVTPM")
attestationVariant="qemu-vtpm"
;;
*)
echo "Unknown attestation variant: $(yq '.attestation | keys | .[0]' constellation-conf.yaml)"
exit 1
Expand Down Expand Up @@ -106,6 +109,16 @@ runs:
project_id = "$(yq '.infrastructure.gcp.projectID' constellation-state.yaml)"
service_account_key = sensitive("$(cat $(yq '.provider.gcp.serviceAccountKeyPath' constellation-conf.yaml) | base64 -w0)")
}
openstack = {
cloud = "stackit"
clouds_yaml_path = "~/.config/openstack/clouds.yaml"
floating_ip_pool_id = "970ace5c-458f-484a-a660-0903bcfd91ad"
deploy_yawol_load_balancer = true
yawol_image_id = "bcd6c13e-75d1-4c3f-bf0f-8f83580cc1be"
yawol_flavor_id = "3b11b27e-6c73-470d-b595-1d85b95a8cdf"
network_id = "$(yq '.infrastructure.networkID' constellation-state.yaml)"
subnet_id = "$(yq '.infrastructure.subnetID' constellation-state.yaml)"
}
network_config = {
ip_cidr_node = "$(yq '.infrastructure.ipCidrNode' constellation-state.yaml)"
ip_cidr_service = "$(yq '.serviceCIDR' constellation-conf.yaml)"
Expand Down
153 changes: 153 additions & 0 deletions .github/workflows/e2e-test-stackit.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
name: e2e test STACKIT

on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *" # Every day at midnight.

jobs:
find-latest-image:
name: Find latest image
runs-on: ubuntu-24.04
permissions:
id-token: write
contents: read
outputs:
image-release-stable: ${{ steps.relabel-output.outputs.image-release-stable }}
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

- name: Select relevant image
id: select-image-action
uses: ./.github/actions/select_image
with:
osImage: "ref/release/stream/stable/?"

- name: Relabel output
id: relabel-output
shell: bash
run: |
ref=$(echo 'ref/release/stream/stable/?' | cut -d/ -f2)
stream=$(echo 'ref/release/stream/stable/?' | cut -d/ -f4)
echo "image-$ref-$stream=${{ steps.select-image-action.outputs.osImage }}" | tee -a "$GITHUB_OUTPUT"
e2e-stackit:
strategy:
fail-fast: false
max-parallel: 6
matrix:
kubernetesVersion: [ "1.28", "1.29", "1.30" ]
clusterCreation: [ "cli", "terraform" ]
test: [ "sonobuoy quick" ]
runs-on: ubuntu-24.04
permissions:
id-token: write
checks: write
contents: read
packages: write
actions: write
needs: [find-latest-image]
steps:
- name: Check out repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

- name: Setup bazel
uses: ./.github/actions/setup_bazel_nix
with:
nixTools: terraform

- name: Run E2E test
id: e2e_test
uses: ./.github/actions/e2e_test
with:
workerNodesCount: "1"
controlNodesCount: "1"
cloudProvider: stackit
attestationVariant: qemu-vtpm
osImage: ${{ needs.find-latest-image.outputs.image-release-stable }}
isDebugImage: false
cliVersion: ${{ needs.find-latest-image.outputs.image-release-stable || '' }}
kubernetesVersion: ${{ matrix.kubernetesVersion }}
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
gcpProject: constellation-e2e
gcpClusterCreateServiceAccount: "[email protected]"
gcpIAMCreateServiceAccount: "[email protected]"
test: ${{ matrix.test }}
azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
registry: ghcr.io
githubToken: ${{ secrets.GITHUB_TOKEN }}
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
fetchMeasurements: false
clusterCreation: ${{ matrix.clusterCreation }}
s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }}
s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }}
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }}
stackitUat: ${{ secrets.STACKIT_CI_UAT }}
stackitProjectID: ${{ secrets.STACKIT_CI_PROJECT_ID }}

- name: Always terminate cluster
if: always()
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
clusterCreation: ${{ matrix.clusterCreation }}
cloudProvider: stackit
azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
gcpClusterDeleteServiceAccount: "[email protected]"

- name: Always delete IAM configuration
if: always()
uses: ./.github/actions/constellation_iam_destroy
with:
cloudProvider: stackit
azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
gcpServiceAccount: "[email protected]"

- name: Update tfstate
if: always()
env:
GH_TOKEN: ${{ github.token }}
uses: ./.github/actions/update_tfstate
with:
name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }}
runID: ${{ github.run_id }}
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

- name: Notify about failure
if: |
failure() &&
github.ref == 'refs/heads/main' &&
github.event_name == 'schedule'
continue-on-error: true
uses: ./.github/actions/notify_e2e_failure
with:
projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
refStream: "ref/release/stream/stable/?"
test: ${{ matrix.test }}
kubernetesVersion: ${{ matrix.kubernetesVersion }}
provider: stackit
attestationVariant: qemu-vtpm
clusterCreation: ${{ matrix.clusterCreation }}

- name: Notify STACKIT
if: |
failure() &&
github.ref == 'refs/heads/main' &&
github.event_name == 'schedule'
continue-on-error: true
uses: ./.github/actions/notify_stackit
with:
slackToken: ${{ secrets.SLACK_TOKEN }}
13 changes: 12 additions & 1 deletion .github/workflows/e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ on:
- "azure-sev-snp"
- "azure-tdx"
- "aws-sev-snp"
- "stackit-qemu-vtpm"
default: "azure-sev-snp"
required: true
runner:
Expand Down Expand Up @@ -137,6 +138,7 @@ jobs:
workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }}
controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }}
cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }}
attestationVariant: ${{ steps.split-attestationVariant.outputs.attestationVariant }}
steps:
- name: Split nodeCount
id: split-nodeCount
Expand All @@ -161,6 +163,12 @@ jobs:
attestationVariant="${{ inputs.attestationVariant }}"
cloudProvider="${attestationVariant%%-*}"
# special case for STACKIT, as there's no special attestation variant for it
if [[ "${cloudProvider}" == "stackit" ]]; then
attestationVariant="qemu-vtpm"
fi
echo "attestationVariant=${attestationVariant}" | tee -a "$GITHUB_OUTPUT"
echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT"
find-latest-image:
Expand Down Expand Up @@ -233,7 +241,7 @@ jobs:
workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }}
controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }}
cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
attestationVariant: ${{ inputs.attestationVariant }}
attestationVariant: ${{ needs.generate-input-parameters.outputs.attestationVariant }}
machineType: ${{ inputs.machineType }}
regionZone: ${{ inputs.regionZone }}
gcpProject: constellation-e2e
Expand Down Expand Up @@ -262,6 +270,9 @@ jobs:
marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
force: ${{ inputs.force }}
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }}
stackitUat: ${{ secrets.STACKIT_CI_UAT }}
stackitProjectID: ${{ secrets.STACKIT_CI_PROJECT_ID }}

- name: Always terminate cluster
if: always()
Expand Down

0 comments on commit c467ff6

Please sign in to comment.