From 212aa64f107a69d16757735cbbce6db5fa341585 Mon Sep 17 00:00:00 2001
From: Adrian Stobbe <stobbe.adrian@gmail.com>
Date: Thu, 25 Jul 2024 17:38:17 +0200
Subject: [PATCH] ci: add conformance test to weekly (#3230)

---
 .github/actions/constellation_create/action.yml |  9 ++++++++-
 .github/actions/e2e_sonobuoy/action.yml         |  6 ++++++
 .github/actions/e2e_test/action.yml             | 13 +++++++++++--
 .github/workflows/e2e-test-weekly.yml           |  7 +++++++
 .github/workflows/e2e-test.yml                  |  5 +++--
 5 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml
index 9aed0ac589..63cddf9d85 100644
--- a/.github/actions/constellation_create/action.yml
+++ b/.github/actions/constellation_create/action.yml
@@ -192,6 +192,13 @@ runs:
       run: |
         echo "flag=--force" | tee -a $GITHUB_OUTPUT
 
+    - name: Set conformance flag
+      id: set-conformance-flag
+      if: inputs.test == 'sonobuoy conformance'
+      shell: bash
+      run: |
+        echo "flag=--conformance" | tee -a $GITHUB_OUTPUT
+
     - name: Constellation apply (Terraform)
       id: constellation-apply-terraform
       if: inputs.clusterCreation == 'terraform'
@@ -204,7 +211,7 @@ runs:
       if: inputs.clusterCreation != 'terraform'
       shell: bash
       run: |
-        constellation apply --skip-phases=infrastructure --debug ${{ steps.set-force-flag.outputs.flag }}
+        constellation apply --skip-phases=infrastructure --debug ${{ steps.set-force-flag.outputs.flag }} ${{ steps.set-conformance-flag.outputs.flag }}
 
     - name: Get kubeconfig
       id: get-kubeconfig
diff --git a/.github/actions/e2e_sonobuoy/action.yml b/.github/actions/e2e_sonobuoy/action.yml
index 70dfed0e0b..5671e7c9cd 100644
--- a/.github/actions/e2e_sonobuoy/action.yml
+++ b/.github/actions/e2e_sonobuoy/action.yml
@@ -48,6 +48,12 @@ runs:
         sonobuoy results *_sonobuoy_*.tar.gz
         sonobuoy results *_sonobuoy_*.tar.gz --mode detailed | jq 'select(.status!="passed")' | jq 'select(.status!="skipped")' || true
 
+    - name: Cleanup sonobuoy deployment
+      env:
+        KUBECONFIG: ${{ inputs.kubeconfig }}
+      shell: bash
+      run: sonobuoy delete --wait
+
     - name: Upload test results
       if: always() && !env.ACT
       uses: ./.github/actions/artifact_upload
diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml
index 05ccbd016b..bcd315cbd6 100644
--- a/.github/actions/e2e_test/action.yml
+++ b/.github/actions/e2e_test/action.yml
@@ -53,7 +53,7 @@ inputs:
     description: "Azure credentials authorized to create an IAM configuration."
     required: true
   test:
-    description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, malicious join, nop, upgrade]."
+    description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, sonobuoy conformance, autoscaling, lb, perf-bench, verify, recover, malicious join, nop, upgrade]."
     required: true
   sonobuoyTestSuiteCmd:
     description: "The sonobuoy test suite to run."
@@ -103,7 +103,7 @@ runs:
   using: "composite"
   steps:
     - name: Check input
-      if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "malicious join", "s3proxy", "nop", "upgrade"]'), inputs.test))
+      if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "sonobuoy conformance", "autoscaling", "perf-bench", "verify", "lb", "recover", "malicious join", "s3proxy", "nop", "upgrade"]'), inputs.test))
       shell: bash
       run: |
         echo "::error::Invalid input for test field: ${{ inputs.test }}"
@@ -348,6 +348,15 @@ runs:
         artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
         encryptionSecret: ${{ inputs.encryptionSecret }}
 
+    - name: Run sonobuoy conformance
+      if: inputs.test == 'sonobuoy conformance'
+      uses: ./.github/actions/e2e_sonobuoy
+      with:
+        sonobuoyTestSuiteCmd: "--plugin e2e --mode certified-conformance"
+        kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
+        artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
+        encryptionSecret: ${{ inputs.encryptionSecret }}
+
     - name: Run autoscaling test
       if: inputs.test == 'autoscaling'
       uses: ./.github/actions/e2e_autoscaling
diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml
index 4aaa8f3727..6ae6382f12 100644
--- a/.github/workflows/e2e-test-weekly.yml
+++ b/.github/workflows/e2e-test-weekly.yml
@@ -78,6 +78,13 @@ jobs:
             kubernetes-version: "v1.30"
             clusterCreation: "cli"
 
+          # Sonobuoy conformance test
+          - test: "sonobuoy conformance"
+            refStream: "ref/main/stream/debug/?"
+            attestationVariant: "gcp-sev-snp"
+            kubernetes-version: "v1.30"
+            clusterCreation: "cli"
+
           # Sonobuoy quick test on all but the latest k8s versions
           - test: "sonobuoy quick"
             refStream: "ref/main/stream/debug/?"
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index bc0d06ac34..1694b5ac66 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -26,11 +26,12 @@ on:
           - "macos-12"
         default: "ubuntu-22.04"
       test:
-        description: "The test to run."
+        description: "The test to run. The conformance test is only supported for clusterCreation=cli."
         type: choice
         options:
           - "sonobuoy quick"
           - "sonobuoy full"
+          - "sonobuoy conformance"
           - "autoscaling"
           - "lb"
           - "perf-bench"
@@ -82,7 +83,7 @@ on:
         type: string
         required: true
       test:
-        description: "The test to run."
+        description: "The test to run. The conformance test is only supported for clusterCreation=cli."
         type: string
         required: true
       kubernetesVersion: