Run all CI tests on prerelease

.github/workflows/release-prod.yaml

@@ -275,17 +275,40 @@ jobs:
           APP_CHANNEL: CI
           USES_DEV_BUCKET: "0"
         run: |
+          # re-promote a release containing an old version of embedded-cluster to test upgrades
+          export APP_VERSION="appver-${{ github.ref_name }}-pre-minio-removal"
+          replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
+
           # re-promote a release containing an old version of embedded-cluster to test upgrades
           export APP_VERSION="appver-${{ github.ref_name }}-1.8.0-k8s-1.28"
           replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
           replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}"
 
+          # promote a release containing the previous stable version of embedded-cluster to test upgrades
+          export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}"
+          export APP_VERSION="appver-${{ github.ref_name }}-previous-stable"
+          export RELEASE_YAML_DIR=e2e/kots-release-install-stable
+          ./scripts/ci-release-app.sh
+
+          # install the previous k0s version to ensure an upgrade occurs
+          # we do not actually run k0s upgrade tests on prerelease at present (as we don't build a previous k0s binary)
+          export EC_VERSION="${{ github.ref_name }}"
+          export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s"
+          export RELEASE_YAML_DIR=e2e/kots-release-install
+          ./scripts/ci-release-app.sh
+
           # install the current k0s version	
           export EC_VERSION="${{ github.ref_name }}"
           export APP_VERSION="appver-${{ github.ref_name }}"
           export RELEASE_YAML_DIR=e2e/kots-release-install	
           ./scripts/ci-release-app.sh
 
+          # then a noop upgrade
+          export EC_VERSION="${{ github.ref_name }}"
+          export APP_VERSION="appver-${{ github.ref_name }}-noop"
+          export RELEASE_YAML_DIR=e2e/kots-release-install
+          ./scripts/ci-release-app.sh
+
           # and finally an app upgrade	
           export EC_VERSION="${{ github.ref_name }}"
           export APP_VERSION="appver-${{ github.ref_name }}-upgrade"
@@ -327,19 +350,48 @@ jobs:
           APP_CHANNEL: Stable
           USES_DEV_BUCKET: "0"
         run: |
-          # install the current k0s version
+          # create a release in the stable channel of our test app for use by CMX
           export EC_VERSION="${{ github.ref_name }}"
           export APP_VERSION="${{ github.ref_name }}"
           export RELEASE_YAML_DIR=e2e/kots-release-install
           ./scripts/ci-release-app.sh
 
+  embedded-cluster-release-builder:
+    name: Build embedded-cluster-release-builder
+    runs-on: ubuntu-latest
+    needs:
+      - get-tag
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: "**/*.sum"
+      - name: Compile embedded-cluster-release-builder
+        run: |
+          make output/bin/embedded-cluster-release-builder
+      - name: Upload embedded-cluster-release-builder artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: embedded-cluster-release-builder
+          path: |
+            output/bin/embedded-cluster-release-builder
+
   download-current:
     name: Download the current release binary
     runs-on: ubuntu-latest
     needs:
       - release-app
       - get-tag
+      - embedded-cluster-release-builder
     steps:
+      - name: Download embedded-cluster-release-builder
+        uses: actions/download-artifact@v4
+        with:
+          name: embedded-cluster-release-builder
+          path: output/bin
       - name: Download current binary
         env:
           LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
@@ -350,13 +402,114 @@ jobs:
           mv embedded-cluster-smoke-test-staging-app embedded-cluster
           mkdir -p output/bin
           mv embedded-cluster output/bin
+
+          # download the embedded-cluster binary from the github release
+          curl -L "https://github.com/replicatedhq/embedded-cluster/releases/download/${{ github.ref_name }}/embedded-cluster-linux-amd64.tgz" -o embedded-cluster-github.tgz
+          tar -xzf embedded-cluster-github.tgz
+          mv embedded-cluster output/bin/embedded-cluster-original
 
       - name: Upload release
         uses: actions/upload-artifact@v4
         with:
           name: current-release
           path: |
             output/bin/embedded-cluster
+            output/bin/embedded-cluster-original
+            output/bin/embedded-cluster-release-builder
+
+  # e2e-docker runs the e2e tests inside a docker container rather than a full VM
+  e2e-docker:
+    name: E2E docker
+    runs-on: ubuntu-latest
+    needs:
+      - release
+      - release-app
+      - get-tag
+      - download-current
+      - find-previous-stable
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - TestPreflights
+          - TestMaterialize
+          - TestHostPreflightCustomSpec
+          - TestHostPreflightInBuiltSpec
+          - TestSingleNodeInstallation
+          - TestSingleNodeInstallationAlmaLinux8
+          - TestSingleNodeInstallationDebian11
+          - TestSingleNodeInstallationDebian12
+          - TestSingleNodeInstallationCentos9Stream
+          - TestSingleNodeUpgradePreviousStable
+          - TestInstallFromReplicatedApp
+          - TestUpgradeFromReplicatedApp
+          - TestInstallWithoutEmbed
+#          - TestUpgradeEC18FromReplicatedApp
+          - TestResetAndReinstall
+          - TestOldVersionUpgrade
+          - TestInstallSnapshotFromReplicatedApp
+          - TestMultiNodeInstallation
+          - TestMultiNodeHAInstallation
+          - TestSingleNodeDisasterRecovery
+          - TestSingleNodeResumeDisasterRecovery
+          - TestMultiNodeHADisasterRecovery
+          - TestSingleNodeInstallationNoopUpgrade
+          - TestCustomCIDR
+          - TestLocalArtifactMirror
+          - TestMultiNodeReset
+          - TestCollectSupportBundle
+          - TestUnsupportedOverrides
+          - TestHostCollectSupportBundleInCluster
+          - TestInstallWithConfigValues
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download binary
+        uses: actions/download-artifact@v4
+        with:
+          name: current-release
+          path: output/bin
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: "**/*.sum"
+      - name: Login to DockerHub to avoid rate limiting
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Free up runner disk space
+        uses: ./.github/actions/free-disk-space
+      - name: Write license files
+        run: |
+          echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml
+          echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}" | base64 --decode > e2e/snapshot-license.yaml
+      - name: Run test
+        env:
+          SHORT_SHA: ${{ github.ref_name }}
+          LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
+          AIRGAP_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
+          SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
+          AIRGAP_SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
+          DR_AWS_S3_ENDPOINT: https://s3.amazonaws.com
+          DR_AWS_S3_REGION: us-east-1
+          DR_AWS_S3_BUCKET: kots-testim-snapshots
+          DR_AWS_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
+          DR_AWS_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
+          DR_AWS_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
+          DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
+          EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }}
+          EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }}
+          EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }}
+          EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION: ${{ github.ref_name }}
+        run: |
+          make e2e-test TEST_NAME=${{ matrix.test }}
+      - name: Troubleshoot
+        if: ${{ !cancelled() }}
+        uses: ./.github/actions/e2e-troubleshoot
+        with:
+          test-name: '${{ matrix.test }}'
 
   e2e:
     name: E2E
@@ -371,8 +524,12 @@ jobs:
       fail-fast: false
       matrix:
         test:
+          - TestVersion
+          - TestCommandsRequireSudo
           - TestResetAndReinstallAirgap
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeConfigValues
+          - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestSingleNodeDisasterRecoveryWithProxy
           - TestProxiedEnvironment
           - TestProxiedCustomCIDR
@@ -381,10 +538,20 @@ jobs:
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
-          - test: TestAirgapUpgradeFromEC18
+          - test: TestMultiNodeAirgapUpgradeSameK0s
             runner: embedded-cluster
           - test: TestMultiNodeAirgapUpgradePreviousStable
             runner: embedded-cluster
+          - test: TestAirgapUpgradeFromEC18
+            runner: embedded-cluster
+          - test: TestSingleNodeAirgapDisasterRecovery
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapHAInstallation
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
+          - test: TestFiveNodesAirgapUpgrade
+            runner: embedded-cluster
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -419,6 +586,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs:
       - e2e
+      - e2e-docker
       - release
       - release-app
     if: always()
@@ -427,5 +595,8 @@ jobs:
       - name: fail if e2e job was not successful
         if: needs.e2e.result != 'success'
         run: exit 1
+      - name: fail if e2e-docker job was not successful
+        if: needs.e2e-docker.result != 'success'
+        run: exit 1
       - name: succeed if everything else passed
         run: echo "Validation succeeded"