From 2e0bfbd1b1395941a6ee04f4a4cc2757d4166581 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 27 Nov 2024 11:07:28 -0500 Subject: [PATCH 1/7] run all vm tests on prerelease --- .github/workflows/release-prod.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index d1dc826af..fa2d7d8ce 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -371,8 +371,12 @@ jobs: fail-fast: false matrix: test: + - TestVersion + - TestCommandsRequireSudo - TestResetAndReinstallAirgap - TestSingleNodeAirgapUpgrade + - TestSingleNodeAirgapUpgradeConfigValues + - TestSingleNodeAirgapUpgradeCustomCIDR - TestSingleNodeDisasterRecoveryWithProxy - TestProxiedEnvironment - TestProxiedCustomCIDR @@ -381,10 +385,20 @@ jobs: include: - test: TestMultiNodeAirgapUpgrade runner: embedded-cluster - - test: TestAirgapUpgradeFromEC18 + - test: TestMultiNodeAirgapUpgradeSameK0s runner: embedded-cluster - test: TestMultiNodeAirgapUpgradePreviousStable runner: embedded-cluster + - test: TestAirgapUpgradeFromEC18 + runner: embedded-cluster + - test: TestSingleNodeAirgapDisasterRecovery + runner: embedded-cluster + - test: TestMultiNodeAirgapHAInstallation + runner: embedded-cluster + - test: TestMultiNodeAirgapHADisasterRecovery + runner: embedded-cluster + - test: TestFiveNodesAirgapUpgrade + runner: embedded-cluster steps: - name: Checkout uses: actions/checkout@v4 From 3a841d4ea11906db2a63d78724d43674a6e21d9d Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 27 Nov 2024 11:36:50 -0500 Subject: [PATCH 2/7] run docker tests on prerelease --- .github/workflows/release-prod.yaml | 122 +++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index fa2d7d8ce..956238f52 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -275,17 +275,40 @@ jobs: APP_CHANNEL: CI USES_DEV_BUCKET: "0" run: | + # re-promote a release containing an old version of embedded-cluster to test upgrades + export APP_VERSION="appver-${{ github.ref_name }}-pre-minio-removal" + replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}" + # re-promote a release containing an old version of embedded-cluster to test upgrades export APP_VERSION="appver-${{ github.ref_name }}-1.8.0-k8s-1.28" replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}" replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}" + # promote a release containing the previous stable version of embedded-cluster to test upgrades + export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" + export APP_VERSION="appver-${{ github.ref_name }}-previous-stable" + export RELEASE_YAML_DIR=e2e/kots-release-install-stable + ./scripts/ci-release-app.sh + + # install the previous k0s version to ensure an upgrade occurs + # we do not actually run k0s upgrade tests on prerelease at present (as we don't build a previous k0s binary) + export EC_VERSION="${{ github.ref_name }}" + export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s" + export RELEASE_YAML_DIR=e2e/kots-release-install + ./scripts/ci-release-app.sh + # install the current k0s version export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="appver-${{ github.ref_name }}" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh + # then a noop upgrade + export EC_VERSION="${{ github.ref_name }}" + export APP_VERSION="appver-${{ github.ref_name }}-noop" + export RELEASE_YAML_DIR=e2e/kots-release-install + ./scripts/ci-release-app.sh + # and finally an app upgrade export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="appver-${{ github.ref_name }}-upgrade" @@ -327,7 +350,7 @@ jobs: APP_CHANNEL: Stable USES_DEV_BUCKET: "0" run: | - # install the current k0s version + # create a release in the stable channel of our test app for use by CMX export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="${{ github.ref_name }}" export RELEASE_YAML_DIR=e2e/kots-release-install @@ -358,6 +381,99 @@ jobs: path: | output/bin/embedded-cluster + # e2e-docker runs the e2e tests inside a docker container rather than a full VM + e2e-docker: + name: E2E docker + runs-on: ubuntu-latest + needs: + - release + - release-app + - get-tag + - download-current + - find-previous-stable + strategy: + fail-fast: false + matrix: + test: + - TestPreflights + - TestMaterialize + - TestHostPreflightCustomSpec + - TestHostPreflightInBuiltSpec + - TestSingleNodeInstallation + - TestSingleNodeInstallationAlmaLinux8 + - TestSingleNodeInstallationDebian11 + - TestSingleNodeInstallationDebian12 + - TestSingleNodeInstallationCentos9Stream + - TestSingleNodeUpgradePreviousStable + - TestInstallFromReplicatedApp + - TestUpgradeFromReplicatedApp + - TestInstallWithoutEmbed + - TestUpgradeEC18FromReplicatedApp + - TestResetAndReinstall + - TestOldVersionUpgrade + - TestInstallSnapshotFromReplicatedApp + - TestMultiNodeInstallation + - TestMultiNodeHAInstallation + - TestSingleNodeDisasterRecovery + - TestSingleNodeResumeDisasterRecovery + - TestMultiNodeHADisasterRecovery + - TestSingleNodeInstallationNoopUpgrade + - TestCustomCIDR + - TestLocalArtifactMirror + - TestMultiNodeReset + - TestCollectSupportBundle + - TestUnsupportedOverrides + - TestHostCollectSupportBundleInCluster + - TestInstallWithConfigValues + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download binary + uses: actions/download-artifact@v4 + with: + name: current-release + path: output/bin + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + - name: Login to DockerHub to avoid rate limiting + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Free up runner disk space + uses: ./.github/actions/free-disk-space + - name: Write license files + run: | + echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml + echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}" | base64 --decode > e2e/snapshot-license.yaml + - name: Run test + env: + SHORT_SHA: ${{ github.ref_name }} + LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }} + AIRGAP_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }} + SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }} + AIRGAP_SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }} + DR_AWS_S3_ENDPOINT: https://s3.amazonaws.com + DR_AWS_S3_REGION: us-east-1 + DR_AWS_S3_BUCKET: kots-testim-snapshots + DR_AWS_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + DR_AWS_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap + DR_AWS_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} + EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }} + EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }} + run: | + make e2e-test TEST_NAME=${{ matrix.test }} + - name: Troubleshoot + if: ${{ !cancelled() }} + uses: ./.github/actions/e2e-troubleshoot + with: + test-name: '${{ matrix.test }}' + e2e: name: E2E runs-on: ${{ matrix.runner || 'ubuntu-22.04' }} @@ -433,6 +549,7 @@ jobs: runs-on: ubuntu-20.04 needs: - e2e + - e2e-docker - release - release-app if: always() @@ -441,5 +558,8 @@ jobs: - name: fail if e2e job was not successful if: needs.e2e.result != 'success' run: exit 1 + - name: fail if e2e-docker job was not successful + if: needs.e2e-docker.result != 'success' + run: exit 1 - name: succeed if everything else passed run: echo "Validation succeeded" From d79bc3bb22b3e676f6ac05aaaba626f0921e2db8 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Dec 2024 15:17:33 -0500 Subject: [PATCH 3/7] handle unmodified-binary tests and in-docker upgrade tests in postrelease testing --- .github/workflows/release-prod.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index 956238f52..c8b5bec51 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -373,6 +373,11 @@ jobs: mv embedded-cluster-smoke-test-staging-app embedded-cluster mkdir -p output/bin mv embedded-cluster output/bin + + # download the embedded-cluster binary from the github release + curl -L "https://github.com/replicatedhq/embedded-cluster/releases/download/${{ github.ref_name }}/embedded-cluster-linux-amd64.tgz" -o embedded-cluster-github.tgz + tar -xzf embedded-cluster-github.tgz + mv embedded-cluster output/bin/embedded-cluster-original - name: Upload release uses: actions/upload-artifact@v4 @@ -466,6 +471,7 @@ jobs: EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }} EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }} EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }} + EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION: ${{ github.ref_name }} run: | make e2e-test TEST_NAME=${{ matrix.test }} - name: Troubleshoot From 9ab1ad10eb717295e75dde251c8d63d3fb9e8abe Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Dec 2024 15:57:23 -0500 Subject: [PATCH 4/7] actually upload the original binary --- .github/workflows/release-prod.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index c8b5bec51..2c672b2db 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -385,6 +385,7 @@ jobs: name: current-release path: | output/bin/embedded-cluster + output/bin/embedded-cluster-original # e2e-docker runs the e2e tests inside a docker container rather than a full VM e2e-docker: From 3d9d645009a04f89ff1d0e1cf417e4e711fd358b Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Dec 2024 17:04:59 -0500 Subject: [PATCH 5/7] include embedded-cluster-release-builder binary --- .github/workflows/release-prod.yaml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index 2c672b2db..2248d28a1 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -356,13 +356,42 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh + embedded-cluster-release-builder: + name: Build embedded-cluster-release-builder + runs-on: ubuntu-latest + needs: + - get-tag + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + - name: Compile embedded-cluster-release-builder + run: | + make embedded-cluster-release-builder + - name: Upload embedded-cluster-release-builder artifact + uses: actions/upload-artifact@v4 + with: + name: embedded-cluster-release-builder + path: | + output/bin/embedded-cluster-release-builder + download-current: name: Download the current release binary runs-on: ubuntu-latest needs: - release-app - get-tag + - embedded-cluster-release-builder steps: + - name: Download embedded-cluster-release-builder + uses: actions/download-artifact@v4 + with: + name: embedded-cluster-release-builder + path: output/bin - name: Download current binary env: LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }} @@ -386,6 +415,7 @@ jobs: path: | output/bin/embedded-cluster output/bin/embedded-cluster-original + output/bin/embedded-cluster-release-builder # e2e-docker runs the e2e tests inside a docker container rather than a full VM e2e-docker: From 7ce3f6b51b06981da793b6c90cef3fc3602c08ae Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Dec 2024 17:24:02 -0500 Subject: [PATCH 6/7] f --- .github/workflows/release-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index 2248d28a1..782ab6cc2 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -371,7 +371,7 @@ jobs: cache-dependency-path: "**/*.sum" - name: Compile embedded-cluster-release-builder run: | - make embedded-cluster-release-builder + make output/bin/embedded-cluster-release-builder - name: Upload embedded-cluster-release-builder artifact uses: actions/upload-artifact@v4 with: From 8a38a9863e82fe3f927bb2d0736d30818d13f7c0 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Dec 2024 18:21:01 -0500 Subject: [PATCH 7/7] disable TestUpgradeEC18FromReplicatedApp --- .github/workflows/release-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index 782ab6cc2..c2fdb6267 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -444,7 +444,7 @@ jobs: - TestInstallFromReplicatedApp - TestUpgradeFromReplicatedApp - TestInstallWithoutEmbed - - TestUpgradeEC18FromReplicatedApp +# - TestUpgradeEC18FromReplicatedApp - TestResetAndReinstall - TestOldVersionUpgrade - TestInstallSnapshotFromReplicatedApp