diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d852520d8e1..c50a4274cf7 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -45,7 +45,7 @@ jobs: echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT - name: Enforce CTF Version if: steps.condition-check.outputs.should-enforce == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: go-project-path: ./integration-tests module-name: github.com/smartcontractkit/chainlink-testing-framework @@ -87,7 +87,7 @@ jobs: - name: Checkout the repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Setup Go - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: test_download_vendor_packages_command: cd ./integration-tests && go mod download go_mod_path: ./integration-tests/go.mod @@ -275,7 +275,7 @@ jobs: ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} @@ -295,6 +295,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 eth-smoke-tests-matrix: if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }} @@ -442,7 +445,7 @@ jobs: ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} @@ -473,7 +476,7 @@ jobs: ## Run this step when changes that do not need the test to run are made - name: Run Setup if: needs.changes.outputs.src == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: test_download_vendor_packages_command: cd ./integration-tests && go mod download go_mod_path: ./integration-tests/go.mod @@ -498,25 +501,7 @@ jobs: path: ./integration-tests/smoke/traces/trace-data.json - name: Print failed test summary if: always() - run: | - directory="./integration-tests/smoke/.test_summary" - files=("$directory"/*) - if [ -d "$directory" ]; then - echo "Test summary folder found" - if [ ${#files[@]} -gt 0 ]; then - first_file="${files[0]}" - echo "Name of the first test summary file: $(basename "$first_file")" - echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY - cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY - if [ ${#files[@]} -gt 1 ]; then - echo "Found more than one test summary file. This is incorrect, there should be only one file" - fi - else - echo "Test summary directory is empty. This should not happen" - fi - else - echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting" - fi + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 ### Used to check the required checks box when the matrix completes eth-smoke-tests: @@ -582,7 +567,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Run Setup - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: test_download_vendor_packages_command: | cd ./integration-tests @@ -630,7 +615,7 @@ jobs: run: | echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ github.sha }}'" - name: Run Migration Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -742,7 +727,7 @@ jobs: steps: - name: Check if image exists id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: repository: chainlink-solana-tests tag: ${{ needs.get_solana_sha.outputs.sha }} @@ -880,7 +865,7 @@ jobs: ref: ${{ needs.get_solana_sha.outputs.sha }} - name: Run Setup if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: go_mod_path: ./integration-tests/go.mod cache_restore_only: true @@ -909,7 +894,7 @@ jobs: docker rm "$CONTAINER_ID" - name: Run Tests if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke cl_repo: ${{ env.CHAINLINK_IMAGE }} diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index c970751779d..a4c0c244f03 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -123,7 +123,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Build Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 with: test_download_vendor_packages_command: cd ./integration-tests && go mod download token: ${{ secrets.GITHUB_TOKEN }} @@ -258,7 +258,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-sepolia @@ -278,6 +278,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 bsc-testnet-tests: # TODO: BSC RPCs are all in a bad state right now, so we're skipping these tests until they're fixed @@ -310,7 +313,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-bsc-testnet @@ -330,6 +333,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 optimism-sepolia-smoke-tests: environment: integration @@ -360,7 +366,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-optimism-sepolia @@ -380,6 +386,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 arbitrum-sepolia-smoke-tests: environment: integration @@ -410,7 +419,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-arbitrum-sepolia @@ -430,6 +439,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 base-sepolia-smoke-tests: environment: integration @@ -456,7 +468,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-base-sepolia @@ -476,6 +488,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 polygon-mumbai-smoke-tests: environment: integration @@ -506,7 +521,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-polygon-mumbai @@ -526,6 +541,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 avalanche-fuji-smoke-tests: environment: integration @@ -556,7 +574,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-avalanche-fuji @@ -576,6 +594,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 fantom-testnet-smoke-tests: environment: integration @@ -606,7 +627,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-fantom-testnet @@ -626,6 +647,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 celo-alfajores-smoke-tests: environment: integration @@ -652,7 +676,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-celo-alfajores @@ -672,6 +696,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 scroll-sepolia-smoke-tests: # TODO: Disabled until bug TT-767 is fixed @@ -700,7 +727,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-scroll-sepolia @@ -720,6 +747,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 linea-goerli-smoke-tests: environment: integration @@ -746,7 +776,7 @@ jobs: with: name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 env: PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-linea-goerli @@ -765,4 +795,7 @@ jobs: cache_restore_only: "true" QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} \ No newline at end of file + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@952abab6367aaff96102abf9ef751ef36fe9ea29 # v2.3.3 \ No newline at end of file