From a2fc48d148d68132fb38ce1f7f137d40597d091f Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 25 Jan 2024 16:21:02 -0500 Subject: [PATCH] Break OCR2 Tests into Matrix --- .github/workflows/integration-tests.yml | 99 +++++++++++++++++-- .../smoke/ocr2_test.go_test_list.json | 17 ++++ 2 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 integration-tests/smoke/ocr2_test.go_test_list.json diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e8e9c8904d0..c3f0cf813d0 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -191,6 +191,7 @@ jobs: outputs: automation-matrix: ${{ env.AUTOMATION_JOB_MATRIX_JSON }} lp-matrix: ${{ env.LP_JOB_MATRIX_JSON }} + ocr2-matrix: ${{ env.OCR2_JOB_MATRIX_JSON }} steps: - name: Check for Skip Tests Label if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') @@ -205,6 +206,7 @@ jobs: ./scripts/compareTestList.sh ./smoke/automation_test.go ./scripts/compareTestList.sh ./smoke/keeper_test.go ./scripts/compareTestList.sh ./smoke/log_poller_test.go + ./scripts/compartTestList.sh ./smoke/ocr2_test.go - name: Build Test Matrix Lists id: build-test-matrix-list run: | @@ -216,6 +218,9 @@ jobs: LOG_POLLER_MATRIX_JSON=$(./scripts/buildTestMatrixList.sh ./smoke/log_poller_test.go log_poller ubuntu-latest 1) echo "LP_JOB_MATRIX_JSON=${LOG_POLLER_MATRIX_JSON}" >> $GITHUB_ENV + OCR2_MATRIX_JSON=$(./scripts/buildTestMatrixList.sh ./smoke/ocr2_test.go ocr2 ubuntu-latest 1) + echo "OCR2_JOB_MATRIX_JSON=${OCR2_MATRIX_JSON}" >> $GITHUB_ENV + # if we running a PR against the develop branch we should only run the automation tests unless we are in the merge group event if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then echo "We are in a merge_group event, run both automation and keepers tests" @@ -400,6 +405,90 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" + eth-smoke-tests-matrix-ocr2: + if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }} + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + needs: + [build-chainlink, changes, compare-tests, build-lint-integration-tests] + env: + SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 + CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_LOG_LEVEL: debug + strategy: + fail-fast: false + matrix: + product: ${{fromJson(needs.compare-tests.outputs.ocr2-matrix)}} + tag_suffix: ["", "-plugins"] + runs-on: ${{ matrix.product.os }} + name: ETH Smoke Tests ${{ matrix.product.name }} + steps: + - name: Collect Metrics + if: needs.changes.outputs.src == 'true' + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: ETH Smoke Tests ${{ matrix.product.name }} + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Go Test Command + id: build-go-test-command + run: | + # if the matrix.product.run is set, use it for a different command + if [ "${{ matrix.product.run }}" != "" ]; then + echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" + else + echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" + fi + - name: Prepare Base64 TOML override + uses: ./.github/actions/setup-create-base64-config + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + selectedNetworks: ${{ env.SELECTED_NETWORKS }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ github.sha }} + pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} + pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + ## Run this step when changes that require tests to be run are made + - name: Run Tests + if: needs.changes.outputs.src == 'true' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + with: + test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_location: ./integration-tests/smoke/logs/ + publish_check_name: ${{ matrix.product.name }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: "" + + eth-smoke-tests-matrix: if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} environment: integration @@ -435,16 +524,6 @@ jobs: os: ubuntu-latest file: ocr pyroscope_env: ci-smoke-ocr-evm-simulated - - name: ocr2 - nodes: 6 - os: ubuntu-latest - file: ocr2 - pyroscope_env: ci-smoke-ocr2-evm-simulated - - name: ocr2 - nodes: 6 - os: ubuntu-latest - pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated - tag_suffix: "-plugins" - name: vrf nodes: 2 os: ubuntu-latest diff --git a/integration-tests/smoke/ocr2_test.go_test_list.json b/integration-tests/smoke/ocr2_test.go_test_list.json new file mode 100644 index 00000000000..facb876b959 --- /dev/null +++ b/integration-tests/smoke/ocr2_test.go_test_list.json @@ -0,0 +1,17 @@ +{ + "tests": [ + { + "name": "TestOCRv2Basic", + "label": "ubuntu-latest", + "nodes": 4 + }, + { + "name": "TestOCRv2Request", + "label": "ubuntu-latest" + }, + { + "name": "TestOCRv2JobReplacement", + "label": "ubuntu-latest" + } + ] +} \ No newline at end of file