Break OCR2 Tests into Matrix

smartcontractkit · Jan 25, 2024 · a2fc48d · a2fc48d
1 parent e214014
commit a2fc48d
Show file tree

Hide file tree

Showing 2 changed files with 106 additions and 10 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -191,6 +191,7 @@ jobs:
     outputs:
       automation-matrix: ${{ env.AUTOMATION_JOB_MATRIX_JSON }}
       lp-matrix: ${{ env.LP_JOB_MATRIX_JSON }}
+      ocr2-matrix: ${{ env.OCR2_JOB_MATRIX_JSON }}
     steps:
       - name: Check for Skip Tests Label
         if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests')
@@ -205,6 +206,7 @@ jobs:
           ./scripts/compareTestList.sh ./smoke/automation_test.go
           ./scripts/compareTestList.sh ./smoke/keeper_test.go
           ./scripts/compareTestList.sh ./smoke/log_poller_test.go
+          ./scripts/compartTestList.sh ./smoke/ocr2_test.go
       - name: Build Test Matrix Lists
         id: build-test-matrix-list
         run: |
@@ -216,6 +218,9 @@ jobs:
           LOG_POLLER_MATRIX_JSON=$(./scripts/buildTestMatrixList.sh ./smoke/log_poller_test.go log_poller ubuntu-latest 1)
           echo "LP_JOB_MATRIX_JSON=${LOG_POLLER_MATRIX_JSON}" >> $GITHUB_ENV
 
+          OCR2_MATRIX_JSON=$(./scripts/buildTestMatrixList.sh ./smoke/ocr2_test.go ocr2 ubuntu-latest 1)
+          echo "OCR2_JOB_MATRIX_JSON=${OCR2_MATRIX_JSON}" >> $GITHUB_ENV
+
           # if we running a PR against the develop branch we should only run the automation tests unless we are in the merge group event
           if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
             echo "We are in a merge_group event, run both automation and keepers tests"
@@ -400,6 +405,90 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
 
+  eth-smoke-tests-matrix-ocr2:
+    if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }}
+    environment: integration
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    needs:
+      [build-chainlink, changes, compare-tests, build-lint-integration-tests]
+    env:
+      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
+      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+    strategy:
+      fail-fast: false
+      matrix:
+        product: ${{fromJson(needs.compare-tests.outputs.ocr2-matrix)}}
+        tag_suffix: ["", "-plugins"]
+    runs-on: ${{ matrix.product.os }}
+    name: ETH Smoke Tests ${{ matrix.product.name }}
+    steps:
+      - name: Collect Metrics
+        if: needs.changes.outputs.src == 'true'
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Build Go Test Command
+        id: build-go-test-command
+        run: |
+          # if the matrix.product.run is set, use it for a different command
+          if [ "${{ matrix.product.run }}" != "" ]; then
+            echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Prepare Base64 TOML override
+        uses: ./.github/actions/setup-create-base64-config
+        with:
+          runId: ${{ github.run_id }}
+          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
+          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
+          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
+          chainlinkVersion: ${{ github.sha }}
+          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
+          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
+          lokiEndpoint: ${{ secrets.LOKI_URL }}
+          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
+          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafanaUrl: ${{ vars.GRAFANA_URL }}
+          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+      ## Run this step when changes that require tests to be run are made
+      - name: Run Tests
+        if: needs.changes.outputs.src == 'true'
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
+        with:
+          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          artifacts_location: ./integration-tests/smoke/logs/
+          publish_check_name: ${{ matrix.product.name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ""
+
+
   eth-smoke-tests-matrix:
     if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
     environment: integration
@@ -435,16 +524,6 @@ jobs:
             os: ubuntu-latest
             file: ocr
             pyroscope_env: ci-smoke-ocr-evm-simulated
-          - name: ocr2
-            nodes: 6
-            os: ubuntu-latest
-            file: ocr2
-            pyroscope_env: ci-smoke-ocr2-evm-simulated
-          - name: ocr2
-            nodes: 6
-            os: ubuntu-latest
-            pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated
-            tag_suffix: "-plugins"
           - name: vrf
             nodes: 2
             os: ubuntu-latest

diff --git a/integration-tests/smoke/ocr2_test.go_test_list.json b/integration-tests/smoke/ocr2_test.go_test_list.json
@@ -0,0 +1,17 @@
+{
+  "tests": [
+    {
+      "name": "TestOCRv2Basic",
+      "label": "ubuntu-latest",
+      "nodes": 4
+    },
+    {
+      "name": "TestOCRv2Request",
+      "label": "ubuntu-latest"
+    },
+    {
+      "name": "TestOCRv2JobReplacement",
+      "label": "ubuntu-latest"
+    }
+  ]
+}