[TT-524] Separate Live Testnet Tests (#11196)

* Separate Live Testnet Tests

* \n

* Remove typo

* Extrapolates building Chainlink image

.github/actions/build-chainlink-image/action.yml

@@ -0,0 +1,48 @@
+name: Build Chainlink Image
+description: A composite action that allows building and publishing the Chainlink image for integration testing
+
+inputs:
+  tag_suffix:
+    description: The suffix to append to the image tag (usually blank or "-plugins")
+    default: ""
+  dockerfile:
+    description: The path to the Dockerfile to use (usually core/chainlink.Dockerfile or plugins/chainlink.Dockerfile)
+    default: core/chainlink.Dockerfile
+  git_commit_sha:
+    description: The git commit sha to use for the image tag
+    default: ${{ github.sha }}
+  GRAFANA_CLOUD_BASIC_AUTH:
+    description: "grafana cloud basic auth"
+  GRAFANA_CLOUD_HOST:
+    description: "grafana cloud hostname"
+  AWS_REGION:
+    description: "AWS region to use for ECR"
+  AWS_ROLE_TO_ASSUME:
+    description: "AWS role to assume for ECR"
+
+runs:
+  using: composite
+  steps:
+    - name: Check if image exists
+      id: check-image
+      uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
+      with:
+        repository: chainlink
+        tag: ${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
+        AWS_REGION: ${{ inputs.AWS_REGION }}
+        AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
+    - name: Build Image
+      if: steps.check-image.outputs.exists == 'false' && needs.changes.outputs.src == 'true'
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
+      with:
+        cl_repo: smartcontractkit/chainlink
+        cl_ref: ${{ inputs.git_commit_sha }}
+        cl_dockerfile: ${{ inputs.dockerfile }}
+        push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
+        QA_AWS_REGION: ${{ inputs.AWS_REGION }}
+        QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
+    - name: Print Chainlink Image Built
+      shell: sh
+      run: |
+        echo "### Chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
+        echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY

.github/workflows/integration-tests.yml

@@ -2,23 +2,10 @@ name: Integration Tests
 on:
   merge_group:
   pull_request:
-  schedule:
-    - cron: "0 0 * * *"
-    # - cron: "0 * * * *" # DEBUG: Run every hour to nail down flakes
   push:
     tags:
       - "*"
   workflow_dispatch:
-    inputs:
-      simulatedNetwork:
-        description: "Run Simulated Network Tests"
-        required: false
-        type: boolean
-        default: true
-      liveNetwork:
-        description: "Run Live Network Tests"
-        required: false
-        type: boolean
 
 # Only run 1 of this workflow at a time per PR
 concurrency:
@@ -74,9 +61,9 @@ jobs:
           hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
           this-job-name: Check Paths That Require Tests To Run
         continue-on-error: true
-
     outputs:
       src: ${{ steps.changes.outputs.src }}
+
   build-chainlink:
     environment: integration
     permissions:
@@ -108,30 +95,16 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Check if image exists
-        if: needs.changes.outputs.src == 'true'
-        id: check-image
-        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
-        with:
-          repository: chainlink
-          tag: ${{ github.sha }}${{ matrix.image.tag-suffix }}
+      - name: Build Chainlink Image
+        uses: ./.github/actions/build-chainlink-image
+        with: 
+          tag_suffix: ${{ matrix.image.tag-suffix }}
+          dockerfile: ${{ matrix.image.dockerfile }}
+          git_commit_sha: ${{ github.sha }}
+          GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
           AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      - name: Build Image
-        if: steps.check-image.outputs.exists == 'false' && needs.changes.outputs.src == 'true'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
-        with:
-          cl_repo: smartcontractkit/chainlink
-          cl_ref: ${{ github.sha }}
-          cl_dockerfile: ${{ matrix.image.dockerfile }}
-          push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      - name: Print Chainlink Image Built
-        if: needs.changes.outputs.src == 'true'
-        run: |
-          echo "### Chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
 
   build-test-image:
     if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
@@ -908,201 +881,3 @@ jobs:
           matrix-aggregator-status: ${{ needs.solana-smoke-tests-matrix.result }}
         continue-on-error: true
   ### End Solana Section
-
-  ### Start Live Testnet Section
-
-  testnet-smoke-tests-matrix:
-    if: ${{ github.event_name == 'schedule' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || (github.event_name == 'workflow_dispatch' && inputs.liveNetwork) }} ## Only run live tests on new tags, schedule, or on manual request
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink]
-    env:
-      SELECTED_NETWORKS: ${{ matrix.testnet }}
-      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
-      EVM_KEYS: ${{ secrets.QA_EVM_KEYS }}
-
-      OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }}
-      OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }}
-
-      ARBITRUM_GOERLI_URLS: ${{ secrets.QA_ARBITRUM_GOERLI_URLS }}
-      ARBITRUM_GOERLI_HTTP_URLS: ${{ secrets.QA_ARBITRUM_GOERLI_HTTP_URLS }}
-    strategy:
-      fail-fast: false
-      matrix:
-        # NOTE: If changing this matrix, make sure to update the matrix in the testnet-smoke-tests-notify job to be the same
-        # otherwise reporting will be broken. Getting a single matrix for multiple jobs is a pain
-        # https://github.com/orgs/community/discussions/26284#discussioncomment-3251198
-        testnet: [OPTIMISM_GOERLI, ARBITRUM_GOERLI]
-    name: Live Testnet Smoke Tests ${{ matrix.testnet }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      ## Only run OCR smoke test for now
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
-        env:
-          PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          PYROSCOPE_ENVIRONMENT: ci-smoke-ocr-evm-${{ matrix.testnet }} # TODO: Only for OCR for now
-          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-        with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/ocr_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./integration-tests/smoke/logs
-          publish_check_name: ${{ matrix.testnet }} OCR Smoke Test Results
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
-        with:
-          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
-          this-job-name: Live Testnet Smoke Tests ${{ matrix.testnet }}
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
-
-  testnet-smoke-tests-notify:
-    name: Live Testnet Start Slack Thread
-    if: ${{ always() && needs.testnet-smoke-tests-matrix.result != 'skipped' && needs.testnet-smoke-tests-matrix.result != 'cancelled' && github.event_name != 'workflow_dispatch' }}
-    environment: integration
-    outputs:
-      thread_ts: ${{ steps.slack.outputs.thread_ts }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: testnet-smoke-tests-matrix
-    steps:
-      - name: Debug Result
-        run: echo ${{needs.testnet-smoke-tests-matrix.result}}
-      - name: Main Slack Notification
-        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
-        id: slack
-        with:
-          channel-id: ${{ secrets.QA_SLACK_CHANNEL }}
-          payload: |
-            {
-              "attachments": [
-                {
-                  "color": "${{ needs.testnet-smoke-tests-matrix.result == 'success' && '#2E7D32' || '#C62828' }}",
-                  "blocks": [
-                    {
-                      "type": "header",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "Live Smoke Test Results ${{ needs.testnet-smoke-tests-matrix.result == 'success' && ':white_check_mark:' || ':x:'}}",
-                        "emoji": true
-                      }
-                    },
-                    {
-                      "type": "divider"
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
-                      }
-                    }
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-
-  testnet-smoke-tests-results:
-    name: Post Live Testnet Smoke Test Results
-    if: ${{ always() && needs.testnet-smoke-tests-matrix.result != 'skipped' && needs.testnet-smoke-tests-matrix.result != 'cancelled' }}
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: testnet-smoke-tests-notify
-    strategy:
-      fail-fast: false
-      matrix:
-        # NOTE: If changing this matrix, make sure to update the matrix in the testnet-smoke-tests-matrix job to be the same
-        # otherwise reporting will be broken. Getting a single matrix for multiple jobs is a pain
-        # https://github.com/orgs/community/discussions/26284#discussioncomment-3251198
-        testnet: [OPTIMISM_GOERLI, ARBITRUM_GOERLI]
-    steps:
-      - name: Get Results
-        id: test-results
-        run: |
-          echo "Querying test results"
-
-          echo "status=$(curl \
-          -H "Authorization: Bearer ${{ github.token }}" \
-          'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
-          | jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet}}").steps[] | select(.name == "Run Tests").conclusion')" >> $GITHUB_OUTPUT
-
-          echo "status=$(curl \
-          -H "Authorization: Bearer ${{ github.token }}" \
-          'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
-          | jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet}}").steps[] | select(.name == "Run Tests").conclusion')"
-          echo "thread_ts=${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}"
-
-      - name: Test Details
-        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
-        with:
-          channel-id: ${{ secrets.QA_SLACK_CHANNEL }}
-          payload: |
-            {
-              "thread_ts": "${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}",
-              "attachments": [
-                {
-                  "color": "${{ steps.test-results.outputs.status == 'success' && '#2E7D32' || '#C62828' }}",
-                  "blocks": [
-                    {
-                      "type": "header",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "${{ matrix.testnet }} Smoke Test Results ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}",
-                        "emoji": true
-                      }
-                    },
-                    {
-                      "type": "divider"
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "OCR ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}"
-                      }
-                    }
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-
-  ### End Live Testnet Section