integration-tests.yml

name: Integration Tests
on:
  merge_group:
  pull_request:
  push:
    tags:
      - "*"
  workflow_dispatch:

# Only run 1 of this workflow at a time per PR
concurrency:
  group: integration-tests-chainlink-${{ github.ref }}
  cancel-in-progress: true

env:
  # for run-test variables and environment
  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  TEST_SUITE: smoke
  TEST_ARGS: -test.timeout 12m
  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
  MOD_CACHE_VERSION: 2

jobs:
  enforce-ctf-version:
    name: Enforce CTF Version
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Check Merge Group Condition
        id: condition-check
        run: |
          echo "Checking event condition..."
          SHOULD_ENFORCE="false"
          if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
              echo "We are in a merge_group event, now check if we are on the develop branch"
              target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .merge_group.base_ref)
              if [[ "$target_branch" == "refs/heads/develop" ]]; then
                  echo "We are on the develop branch, we should enforce ctf version"
                  SHOULD_ENFORCE="true"
              fi
          fi
          echo "should we enforce ctf version = $SHOULD_ENFORCE"
          echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT
      - name: Enforce CTF Version
        if: steps.condition-check.outputs.should-enforce == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          go-project-path: ./integration-tests
          module-name: github.com/smartcontractkit/chainlink-testing-framework
          enforce-semantic-tag: "true"
  changes:
    environment: integration
    name: Check Paths That Require Tests To Run
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
        id: changes
        with:
          filters: |
            src:
              - '**/*.go'
              - '**/*go.sum'
              - '**/*go.mod'
              - '.github/workflows/integration-tests.yml'
              - '**/*Dockerfile'
              - 'core/**/config/**/*.toml'
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Check Paths That Require Tests To Run
        continue-on-error: true
    outputs:
      src: ${{ steps.changes.outputs.src }}

  build-lint-integration-tests:
    name: Build and Lint integration-tests
    runs-on: ubuntu20.04-16cores-64GB
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup Go
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
      - name: Build Go
        run: |
          cd ./integration-tests
          go build ./...
          SELECTED_NETWORKS=SIMULATED go test -run=^# ./...
      - name: Lint Go
        uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
        with:
          version: v1.55.2
          # We already cache these directories in setup-go
          skip-pkg-cache: true
          skip-build-cache: true
          # only-new-issues is only applicable to PRs, otherwise it is always set to false
          only-new-issues: false # disabled for PRs due to unreliability
          args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml
          working-directory: ./integration-tests

  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        image:
          - name: ""
            dockerfile: core/chainlink.Dockerfile
            tag-suffix: ""
          - name: (plugins)
            dockerfile: plugins/chainlink.Dockerfile
            tag-suffix: -plugins
    name: Build Chainlink Image ${{ matrix.image.name }}
    runs-on: ubuntu20.04-16cores-64GB
    needs: [changes, enforce-ctf-version]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Build Chainlink Image ${{ matrix.image.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Chainlink Image
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/build-chainlink-image
        with:
          tag_suffix: ${{ matrix.image.tag-suffix }}
          dockerfile: ${{ matrix.image.dockerfile }}
          git_commit_sha: ${{ github.sha }}
          GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  build-test-image:
    if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Test Image
    runs-on: ubuntu20.04-16cores-64GB
    needs: [changes]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Test Image
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/build-test-image
        with:
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

  compare-tests:
    needs: [changes]
    runs-on: ubuntu-latest
    name: Compare/Build Automation Test List
    outputs:
      matrix: ${{ env.MATRIX_JSON }}
    steps:
      - name: Check for Skip Tests Label
        if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests')
        run: |
          echo "## \`skip-smoke-tests\` label is active, skipping E2E smoke tests" >>$GITHUB_STEP_SUMMARY
          exit 0
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Compare Test Lists
        run: |
          cd ./integration-tests
          ./scripts/compareTestList.sh ./smoke/automation_test.go
          ./scripts/compareTestList.sh ./smoke/keeper_test.go
      - name: Build Test Matrix Lists
        id: build-test-matrix-list
        run: |
          cd ./integration-tests
          MATRIX_JSON_AUTOMATION=$(./scripts/buildTestMatrixList.sh ./smoke/automation_test.go automation ubuntu-latest 1)
          MATRIX_JSON_KEEPER=$(./scripts/buildTestMatrixList.sh ./smoke/keeper_test.go keeper ubuntu-latest 1)
          COMBINED_ARRAY=$(jq -c -n "$MATRIX_JSON_AUTOMATION + $MATRIX_JSON_KEEPER")

          # if we running a PR against the develop branch we should only run the automation tests unless we are in the merge group event
          if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
            echo "We are in a merge_group event, run both automation and keepers tests"
            echo "MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV
          else
            echo "we are not in a merge_group event, if this is a PR to develop run only automation tests, otherwise run everything because we could be running against a release branch"
            target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.base.ref)
            if [[ "$target_branch" == "develop" ]]; then
              echo "only run automation tests"
              echo "MATRIX_JSON=${MATRIX_JSON_AUTOMATION}" >> $GITHUB_ENV
            else
              echo "run both automation and keepers tests"
              echo "MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV
            fi
          fi

  # eth-smoke-tests-matrix-automation:
  #   if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
  #   environment: integration
  #   permissions:
  #     checks: write
  #     pull-requests: write
  #     id-token: write
  #     contents: read
  #   needs:
  #     [build-chainlink, changes, compare-tests, build-lint-integration-tests]
  #   env:
  #     SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
  #     CHAINLINK_COMMIT_SHA: ${{ github.sha }}
  #     CHAINLINK_ENV_USER: ${{ github.actor }}
  #     TEST_LOG_LEVEL: debug
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       product: ${{fromJson(needs.compare-tests.outputs.matrix)}}
  #   runs-on: ${{ matrix.product.os }}
  #   name: ETH Smoke Tests ${{ matrix.product.name }}
  #   steps:
  #     - name: Collect Metrics
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       id: collect-gha-metrics
  #       uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
  #       with:
  #         basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
  #         hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
  #         this-job-name: ETH Smoke Tests ${{ matrix.product.name }}
  #         test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
  #       continue-on-error: true
  #     - name: Checkout the repo
  #       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  #       with:
  #         ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
  #     - name: Build Go Test Command
  #       id: build-go-test-command
  #       run: |
  #         # if the matrix.product.run is set, use it for a different command
  #         if [ "${{ matrix.product.run }}" != "" ]; then
  #           echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
  #         else
  #           echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
  #         fi
  #     ## Run this step when changes that require tests to be run are made
  #     - name: Run Tests
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
  #       env:
  #         PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
  #         PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
  #         PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
  #       with:
  #         test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
  #         test_download_vendor_packages_command: cd ./integration-tests && go mod download
  #         cl_repo: ${{ env.CHAINLINK_IMAGE }}
  #         cl_image_tag: ${{ github.sha }}
  #         aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
  #         artifacts_location: ./integration-tests/smoke/logs/
  #         publish_check_name: ${{ matrix.product.name }}
  #         token: ${{ secrets.GITHUB_TOKEN }}
  #         go_mod_path: ./integration-tests/go.mod
  #         cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
  #         cache_restore_only: "true"
  #         QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
  #         QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
  #         QA_KUBECONFIG: ""

  eth-smoke-tests-matrix:
    if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes, build-lint-integration-tests]
    env:
      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
    strategy:
      fail-fast: false
      matrix:
        product:
          # - name: cron
          #   nodes: 2
          #   os: ubuntu-latest
          #   pyroscope_env: ""
          # - name: flux
          #   nodes: 1
          #   os: ubuntu-latest
          #   pyroscope_env: ""
          - name: ocr
            nodes: 2
            os: ubuntu-latest
            file: ocr
            pyroscope_env: ci-smoke-ocr-evm-simulated
          # - name: ocr2
          #   nodes: 4
          #   os: ubuntu-latest
          #   file: ocr2
          #   pyroscope_env: ci-smoke-ocr2-evm-simulated
          # - name: ocr2
          #   nodes: 4
          #   os: ubuntu-latest
          #   pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated
          #   tag_suffix: "-plugins"
          # - name: runlog
          #   nodes: 1
          #   os: ubuntu-latest
          #   pyroscope_env: ""
          # - name: vrf
          #   nodes: 2
          #   os: ubuntu-latest
          #   pyroscope_env: ci-smoke-vrf-evm-simulated
          # - name: vrfv2
          #   nodes: 2
          #   os: ubuntu-latest
          #   pyroscope_env: ci-smoke-vrf2-evm-simulated
          # - name: vrfv2plus
          #   nodes: 3
          #   os: ubuntu-latest
          #   pyroscope_env: ci-smoke-vrf2plus-evm-simulated
          # - name: forwarder_ocr
          #   nodes: 1
          #   os: ubuntu-latest
          #   pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
          # - name: forwarders_ocr2
          #   nodes: 1
          #   os: ubuntu-latest
          #   pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
    runs-on: ${{ matrix.product.os }}
    name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Go Test Command
        id: build-go-test-command
        run: |
          # if the matrix.product.run is set, use it for a different command
          if [ "${{ matrix.product.run }}" != "" ]; then
            echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
          else
            echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
          fi
      - name: Check for "enable tracing" label
        id: check-label
        run: |
          label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH")

          if [[ -n "$label" ]]; then
            if [[ "$label" == "enable tracing" ]]; then
              echo "Enable tracing label found."
              echo "trace=true" >> $GITHUB_OUTPUT
            else
              echo "Enable tracing label not found."
              echo "trace=false" >> $GITHUB_OUTPUT
            fi
          else
            echo "No labels present or labels are null."
            echo "trace=false" >> $GITHUB_OUTPUT
          fi

      - name: Setup Grafana and OpenTelemetry
        id: docker-setup
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          # Create network
          docker network create --driver bridge tracing

          # Make trace directory
          cd integration-tests/smoke/
          mkdir ./traces
          chmod -R 777 ./traces

          # Switch directory
          cd ../../.github/tracing

          # Create a Docker volume for traces
          # docker volume create otel-traces

          # Start OpenTelemetry Collector
          # Note the user must be set to the same user as the runner for the trace data to be accessible
          docker run -d --network=tracing --name=otel-collector \
            -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \
            -v $PWD/../../integration-tests/smoke/traces:/tracing \
            --user "$(id -u):$(id -g)" \
            -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml
      - name: Locate Docker Volume
        id: locate-volume
        if: false
        run: |
          echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT
      - name: Show Otel-Collector Logs
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          docker logs otel-collector
      ## Run this step when changes that require tests to be run are made
      - name: Run Tests
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        env:
          PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
          LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
          LOKI_URL: ${{ secrets.LOKI_URL }}
          LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          GRAFANA_URL: ${{ vars.GRAFANA_URL }}
          GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }}
          RUN_ID: ${{ github.run_id }}
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_name: ${{ matrix.product.name }}-test-logs
          artifacts_location: ./integration-tests/smoke/logs/
          publish_check_name: ${{ matrix.product.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
      ## Run this step when changes that do not need the test to run are made
      - name: Run Setup
        if: needs.changes.outputs.src == 'false'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Show Otel-Collector Logs
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          docker logs otel-collector
      - name: Permissions on traces
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          ls -l ./integration-tests/smoke/traces
      - name: Upload Trace Data
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: trace-data
          path: ./integration-tests/smoke/traces/trace-data.json
      - name: Print failed test summary
        if: always()
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@882b45231ed4ab273b0bd56ec4ab6bf0c31324b6 # v2.2.16
        with:
          test_directory: ./integration-tests/smoke/      

  ### Used to check the required checks box when the matrix completes
  eth-smoke-tests:
    if: always()
    runs-on: ubuntu-latest
    name: ETH Smoke Tests
    # needs: [eth-smoke-tests-matrix, eth-smoke-tests-matrix-automation]
    needs: [eth-smoke-tests-matrix]
    steps:
      - name: Check smoke test matrix status
        if: needs.eth-smoke-tests-matrix.result != 'success' || needs.eth-smoke-tests-matrix-automation.result != 'success'
        run: |
          echo "${{ needs.eth-smoke-tests-matrix.result }}"
          exit 1
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: ETH Smoke Tests
          matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }}
        continue-on-error: true

  cleanup:
    name: Clean up integration environment deployments
    if: always()
    needs: [eth-smoke-tests]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: 🧼 Clean up Environment
        if: ${{ github.event_name == 'pull_request' }}
        uses: ./.github/actions/delete-deployments
        with:
          environment: integration
          ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663

      - name: Collect Metrics
        if: ${{ github.event_name == 'pull_request' }}
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Clean up integration environment deployments
        continue-on-error: true

  # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
  # this will also only run if both of the matrix jobs pass
  eth-smoke-go-mod-cache:
    environment: integration
    needs: [eth-smoke-tests]
    runs-on: ubuntu20.04-16cores-64GB
    name: ETH Smoke Tests Go Mod Cache
    continue-on-error: true
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Run Setup
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_download_vendor_packages_command: |
            cd ./integration-tests
            go mod download
            # force download of test dependencies
            go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure"
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "false"

  ### Migration tests
  node-migration-tests:
    name: Version Migration Tests
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [build-chainlink, changes, build-test-image]
    # Only run migration tests on new tags
    if: startsWith(github.ref, 'refs/tags/')
    env:
      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
      UPGRADE_VERSION: ${{ github.sha }}
      UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
      TEST_LOG_LEVEL: debug
      TEST_SUITE: migration
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Get Latest Version
        id: get_latest_version
        run: |
          untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name)
          latest_version="${untrimmed_ver:1}"
          echo "latest_version=${latest_version} | tee -a $GITHUB_OUTPUT"
      - name: Name Versions
        run: |
          echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ github.sha }}'"
      - name: Run Migration Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }}
          artifacts_location: ./integration-tests/migration/logs
          publish_check_name: Node Migration Test Results
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Upload test log
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        if: failure()
        with:
          name: test-log-${{ matrix.product.name }}
          path: /tmp/gotest.log
          retention-days: 7
        continue-on-error: true
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Version Migration Tests
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true

  ### Solana Section
  # get_solana_sha:
  #   name: Get Solana Sha From Go Mod
  #   environment: Integration
  #   runs-on: ubuntu-latest
  #   outputs:
  #     sha: ${{ steps.getsha.outputs.sha }}
  #   steps:
  #     - name: Checkout the repo
  #       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  #       with:
  #         ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
  #     - name: Setup Go
  #       uses: ./.github/actions/setup-go
  #       with:
  #         only-modules: "true"
  #     - name: Get the sha from go mod
  #       id: getshortsha
  #       run: |
  #         sol_ver=$(go list -m -json github.com/smartcontractkit/chainlink-solana  | jq -r .Version)
  #         if [ -z "${sol_ver}" ]; then
  #             echo "Error: could not get the solana version from the go.mod file, look above for error(s)"
  #             exit 1
  #         fi
  #         short_sha="${sol_ver##*-}"
  #         echo "short sha is: ${short_sha}"
  #         echo "short_sha=${short_sha}" >> "$GITHUB_OUTPUT"
  #     - name: Checkout solana
  #       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  #       with:
  #         repository: smartcontractkit/chainlink-solana
  #         ref: develop
  #         fetch-depth: 0
  #         path: solanapath
  #     - name: Get long sha
  #       id: getsha
  #       run: |
  #         cd solanapath
  #         full_sha=$(git rev-parse ${{steps.getshortsha.outputs.short_sha}})
  #         if [ -z "${full_sha}" ]; then
  #             echo "Error: could not get the full sha from the short sha using git, look above for error(s)"
  #             exit 1
  #         fi
  #         echo "sha is: ${full_sha}"
  #         echo "sha=${full_sha}" >> "$GITHUB_OUTPUT"

  # get_projectserum_version:
  #   name: Get ProjectSerum Version
  #   environment: integration
  #   runs-on: ubuntu-latest
  #   needs: [get_solana_sha]
  #   outputs:
  #     projectserum_version: ${{ steps.psversion.outputs.projectserum_version }}
  #   steps:
  #     - name: Checkout the solana repo
  #       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  #       with:
  #         repository: smartcontractkit/chainlink-solana
  #         ref: ${{ needs.get_solana_sha.outputs.sha }}
  #     - name: Get ProjectSerum Version
  #       id: psversion
  #       uses: smartcontractkit/chainlink-solana/.github/actions/projectserum_version@4b971869e26b79c7ce3fb7c98005cc2e3f350915 # stable action on Oct 12 2022

  # solana-test-image-exists:
  #   environment: integration
  #   permissions:
  #     checks: write
  #     pull-requests: write
  #     id-token: write
  #     contents: read
  #   name: Check If Solana Test Image Exists
  #   runs-on: ubuntu-latest
  #   needs: [get_solana_sha]
  #   outputs:
  #     exists: ${{ steps.check-image.outputs.exists }}
  #   steps:
  #     - name: Check if image exists
  #       id: check-image
  #       uses: smartcontractkit/chainlink-github-actions/docker/image-exists@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
  #       with:
  #         repository: chainlink-solana-tests
  #         tag: ${{ needs.get_solana_sha.outputs.sha }}
  #         AWS_REGION: ${{ secrets.QA_AWS_REGION }}
  #         AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  # solana-build-contracts:
  #   environment: integration
  #   permissions:
  #     checks: write
  #     pull-requests: write
  #     id-token: write
  #     contents: read
  #   name: Solana Build Artifacts
  #   runs-on: ubuntu20.04-16cores-64GB
  #   needs:
  #     [
  #       changes,
  #       get_projectserum_version,
  #       solana-test-image-exists,
  #       get_solana_sha,
  #     ]
  #   container:
  #     image: projectserum/build:${{ needs.get_projectserum_version.outputs.projectserum_version }}
  #     env:
  #       RUSTUP_HOME: "/root/.rustup"
  #       FORCE_COLOR: 1
  #   steps:
  #     - name: Collect Metrics
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       id: collect-gha-metrics
  #       uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
  #       with:
  #         basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
  #         hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
  #         this-job-name: Solana Build Artifacts
  #       continue-on-error: true
  #     - name: Checkout the solana repo
  #       # Use v3.6.0 because the custom runner (container configured above)
  #       # doesn't have node20 installed which is required for versions >=4
  #       uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
  #       with:
  #         repository: smartcontractkit/chainlink-solana
  #         ref: ${{ needs.get_solana_sha.outputs.sha }}
  #     - name: Build contracts
  #       if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
  #       uses: smartcontractkit/chainlink-solana/.github/actions/build_contract_artifacts@21675b3a7dcdff8e790391708d4763020cace21e # stable action on December 18 2023
  #       with:
  #         ref: ${{ needs.get_solana_sha.outputs.sha }}

  # solana-build-test-image:
  #   environment: integration
  #   permissions:
  #     checks: write
  #     pull-requests: write
  #     id-token: write
  #     contents: read
  #   name: Solana Build Test Image
  #   runs-on: ubuntu20.04-16cores-64GB
  #   needs:
  #     [
  #       solana-build-contracts,
  #       solana-test-image-exists,
  #       changes,
  #       get_solana_sha,
  #     ]
  #   env:
  #     CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
  #   steps:
  #     - name: Collect Metrics
  #       if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
  #       id: collect-gha-metrics
  #       uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
  #       with:
  #         basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
  #         hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
  #         this-job-name: Solana Build Test Image
  #       continue-on-error: true
  #     - name: Checkout the repo
  #       if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
  #       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  #       with:
  #         repository: smartcontractkit/chainlink-solana
  #         ref: ${{ needs.get_solana_sha.outputs.sha }}
  #     - name: Build Test Image
  #       if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
  #       uses: ./.github/actions/build-test-image
  #       with:
  #         tag: ${{ needs.get_solana_sha.outputs.sha }}
  #         artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }}
  #         QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
  #         QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
  #         QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
  #     - run: echo "this exists so we don't have to run anything else if the build is skipped"
  #       if: needs.changes.outputs.src == 'false' || needs.solana-test-image-exists.outputs.exists == 'true'

  # solana-smoke-tests:
  #   if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
  #   environment: integration
  #   permissions:
  #     checks: write
  #     pull-requests: write
  #     id-token: write
  #     contents: read
  #   name: Solana Smoke Tests
  #   runs-on: ubuntu20.04-16cores-64GB
  #   needs:
  #     [
  #       build-chainlink,
  #       solana-build-contracts,
  #       solana-build-test-image,
  #       changes,
  #       get_solana_sha,
  #     ]
  #   env:
  #     CHAINLINK_COMMIT_SHA: ${{ github.sha }}
  #     CHAINLINK_ENV_USER: ${{ github.actor }}
  #     TEST_LOG_LEVEL: debug
  #     CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
  #   steps:
  #     - name: Collect Metrics
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       id: collect-gha-metrics
  #       uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
  #       with:
  #         basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
  #         hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
  #         this-job-name: Solana Smoke Tests
  #         test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
  #       continue-on-error: true
  #     - name: Checkout the repo
  #       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  #       with:
  #         repository: smartcontractkit/chainlink-solana
  #         ref: ${{ needs.get_solana_sha.outputs.sha }}
  #     - name: Run Setup
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
  #       with:
  #         go_mod_path: ./integration-tests/go.mod
  #         cache_restore_only: true
  #         cache_key_id: core-solana-e2e-${{ env.MOD_CACHE_VERSION }}
  #         aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
  #         dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
  #         dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
  #         QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
  #         QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
  #         QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
  #     - name: Pull Artfacts
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       run: |
  #         IMAGE_NAME=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }}
  #         # Pull the Docker image
  #         docker pull "$IMAGE_NAME"

  #         # Create a container without starting it
  #         CONTAINER_ID=$(docker create "$IMAGE_NAME")

  #         # Copy the artifacts from the container
  #         mkdir -p ./${{env.CONTRACT_ARTIFACTS_PATH}}/
  #         docker cp "$CONTAINER_ID:/go/testdir/${{env.CONTRACT_ARTIFACTS_PATH}}/" "./${{env.CONTRACT_ARTIFACTS_PATH}}/../"

  #         # Remove the created container
  #         docker rm "$CONTAINER_ID"
  #     - name: Run Tests
  #       if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
  #       uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
  #       with:
  #         test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke
  #         cl_repo: ${{ env.CHAINLINK_IMAGE }}
  #         cl_image_tag: ${{ github.sha }}
  #         artifacts_location: /home/runner/work/chainlink-solana/chainlink-solana/integration-tests/logs
  #         publish_check_name: Solana Smoke Test Results
  #         go_mod_path: ./integration-tests/go.mod
  #         cache_key_id: core-solana-e2e-${{ env.MOD_CACHE_VERSION }}
  #         token: ${{ secrets.GITHUB_TOKEN }}
  #         aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
  #         QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
  #         QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
  #         QA_KUBECONFIG: ""
  #         run_setup: false
  #     - name: Upload test log
  #       uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
  #       if: failure()
  #       with:
  #         name: test-log-solana
  #         path: /tmp/gotest.log
  #         retention-days: 7
  #       continue-on-error: true