integration-tests.yml

name: Integration Tests
on:
  merge_group:
  pull_request:
  push:
    tags:
      - "*"
  workflow_dispatch:

# Only run 1 of this workflow at a time per PR
concurrency:
  group: integration-tests-chainlink-${{ github.ref }}
  cancel-in-progress: true

env:
  # for run-test variables and environment
  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  TEST_SUITE: smoke
  TEST_ARGS: -test.timeout 12m
  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
  MOD_CACHE_VERSION: 2

jobs:
  enforce-ctf-version:
    name: Enforce CTF Version
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Check Merge Group Condition
        id: condition-check
        run: |
          echo "Checking event condition..."
          SHOULD_ENFORCE="false"
          if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
              echo "We are in a merge_group event, now check if we are on the develop branch"
              target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .merge_group.base_ref)
              if [[ "$target_branch" == "refs/heads/develop" ]]; then
                  echo "We are on the develop branch, we should enforce ctf version"
                  SHOULD_ENFORCE="true"
              fi
          fi
          echo "should we enforce ctf version = $SHOULD_ENFORCE"
          echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT
      - name: Enforce CTF Version
        if: steps.condition-check.outputs.should-enforce == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          go-project-path: ./integration-tests
          module-name: github.com/smartcontractkit/chainlink-testing-framework
          enforce-semantic-tag: "true"
  changes:
    environment: integration
    name: Check Paths That Require Tests To Run
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
        id: changes
        with:
          filters: |
            src:
              - '**/*.go'
              - '**/*go.sum'
              - '**/*go.mod'
              - '.github/workflows/integration-tests.yml'
              - '**/*Dockerfile'
              - 'core/**/config/**/*.toml'
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Check Paths That Require Tests To Run
        continue-on-error: true
    outputs:
      src: ${{ steps.changes.outputs.src }}

  build-lint-integration-tests:
    name: Build and Lint integration-tests
    runs-on: ubuntu20.04-16cores-64GB
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup Go
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
      - name: Build Go
        run: |
          cd ./integration-tests
          go build ./...
          SELECTED_NETWORKS=SIMULATED go test -run=^# ./...
      - name: Lint Go
        uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
        with:
          version: v1.55.2
          # We already cache these directories in setup-go
          skip-pkg-cache: true
          skip-build-cache: true
          # only-new-issues is only applicable to PRs, otherwise it is always set to false
          only-new-issues: false # disabled for PRs due to unreliability
          args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml
          working-directory: ./integration-tests

  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        image:
          - name: ""
            dockerfile: core/chainlink.Dockerfile
            tag-suffix: ""
          - name: (plugins)
            dockerfile: plugins/chainlink.Dockerfile
            tag-suffix: -plugins
    name: Build Chainlink Image ${{ matrix.image.name }}
    runs-on: ubuntu20.04-16cores-64GB
    needs: [changes, enforce-ctf-version]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Build Chainlink Image ${{ matrix.image.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Chainlink Image
        if: needs.changes.outputs.src == 'true'
        uses: ./.github/actions/build-chainlink-image
        with:
          tag_suffix: ${{ matrix.image.tag-suffix }}
          dockerfile: ${{ matrix.image.dockerfile }}
          git_commit_sha: ${{ github.sha }}
          GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  build-test-image:
    if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Test Image
    runs-on: ubuntu20.04-16cores-64GB
    needs: [changes]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Test Image
        if: needs.changes.outputs.src == 'true'
        uses: ./.github/actions/build-test-image
        with:
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

  compare-tests:
    needs: [changes]
    runs-on: ubuntu-latest
    name: Compare/Build Automation Test List
    outputs:
      matrix: ${{ env.MATRIX_JSON }}
    steps:
      - name: Check for Skip Tests Label
        if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests')
        run: |
          echo "## \`skip-smoke-tests\` label is active, skipping E2E smoke tests" >>$GITHUB_STEP_SUMMARY
          exit 0
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Compare Test Lists
        run: |
          cd ./integration-tests
          ./scripts/compareTestList.sh ./smoke/automation_test.go
          ./scripts/compareTestList.sh ./smoke/keeper_test.go
      - name: Build Test Matrix Lists
        id: build-test-matrix-list
        run: |
          cd ./integration-tests
          MATRIX_JSON_AUTOMATION=$(./scripts/buildTestMatrixList.sh ./smoke/automation_test.go automation ubuntu-latest 1)
          MATRIX_JSON_KEEPER=$(./scripts/buildTestMatrixList.sh ./smoke/keeper_test.go keeper ubuntu-latest 1)
          COMBINED_ARRAY=$(jq -c -n "$MATRIX_JSON_AUTOMATION + $MATRIX_JSON_KEEPER")

          # if we running a PR against the develop branch we should only run the automation tests unless we are in the merge group event
          if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
            echo "We are in a merge_group event, run both automation and keepers tests"
            echo "MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV
          else
            echo "we are not in a merge_group event, if this is a PR to develop run only automation tests, otherwise run everything because we could be running against a release branch"
            target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.base.ref)
            if [[ "$target_branch" == "develop" ]]; then
              echo "only run automation tests"
              echo "MATRIX_JSON=${MATRIX_JSON_AUTOMATION}" >> $GITHUB_ENV
            else
              echo "run both automation and keepers tests"
              echo "MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV
            fi
          fi

  eth-smoke-tests-matrix-automation:
    if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs:
      [build-chainlink, changes, compare-tests, build-lint-integration-tests]
    env:
      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
    strategy:
      fail-fast: false
      matrix:
        product: ${{fromJson(needs.compare-tests.outputs.matrix)}}
    runs-on: ${{ matrix.product.os }}
    name: ETH Smoke Tests ${{ matrix.product.name }}
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Go Test Command
        id: build-go-test-command
        run: |
          # if the matrix.product.run is set, use it for a different command
          if [ "${{ matrix.product.run }}" != "" ]; then
            echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
          else
            echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
          fi
      ## Run this step when changes that require tests to be run are made
      - name: Run Tests
        if: needs.changes.outputs.src == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        env:
          PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ github.sha }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_location: ./integration-tests/smoke/logs/
          publish_check_name: ${{ matrix.product.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true

  eth-smoke-tests-matrix:
    if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes, build-lint-integration-tests]
    env:
      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
    strategy:
      fail-fast: false
      matrix:
        product:
          - name: cron
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ""
          - name: flux
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ""
          - name: ocr
            nodes: 1
            os: ubuntu-latest
            file: ocr
            pyroscope_env: ci-smoke-ocr-evm-simulated                         
          - name: ocr2
            nodes: 1
            os: ubuntu-latest
            file: ocr2
            pyroscope_env: ci-smoke-ocr2-evm-simulated                                                     
          - name: ocr2
            nodes: 1
            os: ubuntu-latest
            run: -run TestOCRv2Request
            file: ocr2
            pyroscope_env: ci-smoke-ocr2-evm-simulated                                                           
          - name: ocr2
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated
            tag_suffix: "-plugins"
          - name: runlog
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ""
          - name: vrf
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ci-smoke-vrf-evm-simulated
          - name: vrfv2
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ci-smoke-vrf2-evm-simulated
          - name: vrfv2plus
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ci-smoke-vrf2plus-evm-simulated                       
          - name: forwarder_ocr
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
          - name: forwarders_ocr2
            nodes: 1
            os: ubuntu-latest
            pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
    runs-on: ${{ matrix.product.os }}
    name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Go Test Command
        id: build-go-test-command
        run: |
          # if the matrix.product.run is set, use it for a different command
          if [ "${{ matrix.product.run }}" != "" ]; then
            echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
          else
            echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
          fi
      - name: Check for "enable tracing" label
        id: check-label
        run: |
          label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH")

          if [[ -n "$label" ]]; then
            if [[ "$label" == "enable tracing" ]]; then
              echo "Enable tracing label found."
              echo "trace=true" >> $GITHUB_OUTPUT
            else
              echo "Enable tracing label not found."
              echo "trace=false" >> $GITHUB_OUTPUT
            fi
          else
            echo "No labels present or labels are null."
            echo "trace=false" >> $GITHUB_OUTPUT
          fi

      - name: Setup Grafana and OpenTelemetry
        id: docker-setup
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          # Create network
          docker network create --driver bridge tracing

          # Make trace directory
          cd integration-tests/smoke/
          mkdir ./traces
          chmod -R 777 ./traces

          # Switch directory
          cd ../../.github/tracing

          # Create a Docker volume for traces
          # docker volume create otel-traces

          # Start OpenTelemetry Collector
          # Note the user must be set to the same user as the runner for the trace data to be accessible
          docker run -d --network=tracing --name=otel-collector \
            -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \
            -v $PWD/../../integration-tests/smoke/traces:/tracing \
            --user "$(id -u):$(id -g)" \
            -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml
      - name: Locate Docker Volume
        id: locate-volume
        if: false
        run: |
          echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT
      - name: Show Otel-Collector Logs
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          docker logs otel-collector
      ## Run this step when changes that require tests to be run are made
      - name: Run Tests
        if: needs.changes.outputs.src == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        env:
          PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
          LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
          LOKI_URL: ${{ secrets.LOKI_URL }}
          LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          GRAFANA_URL: ${{ vars.GRAFANA_URL }}
          GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }}
          RUN_ID: ${{ github.run_id }}
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_name: ${{ matrix.product.name }}-test-logs
          artifacts_location: ./integration-tests/smoke/logs/
          publish_check_name: ${{ matrix.product.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
      ## Run this step when changes that do not need the test to run are made
      - name: Run Setup
        if: needs.changes.outputs.src == 'false'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Show Otel-Collector Logs
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          docker logs otel-collector
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Permissions on traces
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          ls -l ./integration-tests/smoke/traces
      - name: Upload Trace Data
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        uses: actions/upload-artifact@v3
        with:
          name: trace-data
          path: ./integration-tests/smoke/traces/trace-data.json
      - name: Print failed test summary
        if: always()
        run: |
          directory="./integration-tests/smoke/.test_summary"
          files=("$directory"/*)
          if [ -d "$directory" ]; then
            echo "Test summary folder found"
            if [ ${#files[@]} -gt 0 ]; then
              first_file="${files[0]}"
              echo "Name of the first test summary file: $(basename "$first_file")"
              echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY
              cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY
              if [ ${#files[@]} -gt 1 ]; then
                echo "Found more than one test summary file. This is incorrect, there should be only one file"
              fi
            else
                echo "Test summary directory is empty. This should not happen"
            fi
          else
            echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting"
          fi

  ### Used to check the required checks box when the matrix completes
  eth-smoke-tests:
    if: always()
    runs-on: ubuntu-latest
    name: ETH Smoke Tests
    needs: [eth-smoke-tests-matrix, eth-smoke-tests-matrix-automation]
    steps:
      - name: Check smoke test matrix status
        if: needs.eth-smoke-tests-matrix.result != 'success' || needs.eth-smoke-tests-matrix-automation.result != 'success'
        run: |
          echo "${{ needs.eth-smoke-tests-matrix.result }}"
          exit 1
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: ETH Smoke Tests
          matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }}
        continue-on-error: true

  cleanup:
    name: Clean up integration environment deployments
    if: always()
    needs: [eth-smoke-tests]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: 🧼 Clean up Environment
        if: ${{ github.event_name == 'pull_request' }}
        uses: ./.github/actions/delete-deployments
        with:
          environment: integration
          ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663

      - name: Collect Metrics
        if: ${{ github.event_name == 'pull_request' }}
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Clean up integration environment deployments
        continue-on-error: true

  # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
  # this will also only run if both of the matrix jobs pass
  eth-smoke-go-mod-cache:
    environment: integration
    needs: [eth-smoke-tests]
    runs-on: ubuntu20.04-16cores-64GB
    name: ETH Smoke Tests Go Mod Cache
    continue-on-error: true
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Run Setup
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_download_vendor_packages_command: |
            cd ./integration-tests
            go mod download
            # force download of test dependencies
            go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure"
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "false"

  ### Migration tests
  node-migration-tests:
    name: Version Migration Tests
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [build-chainlink, changes, build-test-image]
    # Only run migration tests on new tags
    if: startsWith(github.ref, 'refs/tags/')
    env:
      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
      UPGRADE_VERSION: ${{ github.sha }}
      UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
      TEST_LOG_LEVEL: debug
      TEST_SUITE: migration
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Get Latest Version
        id: get_latest_version
        run: |
          untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name)
          latest_version="${untrimmed_ver:1}"
          echo "latest_version=${latest_version} | tee -a $GITHUB_OUTPUT"
      - name: Name Versions
        run: |
          echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ github.sha }}'"
      - name: Run Migration Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }}
          artifacts_location: ./integration-tests/migration/logs
          publish_check_name: Node Migration Test Results
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Upload test log
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        if: failure()
        with:
          name: test-log-${{ matrix.product.name }}
          path: /tmp/gotest.log
          retention-days: 7
        continue-on-error: true
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Version Migration Tests
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true

  ### Solana Section
  get_solana_sha:
    name: Get Solana Sha From Go Mod
    environment: Integration
    runs-on: ubuntu-latest
    outputs:
      sha: ${{ steps.getsha.outputs.sha }}
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Setup Go
        uses: ./.github/actions/setup-go
        with:
          only-modules: "true"
      - name: Get the sha from go mod
        id: getshortsha
        run: |
          sol_ver=$(go list -m -json github.com/smartcontractkit/chainlink-solana  | jq -r .Version)
          if [ -z "${sol_ver}" ]; then
              echo "Error: could not get the solana version from the go.mod file, look above for error(s)"
              exit 1
          fi
          short_sha="${sol_ver##*-}"
          echo "short sha is: ${short_sha}"
          echo "short_sha=${short_sha}" >> "$GITHUB_OUTPUT"
      - name: Checkout solana
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/chainlink-solana
          ref: develop
          fetch-depth: 0
          path: solanapath
      - name: Get long sha
        id: getsha
        run: |
          cd solanapath
          full_sha=$(git rev-parse ${{steps.getshortsha.outputs.short_sha}})
          if [ -z "${full_sha}" ]; then
              echo "Error: could not get the full sha from the short sha using git, look above for error(s)"
              exit 1
          fi
          echo "sha is: ${full_sha}"
          echo "sha=${full_sha}" >> "$GITHUB_OUTPUT"

  get_projectserum_version:
    name: Get ProjectSerum Version
    environment: integration
    runs-on: ubuntu-latest
    needs: [get_solana_sha]
    outputs:
      projectserum_version: ${{ steps.psversion.outputs.projectserum_version }}
    steps:
      - name: Checkout the solana repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Get ProjectSerum Version
        id: psversion
        uses: smartcontractkit/chainlink-solana/.github/actions/projectserum_version@4b971869e26b79c7ce3fb7c98005cc2e3f350915 # stable action on Oct 12 2022

  solana-test-image-exists:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Check If Solana Test Image Exists
    runs-on: ubuntu-latest
    needs: [get_solana_sha]
    outputs:
      exists: ${{ steps.check-image.outputs.exists }}
    steps:
      - name: Check if image exists
        id: check-image
        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          repository: chainlink-solana-tests
          tag: ${{ needs.get_solana_sha.outputs.sha }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  solana-build-contracts:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Solana Build Artifacts
    runs-on: ubuntu20.04-16cores-64GB
    needs:
      [
        changes,
        get_projectserum_version,
        solana-test-image-exists,
        get_solana_sha,
      ]
    container:
      image: projectserum/build:${{ needs.get_projectserum_version.outputs.projectserum_version }}
      env:
        RUSTUP_HOME: "/root/.rustup"
        FORCE_COLOR: 1
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Solana Build Artifacts
        continue-on-error: true
      - name: Checkout the solana repo
        # Use v3.6.0 because the custom runner (container configured above)
        # doesn't have node20 installed which is required for versions >=4
        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Build contracts
        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
        uses: smartcontractkit/chainlink-solana/.github/actions/build_contract_artifacts@21675b3a7dcdff8e790391708d4763020cace21e # stable action on December 18 2023
        with:
          ref: ${{ needs.get_solana_sha.outputs.sha }}

  solana-build-test-image:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Solana Build Test Image
    runs-on: ubuntu20.04-16cores-64GB
    needs:
      [
        solana-build-contracts,
        solana-test-image-exists,
        changes,
        get_solana_sha,
      ]
    env:
      CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Solana Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Build Test Image
        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
        uses: ./.github/actions/build-test-image
        with:
          tag: ${{ needs.get_solana_sha.outputs.sha }}
          artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      - run: echo "this exists so we don't have to run anything else if the build is skipped"
        if: needs.changes.outputs.src == 'false' || needs.solana-test-image-exists.outputs.exists == 'true'

  solana-smoke-tests:
    if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Solana Smoke Tests
    runs-on: ubuntu20.04-16cores-64GB
    needs:
      [
        build-chainlink,
        solana-build-contracts,
        solana-build-test-image,
        changes,
        get_solana_sha,
      ]
    env:
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
      CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: Solana Smoke Tests
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Run Setup
        if: needs.changes.outputs.src == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          go_mod_path: ./integration-tests/go.mod
          cache_restore_only: true
          cache_key_id: core-solana-e2e-${{ env.MOD_CACHE_VERSION }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Pull Artfacts
        if: needs.changes.outputs.src == 'true'
        run: |
          IMAGE_NAME=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }}
          # Pull the Docker image
          docker pull "$IMAGE_NAME"

          # Create a container without starting it
          CONTAINER_ID=$(docker create "$IMAGE_NAME")

          # Copy the artifacts from the container
          mkdir -p ./${{env.CONTRACT_ARTIFACTS_PATH}}/
          docker cp "$CONTAINER_ID:/go/testdir/${{env.CONTRACT_ARTIFACTS_PATH}}/" "./${{env.CONTRACT_ARTIFACTS_PATH}}/../"

          # Remove the created container
          docker rm "$CONTAINER_ID"
      - name: Run Tests
        if: needs.changes.outputs.src == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
        with:
          test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ github.sha }}
          artifacts_location: /home/runner/work/chainlink-solana/chainlink-solana/integration-tests/logs
          publish_check_name: Solana Smoke Test Results
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-solana-e2e-${{ env.MOD_CACHE_VERSION }}
          token: ${{ secrets.GITHUB_TOKEN }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
          run_setup: false
      - name: Upload test log
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        if: failure()
        with:
          name: test-log-solana
          path: /tmp/gotest.log
          retention-days: 7
        continue-on-error: true