integration-tests.yml

name: Integration Tests
run-name: Integration Tests ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }}
on:
  merge_group:
  pull_request:
  push:
    tags:
      - "*"
  workflow_dispatch:
    inputs:
      cl_ref:
        description: 'The ref to checkout, defaults to the calling branch'
        required: false
        type: string
      evm-ref:
        description: 'The sha of the chainlink-evm commit to use if wanted'
        required: false
        type: string
      run_solana:
        description: 'Run solana tests'
        required: false
        type: string
        default: 'false'
      distinct_run_name:
        description: 'A unique identifier for this run, only use from other repos'
        required: false
        type: string

# Only run 1 of this workflow at a time per PR
concurrency:
  group: ${{ github.ref }}-${{ github.repository }}-${{ github.event_name }}--e2e-tests-${{ inputs.distinct_run_name }}
  cancel-in-progress: true

env:
  # for run-test variables and environment
  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ inputs.evm-ref || github.sha }}
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  TEST_SUITE: smoke
  TEST_ARGS: -test.timeout 12m
  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
  MOD_CACHE_VERSION: 2
  COLLECTION_ID: chainlink-e2e-tests

jobs:
  enforce-ctf-version:
    name: Enforce CTF Version
    runs-on: ubuntu-latest
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    steps:
      - run: echo "${{github.event_name}}"
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref }}
      - name: Check Merge Group Condition
        id: condition-check
        run: |
          echo "Checking event condition..."
          SHOULD_ENFORCE="false"
          if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
              echo "We are in a merge_group event, now check if we are on the develop branch"
              target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .merge_group.base_ref)
              if [[ "$target_branch" == "refs/heads/develop" ]]; then
                  echo "We are on the develop branch, we should enforce ctf version"
                  SHOULD_ENFORCE="true"
              fi
          fi
          echo "should we enforce ctf version = $SHOULD_ENFORCE"
          echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT
      - name: Enforce CTF Version
        if: steps.condition-check.outputs.should-enforce == 'true'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
        with:
          go-project-path: ./integration-tests
          module-name: github.com/smartcontractkit/chainlink-testing-framework
          enforce-semantic-tag: "true"

  changes:
    environment: integration
    name: Check Paths That Require Tests To Run
    runs-on: ubuntu-latest
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref }}
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: changes
        with:
          filters: |
            changes:
              - '**/*.go'
              - '**/*go.sum'
              - '**/*go.mod'
              - '.github/workflows/integration-tests.yml'
              - '**/*Dockerfile'
              - 'core/**/migrations/*.sql'
              - 'core/**/config/**/*.toml'
              - 'integration-tests/**/*.toml'
      - name: Ignore Filter On Workflow Dispatch
        if: ${{ github.event_name == 'workflow_dispatch' }}
        id: ignore-filter
        run: echo "changes=true" >> $GITHUB_OUTPUT
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-check-paths
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Check Paths That Require Tests To Run
        continue-on-error: true
    outputs:
      src: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.changes }}

  build-lint-integration-tests:
    name: Build and Lint ${{ matrix.project.name }}
    runs-on: ubuntu22.04-16cores-64GB
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    strategy:
      matrix:
        project:
          - name: integration-tests
            id: e2e
            path: ./integration-tests
            cache-id: e2e
          - name: load
            id: load
            path: ./integration-tests/load
            cache-id: load
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-build-lint-${{ matrix.project.id }}
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build and Lint ${{ matrix.project.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref }}
      - name: Setup Go
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
        with:
          test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download
          go_mod_path: ${{ matrix.project.path }}/go.mod
          cache_key_id: core-${{ matrix.project.cache-id }}-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
      - name: Build Go
        run: |
          cd ${{ matrix.project.path }}
          go build ./...
          go test -run=^# ./...
      - name: Lint Go
        uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
        with:
          version: v1.59.1
          # We already cache these directories in setup-go
          skip-pkg-cache: true
          skip-build-cache: true
          # only-new-issues is only applicable to PRs, otherwise it is always set to false
          only-new-issues: false # disabled for PRs due to unreliability
          args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml
          working-directory: ${{ matrix.project.path }}

  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        image:
          - name: ""
            dockerfile: core/chainlink.Dockerfile
            tag-suffix: ""
          - name: (plugins)
            dockerfile: plugins/chainlink.Dockerfile
            tag-suffix: -plugins
    name: Build Chainlink Image ${{ matrix.image.name }}
    runs-on: ubuntu22.04-16cores-64GB
    needs: [changes, enforce-ctf-version]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-build-chainlink
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Chainlink Image ${{ matrix.image.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Chainlink Image
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/build-chainlink-image
        with:
          tag_suffix: ${{ matrix.image.tag-suffix }}
          dockerfile: ${{ matrix.image.dockerfile }}
          git_commit_sha: ${{ inputs.evm-ref || github.sha }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          dep_evm_sha: ${{ inputs.evm-ref }}

  build-test-image:
    if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Test Image
    runs-on: ubuntu22.04-16cores-64GB
    needs: [changes]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-build-test-image
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Go Test Command
        id: build-go-test-command
        run: |
          # if the matrix.product.run is set, use it for a different command
          if [ "${{ matrix.product.run }}" != "" ]; then
            echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
          else
            echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
          fi
      - name: Setup GAP for Grafana
        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
        with:
          # aws inputs
          aws-region: ${{ secrets.AWS_REGION }}
          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
          # other inputs
          duplicate-authorization-header: "true"
      - name: Prepare Base64 TOML override
        uses: ./.github/actions/setup-create-base64-config
        with:
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
          chainlinkVersion: ${{ inputs.evm-ref || github.sha }}
          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}

  eth-smoke-tests-matrix:
    if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
    environment: integration
    permissions:
      actions: read
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes, build-lint-integration-tests]
    env:
      SELECTED_NETWORKS: SIMULATED
      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
    strategy:
      fail-fast: false
      matrix:
        product:
          - name: ccip-lm-smoke
            nodes: 1
            os: ubuntu-latest
            file: lm
            dir: ccip-tests/smoke
            run: -run ^TestLmBasic$
          - name: ccip-smoke
            nodes: 1
            os: ubuntu-latest
            file: ccip
            dir: ccip-tests/smoke
            run: -run ^TestSmokeCCIPForBidirectionalLane$
          - name: ccip-smoke-1.4-pools
            nodes: 1
            os: ubuntu-latest
            file: ccip
            dir: ccip-tests/smoke
            run: -run ^TestSmokeCCIPForBidirectionalLane$
            config_path: ./integration-tests/ccip-tests/testconfig/tomls/contract-version1.4.toml
          - name: ccip-smoke-usdc
            nodes: 1
            os: ubuntu-latest
            file: ccip
            dir: ccip-tests/smoke
            run: -run ^TestSmokeCCIPForBidirectionalLane$
            config_path: ./integration-tests/ccip-tests/testconfig/tomls/usdc_mock_deployment.toml
          - name: ccip-smoke-db-compatibility
            nodes: 1
            os: ubuntu-latest
            file: ccip
            dir: ccip-tests/smoke
            run: -run ^TestSmokeCCIPForBidirectionalLane$
            config_path: ./integration-tests/ccip-tests/testconfig/tomls/db-compatibility.toml
          - name: ccip-smoke-rate-limit
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPRateLimit$
          - name: ccip-smoke-rate-limit
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPTokenPoolRateLimits$
          - name: ccip-smoke-multicall
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPMulticall$
          - name: ccip-smoke-manual-exec
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$
          - name: ccip-smoke-on-ramp-limits
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPOnRampLimits$
          - name: ccip-smoke-off-ramp-capacity
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPOffRampCapacityLimit$
          - name: ccip-smoke-off-ramp-agg-rate-limit
            nodes: 1
            dir: ccip-tests/smoke
            os: ubuntu-latest
            file: ccip
            run: -run ^TestSmokeCCIPOffRampAggRateLimit$
          - name: runlog
            id: runlog
            nodes: 2
            os: ubuntu-latest
          - name: cron
            id: cron
            nodes: 2
            os: ubuntu-latest
          - name: flux
            id: flux
            nodes: 1
            os: ubuntu-latest
          - name: ocr
            id: ocr
            nodes: 2
            os: ubuntu-latest
            file: ocr
            pyroscope_env: ci-smoke-ocr-evm-simulated
          - name: reorg_above_finality
            id: reorg_above_finality
            nodes: 1
            os: ubuntu-latest
            file: reorg_above_finality
            pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated
          - name: ocr2
            id: ocr2
            nodes: 6
            os: ubuntu22.04-16cores-64GB
            file: ocr2
          - name: ocr2
            id: ocr2-plugins
            nodes: 6
            os: ubuntu22.04-16cores-64GB
            tag_suffix: "-plugins"
          - name: vrf
            id: vrf
            nodes: 2
            os: ubuntu-latest
          - name: vrfv2
            id: vrfv2
            nodes: 6
            os: ubuntu-latest
          - name: vrfv2plus
            id: vrfv2plus
            nodes: 9
            os: ubuntu-latest
          - name: forwarder_ocr
            id: forwarder_ocr
            nodes: 2
            os: ubuntu-latest
          - name: forwarders_ocr2
            id: forwarders_ocr2
            nodes: 2
            os: ubuntu-latest
    runs-on: ${{ matrix.product.os }}
    name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
    steps:
      # Handy for debugging resource usage
      # - name: Collect Workflow Telemetry
      #   uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
      - name: Collect Metrics
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }}
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Go Test Command
        id: build-go-test-command
        run: |
          # if dir is provided use it, otherwise use the smoke dir
          if [ "${{ matrix.product.dir }}" != "" ]; then
            dir=${{ matrix.product.dir }}
          else
            dir=smoke
          fi
          # if the matrix.product.run is set, use it for a different command
          if [ "${{ matrix.product.run }}" != "" ]; then
            echo "run_command=${{ matrix.product.run }} ./${dir}/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
          else
            echo "run_command=./${dir}/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
          fi
      - name: Check for "enable tracing" label
        id: check-label
        run: |
          label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH")

          if [[ -n "$label" ]]; then
            if [[ "$label" == "enable tracing" ]]; then
              echo "Enable tracing label found."
              echo "trace=true" >> $GITHUB_OUTPUT
            else
              echo "Enable tracing label not found."
              echo "trace=false" >> $GITHUB_OUTPUT
            fi
          else
            echo "No labels present or labels are null."
            echo "trace=false" >> $GITHUB_OUTPUT
          fi

      - name: Setup Grafana and OpenTelemetry
        id: docker-setup
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          # Create network
          docker network create --driver bridge tracing

          # Make trace directory
          cd integration-tests/smoke/
          mkdir ./traces
          chmod -R 777 ./traces

          # Switch directory
          cd ../../.github/tracing

          # Create a Docker volume for traces
          # docker volume create otel-traces

          # Start OpenTelemetry Collector
          # Note the user must be set to the same user as the runner for the trace data to be accessible
          docker run -d --network=tracing --name=otel-collector \
            -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \
            -v $PWD/../../integration-tests/smoke/traces:/tracing \
            --user "$(id -u):$(id -g)" \
            -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml
      - name: Locate Docker Volume
        id: locate-volume
        if: false
        run: |
          echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT
      - name: Show Otel-Collector Logs
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          docker logs otel-collector
      - name: Set Override Config
        id: set_override_config
        run: |
          # if the matrix.product.config_path is set, use it as the override config
          if [ "${{ matrix.product.config_path }}" != "" ]; then
            echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT"
          fi
      - name: Setup GAP for Grafana
        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
        with:
          # aws inputs
          aws-region: ${{ secrets.AWS_REGION }}
          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
          # other inputs
          duplicate-authorization-header: "true"
          # metrics inputs
          metrics-job-name: "grafana"
          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
      - name: Prepare Base64 TOML override
        uses: ./.github/actions/setup-create-base64-config
        with:
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
          chainlinkVersion: ${{ inputs.evm-ref || github.sha }}
          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
          lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          grafanaUrl: "http://localhost:8080/primary" # this is GAP url, it will know where to forward the requests so that they reach the correct Grafana instance
          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
      - name: Prepare Base64 TOML override for CCIP secrets
        uses: ./.github/actions/setup-create-base64-config-ccip
        with:
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          selectedNetworks: SIMULATED_1,SIMULATED_2
          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
          chainlinkVersion: ${{ github.sha }}
          lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          grafanaUrl: "http://localhost:8080/primary"
          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}

      ## Run this step when changes that require tests to be run are made
      - name: Run Tests
        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
        env:
          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }}
          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }}
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs
          artifacts_location: |
            ./integration-tests/smoke/logs/
            ./integration-tests/ccip-tests/smoke/logs/*
            /tmp/gotest.log
          publish_check_name: ${{ matrix.product.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
          should_tidy: "false"
          go_coverage_src_dir: /var/tmp/go-coverage
          go_coverage_dest_dir: ${{ github.workspace }}/.covdata

      - name: Upload Coverage Data
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        timeout-minutes: 2
        continue-on-error: true
        with:
          name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }}
          path: .covdata
          retention-days: 1

      # Run this step when changes that do not need the test to run are made
      - name: Run Setup
        if: needs.changes.outputs.src == 'false'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
        with:
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
          should_tidy: "false"
      - name: Show Otel-Collector Logs
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          docker logs otel-collector
      - name: Permissions on traces
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        run: |
          ls -l ./integration-tests/smoke/traces
      - name: Upload Trace Data
        if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: trace-data
          path: ./integration-tests/smoke/traces/trace-data.json
      - name: Print failed test summary
        if: always()
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@1587f59bfd626b668d303abbc90fee41b12397e6 # v2.3.23
        with:
          test_directories: ./integration-tests/smoke/,./integration-tests/ccip-tests/smoke/

  ### Used to check the required checks box when the matrix completes
  eth-smoke-tests:
    if: always()
    runs-on: ubuntu-latest
    name: ETH Smoke Tests
    needs: [eth-smoke-tests-matrix]
    steps:
      - name: Check smoke test matrix status
        if: needs.eth-smoke-tests-matrix.result != 'success'
        run: |
          echo "${{ needs.eth-smoke-tests-matrix.result }}"
          exit 1
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-matrix-results
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: ETH Smoke Tests
          matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }}
        continue-on-error: true

  cleanup:
    name: Clean up integration environment deployments
    if: always()
    needs: [eth-smoke-tests]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref }}

      - name: 🧼 Clean up Environment
        if: ${{ github.event_name == 'pull_request' }}
        uses: ./.github/actions/delete-deployments
        with:
          environment: integration
          ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663

      - name: Collect Metrics
        if: ${{ github.event_name == 'pull_request' }}
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-env-cleanup
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Clean up integration environment deployments
        continue-on-error: true

  show-coverage:
    name: Show Chainlink Node Go Coverage
    if: always()
    needs: [cleanup]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Download All Artifacts
        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
        with:
          path: cl-node-coverage-data
          pattern: cl-node-coverage-data-*
          merge-multiple: true
      - name: Show Coverage
        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"

  # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
  # this will also only run if both of the matrix jobs pass
  eth-smoke-go-mod-cache:

    environment: integration
    needs: [eth-smoke-tests]
    runs-on: ubuntu-latest
    name: ETH Smoke Tests Go Mod Cache
    continue-on-error: true
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Run Setup
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
        with:
          test_download_vendor_packages_command: |
            cd ./integration-tests
            go mod download
            # force download of test dependencies
            go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure"
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "false"

  ### Migration tests
  node-migration-tests:
    name: Version Migration Tests
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [build-chainlink, changes]
    # Only run migration tests on new tags
    if: startsWith(github.ref, 'refs/tags/')
    env:
      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      CHAINLINK_IMAGE: public.ecr.aws/w0i8p0z9/chainlink-ccip
      UPGRADE_VERSION: ${{ inputs.evm-ref || github.sha }}
      UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
      TEST_LOG_LEVEL: debug
      TEST_SUITE: migration
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-migration-tests
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Version Migration Tests
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Get Latest Version
        id: get_latest_version
        run: |
          untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name)
          latest_version="${untrimmed_ver:1}"
          # Check if latest_version is empty
          if [ -z "$latest_version" ]; then
              echo "Error: The latest_version is empty. The migration tests need a verison to run."
              exit 1
          fi
          echo "latest_version=${latest_version}" >> "$GITHUB_OUTPUT"
      - name: Name Versions
        run: |
          echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref || github.sha }}'"
      - name: Prepare Base64 TOML override
        uses: ./.github/actions/setup-create-base64-upgrade-config
        with:
          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
          chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }}
          upgradeImage: ${{ env.UPGRADE_IMAGE }}
          upgradeVersion: ${{ env.UPGRADE_VERSION }}
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          grafanaUrl: ${{ vars.GRAFANA_URL }}
          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
      - name: Run Migration Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_name: node-migration-test-logs
          artifacts_location: |
            ./integration-tests/migration/logs
            /tmp/gotest.log
          publish_check_name: Node Migration Test Results
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
          go_coverage_src_dir: /var/tmp/go-coverage
          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
          should_tidy: "false"
      - name: Upload Coverage Data
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        timeout-minutes: 2
        continue-on-error: true
        with:
          name: cl-node-coverage-data-migration-tests
          path: .covdata
          retention-days: 1
      - name: Notify Slack
        if: failure() && github.event_name != 'workflow_dispatch'
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        env:
          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
        with:
          channel-id: "#team-test-tooling-internal"
          slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"