client-compatibility-tests.yml

name: CCIP Client Compatibility Tests
on:
  merge_group:
  pull_request:
  push:
    tags:
      - "*"
  workflow_dispatch:

env:
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
  MOD_CACHE_VERSION: 2

jobs:
  # Build Test Dependencies

  changes:
    environment: integration
    name: Check Paths That Require Tests To Run
    runs-on: ubuntu-latest
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/ccip
          ref: ${{ inputs.cl_ref }}
      - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
        id: changes
        with:
          filters: |
            src:
              - '**/*.go'
              - '**/*go.sum'
              - '**/*go.mod'
              - '.github/workflows/integration-tests.yml'
              - '**/*Dockerfile'
              - 'core/**/config/**/*.toml'
              - 'integration-tests/**/*.toml'
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
        with:
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Check Paths That Require Tests To Run
        continue-on-error: true
    outputs:
      src: ${{ inputs.set_changes_output || steps.changes.outputs.src }}

  build-chainlink:
    if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
    needs: [ changes ]
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Chainlink Image
    runs-on: ubuntu-latest
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
        with:
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Chainlink Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/ccip
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Chainlink Image
        uses: ./.github/actions/build-chainlink-image
        with:
          tag_suffix: ""
          dockerfile: core/chainlink.Dockerfile
          git_commit_sha: ${{ github.sha }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  get-latest-available-images:
    if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
    needs: [ changes ]
    environment: integration
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    outputs:
      geth_tag: ${{ env.GETH_TAG }}
      nethermind_tag: ${{ env.NETHERMIND_TAG }}
      besu_tag: ${{ env.BESU_TAG }}
      erigon_tag: ${{ env.ERIGON_TAG }}
    steps:
      # Setup AWS creds
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
        with:
          aws-region: ${{ secrets.QA_AWS_REGION }}
          role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 3600
      # Login to ECR
      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
        with:
          mask-password: 'true'
        env:
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      - name: Get latest docker images from ECR
        run: |
          geth_tag=$(aws ecr describe-images --repository-name ethereum/client-go --region ${{ secrets.QA_AWS_REGION }} --query 'sort_by(imageDetails,& imageTags[0])[-10:]' --output json | jq -r '[.[] | .imageTags[] | {original: ., numeric: (ltrimstr("v") | (split(".") + ["0","0"])[:3] | map(tonumber) | .[0]*10000 + .[1]*100 + .[2])}] | max_by(.numeric) | .original')
          echo "GETH_TAG=$geth_tag" >> $GITHUB_ENV
          echo "Geth latest tag: $geth_tag"
          nethermind_tag=$(aws ecr describe-images --repository-name nethermind/nethermind --region ${{ secrets.QA_AWS_REGION }} --query 'sort_by(imageDetails,& imageTags[0])[-10:]' --output json | jq -r '[.[] | .imageTags[] | {original: ., numeric: (ltrimstr("v") | (split(".") + ["0","0"])[:3] | map(tonumber) | .[0]*10000 + .[1]*100 + .[2])}] | max_by(.numeric) | .original')
          echo "NETHERMIND_TAG=$nethermind_tag" >> $GITHUB_ENV
          echo "Nethermind latest tag: $nethermind_tag"
          besu_tag=$(aws ecr describe-images --repository-name hyperledger/besu --region ${{ secrets.QA_AWS_REGION }} --query 'sort_by(imageDetails,& imageTags[0])[-10:]' --output json | jq -r '[.[] | .imageTags[] | {original: ., numeric: (ltrimstr("v") | (split(".") + ["0","0"])[:3] | map(tonumber) | .[0]*10000 + .[1]*100 + .[2])}] | max_by(.numeric) | .original')
          echo "BESU_TAG=$besu_tag" >> $GITHUB_ENV
          echo "Besu latest tag: $besu_tag"
          erigon_tag=$(aws ecr describe-images --repository-name thorax/erigon --region ${{ secrets.QA_AWS_REGION }} --query 'sort_by(imageDetails,& imageTags[0])[-10:]' --output json |jq -r '[.[] | .imageTags[] | {original: ., numeric: (ltrimstr("v") | (split(".") + ["0","0"])[:3] | map(tonumber) | .[0]*10000 + .[1]*100 + .[2])}] | max_by(.numeric) | .original')
          echo "ERIGON_TAG=$erigon_tag" >> $GITHUB_ENV
          echo "Erigon latest tag: $erigon_tag"

  # End Build Test Dependencies

  evm-compatibility-matrix:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [ build-chainlink, changes, get-latest-available-images ]
    if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
    env:
      SELECTED_NETWORKS: SIMULATED_1,SIMULATED_2
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: bidirectionallane-geth
            os: ubuntu-latest
            test: TestSmokeCCIPForBidirectionalLane
            client: geth
            timeout: 30m
            pyroscope_env: ci-ccip-bidirectionallane-geth
            chainConfig: "1337=ethereum/client-go:${{ needs.get-latest-available-images.outputs.geth_tag }},2337=ethereum/client-go:${{ needs.get-latest-available-images.outputs.geth_tag }}"
          # uncomment when nethermind flake reason is addressed
          # - name: bidirectionallane-nethermind
          #            test: TestSmokeCCIPForBidirectionalLane
          #            client: nethermind
          #            timeout: 30m
          #            pyroscope_env: ci-ccip-bidirectionallane-nethermind
          #         chainConfig: "1337=nethermind/nethermind:${{ needs.get-latest-available-images.outputs.nethermind_tag }},2337=nethermind/nethermind:${{ needs.get-latest-available-images.outputs.nethermind_tag }}"
          # uncomment when 24.4.0  is released with our data/input fix
          # - name: bidirectionallane-besu
          #   test: TestSmokeCCIPForBidirectionalLane
          #   client: besu
          #   timeout: 30m
          #   pyroscope_env: ci-ccip-bidirectionallane-besu
          #   chainConfig: "1337=hyperledger/besu:${{ needs.get-latest-available-images.outputs.besu_tag }},2337=hyperledger/besu:${{ needs.get-latest-available-images.outputs.besu_tag }}"
          - name: bidirectionallane-erigon
            test: TestSmokeCCIPForBidirectionalLane
            client: erigon
            timeout: 30m
            pyroscope_env: ci-ccip-bidirectionallane-erigon
            chainConfig: "1337=thorax/erigon:${{ needs.get-latest-available-images.outputs.erigon_tag }},2337=thorax/erigon:${{ needs.get-latest-available-images.outputs.erigon_tag }}"
    runs-on: ubuntu-latest
    name: CCIP Latest EVM Node Compatibility Test with ${{ matrix.client }}
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: smartcontractkit/ccip
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Go Test Command
        id: build-go-test-command
        run: |
          # if the matrix.test is set, use it for a different command
          if [ "${{ matrix.test }}" != "" ]; then
            echo "run_command=-run '^${{ matrix.test }}$' ./ccip-tests/smoke/ccip_test.go" >> "$GITHUB_OUTPUT"
          else
            echo "run_command=./ccip-tests/smoke/ccip_test.go" >> "$GITHUB_OUTPUT"
          fi
      - name: Prepare Base64 TOML override
        uses: ./.github/actions/setup-create-base64-config
        with:
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
          chainlinkVersion: ${{ github.sha }}
          pyroscopeServer: ${{ matrix.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          pyroscopeEnvironment: ${{ matrix.pyroscope_env }}
          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
          lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          grafanaUrl: ${{ vars.GRAFANA_URL }}
          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
      - name: Prepare Base64 TOML override for CCIP secrets
        uses: ./.github/actions/setup-create-base64-config-ccip
        with:
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          selectedNetworks: SIMULATED_1,SIMULATED_2
          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
          chainlinkVersion: ${{ github.sha }}
          lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          grafanaUrl: ${{ vars.GRAFANA_URL }}
          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
          customEvmNodes: ${{ matrix.chainConfig }}
      - name: Run Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=2 ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ github.sha }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_name: ${{ matrix.client }}-test-logs
          artifacts_location: |
            ./integration-tests/smoke/logs/
            ./integration-tests/ccip-tests/smoke/logs/*
          publish_check_name: ${{ matrix.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
          should_tidy: "false"
      - name: Print failed test summary
        if: always()
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6
        with:
          test_directory: ./integration-tests/smoke/

  start-slack-thread:
    name: Start Slack Thread
    if: ${{ always() && (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
    environment: integration
    outputs:
      thread_ts: ${{ steps.slack.outputs.thread_ts }}
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [ evm-compatibility-matrix, changes ]
    steps:
      - name: Debug Result
        run: echo ${{ join(needs.*.result, ',') }}
      - name: Main Slack Notification
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        id: slack
        with:
          channel-id: ${{ secrets.QA_SLACK_CHANNEL }}
          payload: |
            {
              "attachments": [
                {
                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
                  "blocks": [
                    {
                      "type": "header",
                      "text": {
                        "type": "plain_text",
                        "text": "CCIP EVM Node Compatability Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
                        "emoji": true
                      }
                    },
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U060CGGPY8H> & <@U03GWA0NSF7>' || 'All Good!' }}"
                      }
                    },
                    {
                      "type": "divider"
                    },
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
                      }
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

  post-test-results-to-slack:
    name: Post Test Results for ${{matrix.client}}
    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [ start-slack-thread ]
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Post Test Results to Slack
        uses: ./.github/actions/notify-slack-jobs-result
        with:
          github_token: ${{ github.token }}
          github_repository: ${{ github.repository }}
          workflow_run_id: ${{ github.run_id }}
          github_job_name_regex: ^CCIP Latest EVM Node Compatibility Test with (?<cap>.*?)$
          message_title: BiDirectional Lane Test Results
          slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }}
          slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }}
          slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }}