Integration Tests #6505
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| run-name: Integration Tests ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }} | |
| on: | |
| merge_group: | |
| pull_request: | |
| push: | |
| tags: | |
| - "*" | |
| workflow_dispatch: | |
| inputs: | |
| cl_ref: | |
| description: 'The ref to checkout, defaults to the calling branch' | |
| required: false | |
| type: string | |
| evm-ref: | |
| description: 'The sha of the chainlink-evm commit to use if wanted' | |
| required: false | |
| type: string | |
| run_solana: | |
| description: 'Run solana tests' | |
| required: false | |
| type: string | |
| default: 'false' | |
| distinct_run_name: | |
| description: 'A unique identifier for this run, only use from other repos' | |
| required: false | |
| type: string | |
| # Only run 1 of this workflow at a time per PR | |
| concurrency: | |
| group: ${{ github.ref }}-${{ github.repository }}-${{ github.event_name }}--e2e-tests-${{ inputs.distinct_run_name }} | |
| cancel-in-progress: true | |
| env: | |
| # for run-test variables and environment | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ inputs.evm-ref || github.sha }} | |
| CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| TEST_SUITE: smoke | |
| TEST_ARGS: -test.timeout 12m | |
| INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
| MOD_CACHE_VERSION: 2 | |
| COLLECTION_ID: chainlink-e2e-tests | |
| jobs: | |
| enforce-ctf-version: | |
| name: Enforce CTF Version | |
| runs-on: ubuntu-latest | |
| # We don't directly merge dependabot PRs, so let's not waste the resources | |
| if: github.actor != 'dependabot[bot]' | |
| steps: | |
| - run: echo "${{github.event_name}}" | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref }} | |
| - name: Check Merge Group Condition | |
| id: condition-check | |
| run: | | |
| echo "Checking event condition..." | |
| SHOULD_ENFORCE="false" | |
| if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then | |
| echo "We are in a merge_group event, now check if we are on the develop branch" | |
| target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .merge_group.base_ref) | |
| if [[ "$target_branch" == "refs/heads/develop" ]]; then | |
| echo "We are on the develop branch, we should enforce ctf version" | |
| SHOULD_ENFORCE="true" | |
| fi | |
| fi | |
| echo "should we enforce ctf version = $SHOULD_ENFORCE" | |
| echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT | |
| - name: Enforce CTF Version | |
| if: steps.condition-check.outputs.should-enforce == 'true' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 | |
| with: | |
| go-project-path: ./integration-tests | |
| module-name: github.com/smartcontractkit/chainlink-testing-framework | |
| enforce-semantic-tag: "true" | |
| changes: | |
| environment: integration | |
| name: Check Paths That Require Tests To Run | |
| runs-on: ubuntu-latest | |
| # We don't directly merge dependabot PRs, so let's not waste the resources | |
| if: github.actor != 'dependabot[bot]' | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref }} | |
| - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 | |
| id: changes | |
| with: | |
| filters: | | |
| changes: | |
| - '**/*.go' | |
| - '**/*go.sum' | |
| - '**/*go.mod' | |
| - '.github/workflows/integration-tests.yml' | |
| - '**/*Dockerfile' | |
| - 'core/**/migrations/*.sql' | |
| - 'core/**/config/**/*.toml' | |
| - 'integration-tests/**/*.toml' | |
| - name: Ignore Filter On Workflow Dispatch | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| id: ignore-filter | |
| run: echo "changes=true" >> $GITHUB_OUTPUT | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-check-paths | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Check Paths That Require Tests To Run | |
| continue-on-error: true | |
| outputs: | |
| src: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.changes }} | |
| build-lint-integration-tests: | |
| name: Build and Lint ${{ matrix.project.name }} | |
| runs-on: ubuntu22.04-16cores-64GB | |
| # We don't directly merge dependabot PRs, so let's not waste the resources | |
| if: github.actor != 'dependabot[bot]' | |
| strategy: | |
| matrix: | |
| project: | |
| - name: integration-tests | |
| id: e2e | |
| path: ./integration-tests | |
| cache-id: e2e | |
| - name: load | |
| id: load | |
| path: ./integration-tests/load | |
| cache-id: load | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-build-lint-${{ matrix.project.id }} | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build and Lint ${{ matrix.project.name }} | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref }} | |
| - name: Setup Go | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 | |
| with: | |
| test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download | |
| go_mod_path: ${{ matrix.project.path }}/go.mod | |
| cache_key_id: core-${{ matrix.project.cache-id }}-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| - name: Build Go | |
| run: | | |
| cd ${{ matrix.project.path }} | |
| go build ./... | |
| go test -run=^# ./... | |
| - name: Lint Go | |
| uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0 | |
| with: | |
| version: v1.59.1 | |
| # We already cache these directories in setup-go | |
| skip-pkg-cache: true | |
| skip-build-cache: true | |
| # only-new-issues is only applicable to PRs, otherwise it is always set to false | |
| only-new-issues: false # disabled for PRs due to unreliability | |
| args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml | |
| working-directory: ${{ matrix.project.path }} | |
| build-chainlink: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: | |
| image: | |
| - name: "" | |
| dockerfile: core/chainlink.Dockerfile | |
| tag-suffix: "" | |
| - name: (plugins) | |
| dockerfile: plugins/chainlink.Dockerfile | |
| tag-suffix: -plugins | |
| name: Build Chainlink Image ${{ matrix.image.name }} | |
| runs-on: ubuntu22.04-16cores-64GB | |
| needs: [changes, enforce-ctf-version] | |
| steps: | |
| - name: Collect Metrics | |
| if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-build-chainlink | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Chainlink Image ${{ matrix.image.name }} | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| - name: Build Chainlink Image | |
| if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' | |
| uses: ./.github/actions/build-chainlink-image | |
| with: | |
| tag_suffix: ${{ matrix.image.tag-suffix }} | |
| dockerfile: ${{ matrix.image.dockerfile }} | |
| git_commit_sha: ${{ inputs.evm-ref || github.sha }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| dep_evm_sha: ${{ inputs.evm-ref }} | |
| build-test-image: | |
| if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image') | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu22.04-16cores-64GB | |
| needs: [changes] | |
| steps: | |
| - name: Collect Metrics | |
| if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-build-test-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| - name: Build Go Test Command | |
| id: build-go-test-command | |
| run: | | |
| # if the matrix.product.run is set, use it for a different command | |
| if [ "${{ matrix.product.run }}" != "" ]; then | |
| echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Setup GAP for Grafana | |
| uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected] | |
| with: | |
| # aws inputs | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} | |
| api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} | |
| # other inputs | |
| duplicate-authorization-header: "true" | |
| - name: Prepare Base64 TOML override | |
| uses: ./.github/actions/setup-create-base64-config | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| selectedNetworks: ${{ env.SELECTED_NETWORKS }} | |
| chainlinkImage: ${{ env.CHAINLINK_IMAGE }} | |
| chainlinkVersion: ${{ inputs.evm-ref || github.sha }} | |
| pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 | |
| pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} | |
| pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} | |
| lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push | |
| lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| grafanaUrl: "http://localhost:8080/primary" # This is GAP's address | |
| grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
| eth-smoke-tests-matrix: | |
| if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} | |
| environment: integration | |
| permissions: | |
| actions: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| needs: [build-chainlink, changes, build-lint-integration-tests] | |
| env: | |
| SELECTED_NETWORKS: SIMULATED | |
| CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| TEST_LOG_LEVEL: debug | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| product: | |
| - name: ccip-lm-smoke | |
| nodes: 1 | |
| os: ubuntu-latest | |
| file: lm | |
| dir: ccip-tests/smoke | |
| run: -run ^TestLmBasic$ | |
| - name: ccip-smoke | |
| nodes: 1 | |
| os: ubuntu-latest | |
| file: ccip | |
| dir: ccip-tests/smoke | |
| run: -run ^TestSmokeCCIPForBidirectionalLane$ | |
| - name: ccip-smoke-1.4-pools | |
| nodes: 1 | |
| os: ubuntu-latest | |
| file: ccip | |
| dir: ccip-tests/smoke | |
| run: -run ^TestSmokeCCIPForBidirectionalLane$ | |
| config_path: ./integration-tests/ccip-tests/testconfig/tomls/contract-version1.4.toml | |
| - name: ccip-smoke-usdc | |
| nodes: 1 | |
| os: ubuntu-latest | |
| file: ccip | |
| dir: ccip-tests/smoke | |
| run: -run ^TestSmokeCCIPForBidirectionalLane$ | |
| config_path: ./integration-tests/ccip-tests/testconfig/tomls/usdc_mock_deployment.toml | |
| - name: ccip-smoke-db-compatibility | |
| nodes: 1 | |
| os: ubuntu-latest | |
| file: ccip | |
| dir: ccip-tests/smoke | |
| run: -run ^TestSmokeCCIPForBidirectionalLane$ | |
| config_path: ./integration-tests/ccip-tests/testconfig/tomls/db-compatibility.toml | |
| - name: ccip-smoke-rate-limit | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPRateLimit$ | |
| - name: ccip-smoke-rate-limit | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPTokenPoolRateLimits$ | |
| - name: ccip-smoke-multicall | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPMulticall$ | |
| - name: ccip-smoke-manual-exec | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$ | |
| - name: ccip-smoke-on-ramp-limits | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPOnRampLimits$ | |
| - name: ccip-smoke-off-ramp-capacity | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPOffRampCapacityLimit$ | |
| - name: ccip-smoke-off-ramp-agg-rate-limit | |
| nodes: 1 | |
| dir: ccip-tests/smoke | |
| os: ubuntu-latest | |
| file: ccip | |
| run: -run ^TestSmokeCCIPOffRampAggRateLimit$ | |
| - name: runlog | |
| id: runlog | |
| nodes: 2 | |
| os: ubuntu-latest | |
| - name: cron | |
| id: cron | |
| nodes: 2 | |
| os: ubuntu-latest | |
| - name: flux | |
| id: flux | |
| nodes: 1 | |
| os: ubuntu-latest | |
| - name: ocr | |
| id: ocr | |
| nodes: 2 | |
| os: ubuntu-latest | |
| file: ocr | |
| pyroscope_env: ci-smoke-ocr-evm-simulated | |
| - name: reorg_above_finality | |
| id: reorg_above_finality | |
| nodes: 1 | |
| os: ubuntu-latest | |
| file: reorg_above_finality | |
| pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated | |
| - name: ocr2 | |
| id: ocr2 | |
| nodes: 6 | |
| os: ubuntu22.04-16cores-64GB | |
| file: ocr2 | |
| - name: ocr2 | |
| id: ocr2-plugins | |
| nodes: 6 | |
| os: ubuntu22.04-16cores-64GB | |
| tag_suffix: "-plugins" | |
| - name: vrf | |
| id: vrf | |
| nodes: 2 | |
| os: ubuntu-latest | |
| - name: vrfv2 | |
| id: vrfv2 | |
| nodes: 6 | |
| os: ubuntu-latest | |
| - name: vrfv2plus | |
| id: vrfv2plus | |
| nodes: 9 | |
| os: ubuntu-latest | |
| - name: forwarder_ocr | |
| id: forwarder_ocr | |
| nodes: 2 | |
| os: ubuntu-latest | |
| - name: forwarders_ocr2 | |
| id: forwarders_ocr2 | |
| nodes: 2 | |
| os: ubuntu-latest | |
| runs-on: ${{ matrix.product.os }} | |
| name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} | |
| steps: | |
| # Handy for debugging resource usage | |
| # - name: Collect Workflow Telemetry | |
| # uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0 | |
| - name: Collect Metrics | |
| if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }} | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} | |
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| - name: Build Go Test Command | |
| id: build-go-test-command | |
| run: | | |
| # if dir is provided use it, otherwise use the smoke dir | |
| if [ "${{ matrix.product.dir }}" != "" ]; then | |
| dir=${{ matrix.product.dir }} | |
| else | |
| dir=smoke | |
| fi | |
| # if the matrix.product.run is set, use it for a different command | |
| if [ "${{ matrix.product.run }}" != "" ]; then | |
| echo "run_command=${{ matrix.product.run }} ./${dir}/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_command=./${dir}/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Check for "enable tracing" label | |
| id: check-label | |
| run: | | |
| label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH") | |
| if [[ -n "$label" ]]; then | |
| if [[ "$label" == "enable tracing" ]]; then | |
| echo "Enable tracing label found." | |
| echo "trace=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Enable tracing label not found." | |
| echo "trace=false" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "No labels present or labels are null." | |
| echo "trace=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Setup Grafana and OpenTelemetry | |
| id: docker-setup | |
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | |
| run: | | |
| # Create network | |
| docker network create --driver bridge tracing | |
| # Make trace directory | |
| cd integration-tests/smoke/ | |
| mkdir ./traces | |
| chmod -R 777 ./traces | |
| # Switch directory | |
| cd ../../.github/tracing | |
| # Create a Docker volume for traces | |
| # docker volume create otel-traces | |
| # Start OpenTelemetry Collector | |
| # Note the user must be set to the same user as the runner for the trace data to be accessible | |
| docker run -d --network=tracing --name=otel-collector \ | |
| -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ | |
| -v $PWD/../../integration-tests/smoke/traces:/tracing \ | |
| --user "$(id -u):$(id -g)" \ | |
| -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml | |
| - name: Locate Docker Volume | |
| id: locate-volume | |
| if: false | |
| run: | | |
| echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT | |
| - name: Show Otel-Collector Logs | |
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | |
| run: | | |
| docker logs otel-collector | |
| - name: Set Override Config | |
| id: set_override_config | |
| run: | | |
| # if the matrix.product.config_path is set, use it as the override config | |
| if [ "${{ matrix.product.config_path }}" != "" ]; then | |
| echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Setup GAP for Grafana | |
| uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected] | |
| with: | |
| # aws inputs | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} | |
| api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} | |
| # other inputs | |
| duplicate-authorization-header: "true" | |
| # metrics inputs | |
| metrics-job-name: "grafana" | |
| gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
| gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| - name: Prepare Base64 TOML override | |
| uses: ./.github/actions/setup-create-base64-config | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| selectedNetworks: ${{ env.SELECTED_NETWORKS }} | |
| chainlinkImage: ${{ env.CHAINLINK_IMAGE }} | |
| chainlinkVersion: ${{ inputs.evm-ref || github.sha }} | |
| pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 | |
| pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} | |
| pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} | |
| lokiEndpoint: ${{ secrets.LOKI_URL_CI }} | |
| lokiTenantId: ${{ vars.LOKI_TENANT_ID }} | |
| lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| grafanaUrl: "http://localhost:8080/primary" # this is GAP url, it will know where to forward the requests so that they reach the correct Grafana instance | |
| grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
| - name: Prepare Base64 TOML override for CCIP secrets | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| selectedNetworks: SIMULATED_1,SIMULATED_2 | |
| chainlinkImage: ${{ env.CHAINLINK_IMAGE }} | |
| chainlinkVersion: ${{ github.sha }} | |
| lokiEndpoint: ${{ secrets.LOKI_URL_CI }} | |
| lokiTenantId: ${{ vars.LOKI_TENANT_ID }} | |
| lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| grafanaUrl: "http://localhost:8080/primary" | |
| grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
| ## Run this step when changes that require tests to be run are made | |
| - name: Run Tests | |
| if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 | |
| env: | |
| BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }} | |
| TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }} | |
| with: | |
| test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| cl_repo: ${{ env.CHAINLINK_IMAGE }} | |
| cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs | |
| artifacts_location: | | |
| ./integration-tests/smoke/logs/ | |
| ./integration-tests/ccip-tests/smoke/logs/* | |
| /tmp/gotest.log | |
| publish_check_name: ${{ matrix.product.name }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: "" | |
| should_tidy: "false" | |
| go_coverage_src_dir: /var/tmp/go-coverage | |
| go_coverage_dest_dir: ${{ github.workspace }}/.covdata | |
| - name: Upload Coverage Data | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 | |
| timeout-minutes: 2 | |
| continue-on-error: true | |
| with: | |
| name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }} | |
| path: .covdata | |
| retention-days: 1 | |
| # Run this step when changes that do not need the test to run are made | |
| - name: Run Setup | |
| if: needs.changes.outputs.src == 'false' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 | |
| with: | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| go_mod_path: ./integration-tests/go.mod | |
| cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: "" | |
| should_tidy: "false" | |
| - name: Show Otel-Collector Logs | |
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | |
| run: | | |
| docker logs otel-collector | |
| - name: Permissions on traces | |
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | |
| run: | | |
| ls -l ./integration-tests/smoke/traces | |
| - name: Upload Trace Data | |
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| with: | |
| name: trace-data | |
| path: ./integration-tests/smoke/traces/trace-data.json | |
| - name: Print failed test summary | |
| if: always() | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@1587f59bfd626b668d303abbc90fee41b12397e6 # v2.3.23 | |
| with: | |
| test_directories: ./integration-tests/smoke/,./integration-tests/ccip-tests/smoke/ | |
| ### Used to check the required checks box when the matrix completes | |
| eth-smoke-tests: | |
| if: always() | |
| runs-on: ubuntu-latest | |
| name: ETH Smoke Tests | |
| needs: [eth-smoke-tests-matrix] | |
| steps: | |
| - name: Check smoke test matrix status | |
| if: needs.eth-smoke-tests-matrix.result != 'success' | |
| run: | | |
| echo "${{ needs.eth-smoke-tests-matrix.result }}" | |
| exit 1 | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-matrix-results | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: ETH Smoke Tests | |
| matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }} | |
| continue-on-error: true | |
| cleanup: | |
| name: Clean up integration environment deployments | |
| if: always() | |
| needs: [eth-smoke-tests] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref }} | |
| - name: 🧼 Clean up Environment | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/delete-deployments | |
| with: | |
| environment: integration | |
| ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663 | |
| - name: Collect Metrics | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-env-cleanup | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Clean up integration environment deployments | |
| continue-on-error: true | |
| show-coverage: | |
| name: Show Chainlink Node Go Coverage | |
| if: always() | |
| needs: [cleanup] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6 | |
| with: | |
| path: cl-node-coverage-data | |
| pattern: cl-node-coverage-data-* | |
| merge-multiple: true | |
| - name: Show Coverage | |
| run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged" | |
| # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss | |
| # this will also only run if both of the matrix jobs pass | |
| eth-smoke-go-mod-cache: | |
| environment: integration | |
| needs: [eth-smoke-tests] | |
| runs-on: ubuntu-latest | |
| name: ETH Smoke Tests Go Mod Cache | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| - name: Run Setup | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 | |
| with: | |
| test_download_vendor_packages_command: | | |
| cd ./integration-tests | |
| go mod download | |
| # force download of test dependencies | |
| go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure" | |
| go_mod_path: ./integration-tests/go.mod | |
| cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "false" | |
| ### Migration tests | |
| node-migration-tests: | |
| name: Version Migration Tests | |
| environment: integration | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest | |
| needs: [build-chainlink, changes] | |
| # Only run migration tests on new tags | |
| if: startsWith(github.ref, 'refs/tags/') | |
| env: | |
| SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 | |
| CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| CHAINLINK_IMAGE: public.ecr.aws/w0i8p0z9/chainlink-ccip | |
| UPGRADE_VERSION: ${{ inputs.evm-ref || github.sha }} | |
| UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| TEST_LOG_LEVEL: debug | |
| TEST_SUITE: migration | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: ${{ env.COLLECTION_ID }}-migration-tests | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Version Migration Tests | |
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: smartcontractkit/ccip | |
| ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| - name: Get Latest Version | |
| id: get_latest_version | |
| run: | | |
| untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name) | |
| latest_version="${untrimmed_ver:1}" | |
| # Check if latest_version is empty | |
| if [ -z "$latest_version" ]; then | |
| echo "Error: The latest_version is empty. The migration tests need a verison to run." | |
| exit 1 | |
| fi | |
| echo "latest_version=${latest_version}" >> "$GITHUB_OUTPUT" | |
| - name: Name Versions | |
| run: | | |
| echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref || github.sha }}'" | |
| - name: Prepare Base64 TOML override | |
| uses: ./.github/actions/setup-create-base64-upgrade-config | |
| with: | |
| selectedNetworks: ${{ env.SELECTED_NETWORKS }} | |
| chainlinkImage: ${{ env.CHAINLINK_IMAGE }} | |
| chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }} | |
| upgradeImage: ${{ env.UPGRADE_IMAGE }} | |
| upgradeVersion: ${{ env.UPGRADE_VERSION }} | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push | |
| lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| grafanaUrl: ${{ vars.GRAFANA_URL }} | |
| grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| - name: Run Migration Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 | |
| with: | |
| test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| cl_repo: ${{ env.CHAINLINK_IMAGE }} | |
| cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }} | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| artifacts_name: node-migration-test-logs | |
| artifacts_location: | | |
| ./integration-tests/migration/logs | |
| /tmp/gotest.log | |
| publish_check_name: Node Migration Test Results | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: "" | |
| go_coverage_src_dir: /var/tmp/go-coverage | |
| go_coverage_dest_dir: ${{ github.workspace }}/.covdata | |
| should_tidy: "false" | |
| - name: Upload Coverage Data | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 | |
| timeout-minutes: 2 | |
| continue-on-error: true | |
| with: | |
| name: cl-node-coverage-data-migration-tests | |
| path: .covdata | |
| retention-days: 1 | |
| - name: Notify Slack | |
| if: failure() && github.event_name != 'workflow_dispatch' | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} | |
| with: | |
| channel-id: "#team-test-tooling-internal" | |
| slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" |