Integration Tests #9162

Workflow file for this run

.github/workflows/integration-tests.yml at 9328fed

	name: Integration Tests
	run-name: Integration Tests ${{ inputs.distinct_run_name && inputs.distinct_run_name \|\| '' }}
	on:
	merge_group:
	pull_request:
	push:
	tags:
	- "*"
	workflow_dispatch:
	inputs:
	cl_ref:
	description: 'The ref to checkout, defaults to the calling branch'
	required: false
	type: string
	evm-ref:
	description: 'The sha of the chainlink-evm commit to use if wanted'
	required: false
	type: string
	distinct_run_name:
	description: 'A unique identifier for this run, only use from other repos'
	required: false
	type: string

	# Only run 1 of this workflow at a time per PR
	concurrency:
	group: ${{ github.ref }}-${{ github.repository }}-${{ github.event_name }}--e2e-tests-${{ inputs.distinct_run_name }}
	cancel-in-progress: true

	env:
	# for run-test variables and environment
	ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ inputs.evm-ref \|\| github.sha }}
	CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
	TEST_SUITE: smoke
	TEST_ARGS: -test.timeout 12m
	INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
	MOD_CACHE_VERSION: 2
	COLLECTION_ID: chainlink-e2e-tests

	jobs:
	enforce-ctf-version:
	name: Enforce CTF Version
	runs-on: ubuntu-latest
	# We don't directly merge dependabot PRs, so let's not waste the resources
	if: github.actor != 'dependabot[bot]'
	steps:
	- run: echo "${{github.event_name}}"
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref }}
	- name: Check Merge Group Condition
	id: condition-check
	run: \|
	echo "Checking event condition..."
	SHOULD_ENFORCE="false"
	if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
	echo "We are in a merge_group event, now check if we are on the develop branch"
	target_branch=$(cat $GITHUB_EVENT_PATH \| jq -r .merge_group.base_ref)
	if [[ "$target_branch" == "refs/heads/develop" ]]; then
	echo "We are on the develop branch, we should enforce ctf version"
	SHOULD_ENFORCE="true"
	fi
	fi
	echo "should we enforce ctf version = $SHOULD_ENFORCE"
	echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT
	- name: Enforce CTF Version
	if: steps.condition-check.outputs.should-enforce == 'true'
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
	with:
	go-project-path: ./integration-tests
	module-name: github.com/smartcontractkit/chainlink-testing-framework
	enforce-semantic-tag: "true"

	changes:
	environment: integration
	name: Check Paths That Require Tests To Run
	runs-on: ubuntu-latest
	# We don't directly merge dependabot PRs, so let's not waste the resources
	if: github.actor != 'dependabot[bot]'
	steps:
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref }}
	- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
	id: changes
	with:
	filters: \|
	changes:
	- '*/.go'
	- '*/go.sum'
	- '*/go.mod'
	- '.github/workflows/integration-tests.yml'
	- '*/Dockerfile'
	- 'core/*/migrations/.sql'
	- 'core//config//*.toml'
	- 'integration-tests/*/.toml'
	- name: Ignore Filter On Workflow Dispatch
	if: ${{ github.event_name == 'workflow_dispatch' }}
	id: ignore-filter
	run: echo "changes=true" >> $GITHUB_OUTPUT
	- name: Collect Metrics
	if: always()
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-check-paths
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: Check Paths That Require Tests To Run
	continue-on-error: true
	outputs:
	src: ${{ steps.ignore-filter.outputs.changes \|\| steps.changes.outputs.changes }}

	build-lint-integration-tests:
	name: Build and Lint ${{ matrix.project.name }}
	runs-on: ubuntu22.04-8cores-32GB
	# We don't directly merge dependabot PRs, so let's not waste the resources
	if: github.actor != 'dependabot[bot]'
	strategy:
	matrix:
	project:
	- name: integration-tests
	id: e2e
	path: ./integration-tests
	cache-id: e2e
	- name: load
	id: load
	path: ./integration-tests/load
	cache-id: load
	steps:
	- name: Collect Metrics
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-build-lint-${{ matrix.project.id }}
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: Build and Lint ${{ matrix.project.name }}
	continue-on-error: true
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref }}
	- name: Setup Go
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
	with:
	test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download
	go_mod_path: ${{ matrix.project.path }}/go.mod
	cache_key_id: core-${{ matrix.project.cache-id }}-${{ env.MOD_CACHE_VERSION }}
	cache_restore_only: "true"
	- name: Build Go
	run: \|
	cd ${{ matrix.project.path }}
	go build ./...
	go test -run=^# ./...
	- name: Lint Go
	uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
	with:
	version: v1.59.1
	# We already cache these directories in setup-go
	skip-pkg-cache: true
	skip-build-cache: true
	# only-new-issues is only applicable to PRs, otherwise it is always set to false
	only-new-issues: false # disabled for PRs due to unreliability
	args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml
	working-directory: ${{ matrix.project.path }}
	continue-on-error: true

	build-chainlink:
	environment: integration
	permissions:
	id-token: write
	contents: read
	strategy:
	matrix:
	image:
	- name: ""
	dockerfile: core/chainlink.Dockerfile
	tag-suffix: ""
	- name: (plugins)
	dockerfile: plugins/chainlink.Dockerfile
	tag-suffix: -plugins
	name: Build Chainlink Image ${{ matrix.image.name }}
	runs-on: ubuntu22.04-8cores-32GB
	needs: [changes, enforce-ctf-version]
	steps:
	- name: Collect Metrics
	if: needs.changes.outputs.src == 'true' \|\| github.event_name == 'workflow_dispatch'
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-build-chainlink
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: Build Chainlink Image ${{ matrix.image.name }}
	continue-on-error: true
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref \|\| github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}
	- name: Setup Github Token
	if: ${{ inputs.evm-ref }}
	id: get-gh-token
	uses: smartcontractkit/.github/actions/setup-github-token@ef78fa97bf3c77de6563db1175422703e9e6674f # [email protected]
	with:
	aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
	aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
	aws-region: ${{ secrets.AWS_REGION }}
	set-git-config: "true"
	- name: Build Chainlink Image
	if: needs.changes.outputs.src == 'true' \|\| github.event_name == 'workflow_dispatch'
	uses: ./.github/actions/build-chainlink-image
	with:
	tag_suffix: ${{ matrix.image.tag-suffix }}
	dockerfile: ${{ matrix.image.dockerfile }}
	git_commit_sha: ${{ inputs.evm-ref \|\| github.sha }}
	AWS_REGION: ${{ secrets.QA_AWS_REGION }}
	AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
	dep_evm_sha: ${{ inputs.evm-ref }}

	build-test-image:
	if: startsWith(github.ref, 'refs/tags/') \|\| github.event_name == 'schedule' \|\| contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
	environment: integration
	permissions:
	id-token: write
	contents: read
	name: Build Test Image
	runs-on: ubuntu22.04-16cores-64GB
	needs: [changes]
	steps:
	- name: Collect Metrics
	if: needs.changes.outputs.src == 'true' \|\| github.event_name == 'workflow_dispatch'
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-build-test-image
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: Build Test Image
	continue-on-error: true
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref \|\| github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}
	- name: Build Go Test Command
	id: build-go-test-command
	run: \|
	# if the matrix.product.run is set, use it for a different command
	if [ "${{ matrix.product.run }}" != "" ]; then
	echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
	else
	echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
	fi
	- name: Setup GAP for Grafana
	uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
	id: setup-gap
	with:
	# aws inputs
	aws-region: ${{ secrets.AWS_REGION }}
	aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
	api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
	# other inputs
	duplicate-authorization-header: "true"

	eth-smoke-tests-matrix:
	if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
	environment: integration
	permissions:
	actions: read
	checks: write
	pull-requests: write
	id-token: write
	contents: read
	needs: [build-chainlink, changes, build-lint-integration-tests]
	env:
	SELECTED_NETWORKS: SIMULATED
	CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref \|\| github.sha }}
	CHAINLINK_ENV_USER: ${{ github.actor }}
	TEST_LOG_LEVEL: debug
	strategy:
	fail-fast: false
	matrix:
	product:
	# LM Smoke Test is disabled since project is paused
	# - name: ccip-lm-smoke
	# nodes: 1
	# os: ubuntu-latest
	# file: lm
	# dir: ccip-tests/smoke
	# run: -run ^TestLmBasic$
	- name: ccip-smoke
	nodes: 1
	os: ubuntu-latest
	file: ccip
	dir: ccip-tests/smoke
	run: -run ^TestSmokeCCIPForBidirectionalLane$
	- name: ccip-smoke-1.4-pools
	nodes: 1
	os: ubuntu-latest
	file: ccip
	dir: ccip-tests/smoke
	run: -run ^TestSmokeCCIPForBidirectionalLane$
	config_path: ./integration-tests/ccip-tests/testconfig/tomls/contract-version1.4.toml
	- name: ccip-smoke-usdc
	nodes: 1
	os: ubuntu-latest
	file: ccip
	dir: ccip-tests/smoke
	run: -run ^TestSmokeCCIPForBidirectionalLane$
	config_path: ./integration-tests/ccip-tests/testconfig/tomls/usdc_mock_deployment.toml
	- name: ccip-smoke-db-compatibility
	nodes: 1
	os: ubuntu-latest
	file: ccip
	dir: ccip-tests/smoke
	run: -run ^TestSmokeCCIPForBidirectionalLane$
	config_path: ./integration-tests/ccip-tests/testconfig/tomls/db-compatibility.toml
	- name: ccip-smoke-rate-limit
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPRateLimit$
	- name: ccip-smoke-rate-limit
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPTokenPoolRateLimits$
	- name: ccip-smoke-multicall
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPMulticall$
	- name: ccip-smoke-manual-exec
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$
	- name: ccip-smoke-on-ramp-limits
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPOnRampLimits$
	- name: ccip-smoke-off-ramp-capacity
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPOffRampCapacityLimit$
	- name: ccip-smoke-off-ramp-agg-rate-limit
	nodes: 1
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPOffRampAggRateLimit$
	- name: ccip-smoke-leader-lane
	nodes: 15
	dir: ccip-tests/smoke
	os: ubuntu-latest
	file: ccip
	run: -run ^TestSmokeCCIPForBidirectionalLane$
	config_path: ./integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml
	- name: runlog
	id: runlog
	nodes: 2
	os: ubuntu-latest
	pyroscope_env: "ci-smoke-runlog-evm-simulated"
	- name: cron
	id: cron
	nodes: 2
	os: ubuntu-latest
	pyroscope_env: "ci-smoke-cron-evm-simulated"
	- name: flux
	id: flux
	nodes: 1
	os: ubuntu-latest
	pyroscope_env: "ci-smoke-flux-evm-simulated"
	- name: ocr
	id: ocr
	nodes: 2
	os: ubuntu-latest
	file: ocr
	pyroscope_env: ci-smoke-ocr-evm-simulated
	- name: reorg_above_finality
	id: reorg_above_finality
	nodes: 1
	os: ubuntu-latest
	file: reorg_above_finality
	pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated
	- name: ocr2
	id: ocr2
	nodes: 6
	os: ubuntu22.04-16cores-64GB
	file: ocr2
	pyroscope_env: ci-smoke-ocr2-evm-simulated
	- name: ocr2
	id: ocr2-plugins
	nodes: 6
	os: ubuntu22.04-16cores-64GB
	pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated
	tag_suffix: "-plugins"
	- name: vrf
	id: vrf
	nodes: 2
	os: ubuntu-latest
	pyroscope_env: ci-smoke-vrf-evm-simulated
	- name: vrfv2
	id: vrfv2
	nodes: 6
	os: ubuntu-latest
	pyroscope_env: ci-smoke-vrf2-evm-simulated
	- name: vrfv2plus
	id: vrfv2plus
	nodes: 9
	os: ubuntu-latest
	pyroscope_env: ci-smoke-vrf2plus-evm-simulated
	- name: forwarder_ocr
	id: forwarder_ocr
	nodes: 2
	os: ubuntu-latest
	pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
	- name: forwarders_ocr2
	id: forwarders_ocr2
	nodes: 2
	os: ubuntu-latest
	pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
	runs-on: ${{ matrix.product.os }}
	name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
	steps:
	# Handy for debugging resource usage
	# - name: Collect Workflow Telemetry
	# uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
	- name: Collect Metrics
	if: needs.changes.outputs.src == 'true' \|\| github.event_name == 'workflow_dispatch'
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }}
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
	test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
	continue-on-error: true
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref \|\| github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}
	- name: Build Go Test Command
	id: build-go-test-command
	run: \|
	# if dir is provided use it, otherwise use the smoke dir
	if [ "${{ matrix.product.dir }}" != "" ]; then
	dir=${{ matrix.product.dir }}
	else
	dir=smoke
	fi
	# if the matrix.product.run is set, use it for a different command
	if [ "${{ matrix.product.run }}" != "" ]; then
	echo "run_command=${{ matrix.product.run }} ./${dir}/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
	else
	echo "run_command=./${dir}/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
	fi
	- name: Check for "enable tracing" label
	id: check-label
	run: \|
	label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH")

	if [[ -n "$label" ]]; then
	if [[ "$label" == "enable tracing" ]]; then
	echo "Enable tracing label found."
	echo "trace=true" >> $GITHUB_OUTPUT
	else
	echo "Enable tracing label not found."
	echo "trace=false" >> $GITHUB_OUTPUT
	fi
	else
	echo "No labels present or labels are null."
	echo "trace=false" >> $GITHUB_OUTPUT
	fi

	- name: Setup Grafana and OpenTelemetry
	id: docker-setup
	if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
	run: \|
	# Create network
	docker network create --driver bridge tracing

	# Make trace directory
	cd integration-tests/smoke/
	mkdir ./traces
	chmod -R 777 ./traces

	# Switch directory
	cd ../../.github/tracing

	# Create a Docker volume for traces
	# docker volume create otel-traces

	# Start OpenTelemetry Collector
	# Note the user must be set to the same user as the runner for the trace data to be accessible
	docker run -d --network=tracing --name=otel-collector \
	-v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \
	-v $PWD/../../integration-tests/smoke/traces:/tracing \
	--user "$(id -u):$(id -g)" \
	-p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml

	- name: Locate Docker Volume
	id: locate-volume
	if: false
	run: \|
	echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT

	- name: Show Otel-Collector Logs
	if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
	run: \|
	docker logs otel-collector
	- name: Set Override Config
	id: set_override_config
	run: \|
	# if the matrix.product.config_path is set, use it as the override config
	if [ "${{ matrix.product.config_path }}" != "" ]; then
	echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT"
	fi
	- name: Setup GAP for Grafana
	uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
	id: setup-gap
	with:
	# aws inputs
	aws-region: ${{ secrets.AWS_REGION }}
	aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
	api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
	# other inputs
	duplicate-authorization-header: "true"
	# metrics inputs
	metrics-job-name: "grafana"
	gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
	gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	- name: Prepare Base64 CCIP TOML secrets
	uses: ./.github/actions/setup-create-base64-config-ccip
	with:
	runId: ${{ github.run_id }}
	pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' \|\| !startsWith(github.ref, 'refs/tags/') && '' \|\| secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
	pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
	pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
	testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
	selectedNetworks: SIMULATED_1,SIMULATED_2
	chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
	chainlinkVersion: ${{ github.sha }}
	lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
	lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
	lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
	logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
	grafanaUrl: "http://localhost:8080/primary"
	grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
	grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}

	## Run this step when changes that require tests to be run are made
	- name: Run Tests
	if: needs.changes.outputs.src == 'true' \|\| github.event_name == 'workflow_dispatch'
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28
	env:
	BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }}
	TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }}
	DEBUG_RESTY: false
	with:
	test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 \| tee /tmp/gotest.log \| gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
	test_download_vendor_packages_command: cd ./integration-tests && go mod download
	test_config_chainlink_version: ${{ inputs.evm-ref \|\| github.sha }}
	test_config_selected_networks: ${{ env.SELECTED_NETWORKS }}
	test_config_logging_run_id: ${{ github.run_id }}
	test_config_logstream_log_targets: "file"
	test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
	cl_repo: ${{ env.CHAINLINK_IMAGE }}
	cl_image_tag: ${{ inputs.evm-ref \|\| github.sha }}${{ matrix.product.tag_suffix }}
	aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
	artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs
	artifacts_location: \|
	./integration-tests/smoke/logs/
	./integration-tests/ccip-tests/smoke/logs/*
	/tmp/gotest.log
	publish_check_name: ${{ matrix.product.name }}
	token: ${{ secrets.GITHUB_TOKEN }}
	go_mod_path: ./integration-tests/go.mod
	cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
	cache_restore_only: "true"
	QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
	QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
	QA_KUBECONFIG: ""
	should_tidy: "false"
	go_coverage_src_dir: /var/tmp/go-coverage
	go_coverage_dest_dir: ${{ github.workspace }}/.covdata
	DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
	DEFAULT_LOKI_TENANT_ID: "promtail"
	DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
	DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
	DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
	DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
	DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' \|\| !startsWith(github.ref, 'refs/tags/') && '' \|\| secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
	DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
	DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
	DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' \|\| !startsWith(github.ref, 'refs/tags/') && 'false' \|\| 'true' }}

	- name: Upload Coverage Data
	uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
	timeout-minutes: 2
	continue-on-error: true
	with:
	name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }}
	path: .covdata
	retention-days: 1

	# Run this step when changes that do not need the test to run are made
	- name: Run Setup
	if: needs.changes.outputs.src == 'false'
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
	with:
	test_download_vendor_packages_command: cd ./integration-tests && go mod download
	go_mod_path: ./integration-tests/go.mod
	cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
	cache_restore_only: "true"
	QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
	QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
	QA_KUBECONFIG: ""
	should_tidy: "false"

	- name: Show Otel-Collector Logs
	if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
	run: \|
	docker logs otel-collector

	- name: Permissions on traces
	if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
	run: \|
	ls -l ./integration-tests/smoke/traces

	- name: Upload Trace Data
	if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
	uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
	with:
	name: trace-data
	path: ./integration-tests/smoke/traces/trace-data.json

	- name: Print failed test summary
	if: always()
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
	with:
	test_directories: ./integration-tests/smoke/,./integration-tests/ccip-tests/smoke/

	### Used to check the required checks box when the matrix completes
	eth-smoke-tests:
	if: always()
	runs-on: ubuntu-latest
	name: ETH Smoke Tests
	needs: [eth-smoke-tests-matrix]
	steps:
	- name: Check smoke test matrix status
	if: needs.eth-smoke-tests-matrix.result != 'success'
	run: \|
	echo "${{ needs.eth-smoke-tests-matrix.result }}"
	exit 1
	- name: Collect Metrics
	if: always()
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-matrix-results
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: ETH Smoke Tests
	matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }}
	continue-on-error: true

	cleanup:
	name: Clean up integration environment deployments
	if: always()
	needs: [eth-smoke-tests]
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repo
	if: ${{ github.event_name == 'pull_request' }}
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref }}

	- name: 🧼 Clean up Environment
	if: ${{ github.event_name == 'pull_request' }}
	uses: ./.github/actions/delete-deployments
	with:
	environment: integration
	ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663

	- name: Collect Metrics
	if: ${{ github.event_name == 'pull_request' }}
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-env-cleanup
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: Clean up integration environment deployments
	continue-on-error: true

	show-coverage:
	name: Show Chainlink Node Go Coverage
	if: always()
	needs: [cleanup]
	runs-on: ubuntu-latest
	steps:
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref \|\| github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}
	- name: Download All Artifacts
	uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
	with:
	path: cl-node-coverage-data
	pattern: cl-node-coverage-data-*
	merge-multiple: true
	- name: Show Coverage
	run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"

	# Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
	# this will also only run if both of the matrix jobs pass
	eth-smoke-go-mod-cache:
	environment: integration
	needs: [eth-smoke-tests]
	runs-on: ubuntu-latest
	name: ETH Smoke Tests Go Mod Cache
	continue-on-error: true
	steps:
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref \|\| github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}
	- name: Run Setup
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
	with:
	test_download_vendor_packages_command: \|
	cd ./integration-tests
	go mod download
	# force download of test dependencies
	go test -run=NonExistentTest ./smoke/... \|\| echo "ignore expected test failure"
	go_mod_path: ./integration-tests/go.mod
	cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
	cache_restore_only: "false"

	### Migration tests
	node-migration-tests:
	name: Version Migration Tests
	environment: integration
	permissions:
	checks: write
	pull-requests: write
	id-token: write
	contents: read
	runs-on: ubuntu-latest
	needs: [build-chainlink, changes]
	# Only run migration tests on new tags
	if: startsWith(github.ref, 'refs/tags/')
	env:
	SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
	CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref \|\| github.sha }}
	CHAINLINK_ENV_USER: ${{ github.actor }}
	CHAINLINK_IMAGE: public.ecr.aws/w0i8p0z9/chainlink-ccip
	UPGRADE_VERSION: ${{ inputs.evm-ref \|\| github.sha }}
	UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
	TEST_LOG_LEVEL: debug
	TEST_SUITE: migration
	steps:
	- name: Collect Metrics
	id: collect-gha-metrics
	uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
	with:
	id: ${{ env.COLLECTION_ID }}-migration-tests
	org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
	this-job-name: Version Migration Tests
	test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
	continue-on-error: true
	- name: Checkout the repo
	uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
	with:
	repository: smartcontractkit/ccip
	ref: ${{ inputs.cl_ref \|\| github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}
	- name: Get Latest Version
	id: get_latest_version
	run: \|
	untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest \| jq -r .name)
	latest_version="${untrimmed_ver:1}"
	# Check if latest_version is empty
	if [ -z "$latest_version" ]; then
	echo "Error: The latest_version is empty. The migration tests need a verison to run."
	exit 1
	fi
	# CCIP uses the `-release` suffix for their versions
	echo "latest_version=${latest_version}-release" >> "$GITHUB_OUTPUT"
	- name: Name Versions
	run: \|
	echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref \|\| github.sha }}'"
	- name: Prepare Base64 TOML override
	uses: ./.github/actions/setup-create-base64-upgrade-config
	with:
	selectedNetworks: ${{ env.SELECTED_NETWORKS }}
	chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }}
	upgradeVersion: ${{ env.UPGRADE_VERSION }}
	runId: ${{ github.run_id }}
	testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
	logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
	- name: Run Migration Tests
	uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28
	with:
	test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 \| tee /tmp/gotest.log \| gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
	test_download_vendor_packages_command: cd ./integration-tests && go mod download
	test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
	cl_repo: ${{ env.CHAINLINK_IMAGE }}
	cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }}
	aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
	artifacts_name: node-migration-test-logs
	artifacts_location: \|
	./integration-tests/migration/logs
	/tmp/gotest.log
	publish_check_name: Node Migration Test Results
	token: ${{ secrets.GITHUB_TOKEN }}
	go_mod_path: ./integration-tests/go.mod
	cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
	cache_restore_only: "true"
	QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
	QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
	QA_KUBECONFIG: ""
	go_coverage_src_dir: /var/tmp/go-coverage
	go_coverage_dest_dir: ${{ github.workspace }}/.covdata
	should_tidy: "false"
	DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
	DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ env.UPGRADE_IMAGE }}
	DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
	DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
	DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
	DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
	DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
	DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}

	- name: Upload Coverage Data
	uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
	timeout-minutes: 2
	continue-on-error: true
	with:
	name: cl-node-coverage-data-migration-tests
	path: .covdata
	retention-days: 1
	- name: Notify Slack
	if: failure() && github.event_name != 'workflow_dispatch'
	uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
	env:
	SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
	with:
	channel-id: "C05QQ8Z342V"
	slack-message: "CCIP Node Migration Tests Failed :x: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
	payload: \|
	{
	"attachments": [
	{
	"color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' \|\| '#2E7D32' }}",
	"blocks": [
	{
	"type": "header",
	"text": {
	"type": "plain_text",
	"text": "CCIP Version Migration Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' \|\| ':white_check_mark:'}}",
	"emoji": true
	}
	},
	{
	"type": "section",
	"text": {
	"type": "mrkdwn",
	"text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed! Notifying <!subteam^S06HJ46A1EX\|ccip-qa>' \|\| 'All Good!' }}"
	}
	},
	{
	"type": "divider"
	},
	{
	"type": "section",
	"text": {
	"type": "mrkdwn",
	"text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}\|${{ github.ref_name }}> \| <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}\|${{ github.sha }}> \| <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\|Run>"
	}
	}
	]
	}
	]
	}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Integration Tests #9162

Workflow file

Integration Tests #9162

Jobs

Run details

Workflow file for this run