diff --git a/.github/actions/setup-create-base64-config-ccip/action.yml b/.github/actions/setup-create-base64-config-ccip/action.yml new file mode 100644 index 00000000000..88d9fe8078c --- /dev/null +++ b/.github/actions/setup-create-base64-config-ccip/action.yml @@ -0,0 +1,187 @@ +name: Create Base64 Config for CCIP Tests +description: A composite action that creates a base64-encoded config to be used by ccip integration tests + +inputs: + runId: + description: The run id + existingNamespace: + description: If test needs to run against already deployed namespace + testLogCollect: + description: Whether to always collect logs, even for passing tests + default: "false" + selectedNetworks: + description: The networks to run tests against + chainlinkImage: + description: The chainlink image to use + default: "public.ecr.aws/chainlink/chainlink" + chainlinkVersion: + description: The git commit sha to use for the image tag + upgradeImage: + description: The chainlink image to upgrade to + default: "" + upgradeVersion: + description: The git commit sha to use for the image tag + lokiEndpoint: + description: Loki push endpoint + lokiTenantId: + description: Loki tenant id + lokiBasicAuth: + description: Loki basic auth + logstreamLogTargets: + description: Where to send logs (e.g. file, loki) + grafanaUrl: + description: Grafana URL + grafanaDashboardUrl: + description: Grafana dashboard URL + grafanaBearerToken: + description: Grafana bearer token + customEvmNodes: + description: Custom EVM nodes to use in key=value format, where key is chain id and value is docker image to use. If they are provided the number of networksSelected must be equal to the number of customEvmNodes + evmNodeLogLevel: + description: Log level for the custom EVM nodes + default: "info" + +runs: + using: composite + steps: + - name: Prepare Base64 TOML override + shell: bash + id: base64-config-override + env: + RUN_ID: ${{ inputs.runId }} + SELECTED_NETWORKS: ${{ inputs.selectedNetworks }} + EXISTING_NAMESPACE: ${{ inputs.existingNamespace }} + TEST_LOG_COLLECT: ${{ inputs.testLogCollect }} + CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }} + CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }} + UPGRADE_IMAGE: ${{ inputs.upgradeImage }} + UPGRADE_VERSION: ${{ inputs.upgradeVersion }} + LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }} + LOKI_TENANT_ID: ${{ inputs.lokiTenantId }} + LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }} + LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} + GRAFANA_URL: ${{ inputs.grafanaUrl }} + GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} + GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }} + CUSTOM_EVM_NODES: ${{ inputs.customEvmNodes }} + EVM_NODE_LOG_LEVEL: ${{ inputs.evmNodeLogLevel }} + run: | + echo ::add-mask::$CHAINLINK_IMAGE + function convert_to_toml_array() { + local IFS=',' + local input_array=($1) + local toml_array_format="[" + + for element in "${input_array[@]}"; do + toml_array_format+="\"$element\"," + done + + toml_array_format="${toml_array_format%,}]" + echo "$toml_array_format" + } + + selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") + log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") + + if [ -n "$TEST_LOG_COLLECT" ]; then + test_log_collect=true + else + test_log_collect=false + fi + + # make sure the number of networks and nodes match + IFS=',' read -r -a networks_array <<< "$SELECTED_NETWORKS" + IFS=',' read -r -a nodes_array <<< "$CUSTOM_EVM_NODES" + + networks_count=${#networks_array[@]} + nodes_count=${#nodes_array[@]} + + # Initialize or clear CONFIG_TOML environment variable + custom_nodes_toml="" + + # Check if the number of CUSTOM_EVM_NODES is zero + if [ $nodes_count -eq 0 ]; then + echo "The number of CUSTOM_EVM_NODES is zero, won't output any custom private Ethereum network configurations." + else + if [ $networks_count -ne $nodes_count ]; then + echo "The number of elements in SELECTED_NETWORKS (${networks_count}) and CUSTOM_EVM_NODES does not match (${nodes_count})." + exit 1 + else + for i in "${!networks_array[@]}"; do + IFS='=' read -r chain_id docker_image <<< "${nodes_array[i]}" + custom_nodes_toml+=" + [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}] + ethereum_version=\"\" + execution_layer=\"\" + + [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}.EthereumChainConfig] + seconds_per_slot=3 + slots_per_epoch=2 + genesis_delay=15 + validator_count=4 + chain_id=${chain_id} + addresses_to_fund=[\"0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266\", \"0x70997970C51812dc3A010C7d01b50e0d17dc79C8\"] + node_log_level=\"${EVM_NODES_LOG_LEVEL}\" + + [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}.EthereumChainConfig.HardForkEpochs] + Deneb=500 + + [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}.CustomDockerImages] + execution_layer=\"${docker_image}\" + " + done + fi + fi + + grafana_bearer_token="" + if [ -n "$GRAFANA_BEARER_TOKEN" ]; then + grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\"" + fi + + cat << EOF > config.toml + [CCIP] + [CCIP.Env] + EnvToConnect="$EXISTING_NAMESPACE" + [CCIP.Env.Network] + selected_networks = $selected_networks + [CCIP.Env.NewCLCluster] + [CCIP.Env.NewCLCluster.Common] + [CCIP.Env.NewCLCluster.Common.ChainlinkImage] + image="$CHAINLINK_IMAGE" + version="$CHAINLINK_VERSION" + + [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage] + image="$UPGRADE_IMAGE" + version="$UPGRADE_VERSION" + + $custom_nodes_toml + + [CCIP.Env.Logging] + test_log_collect=$test_log_collect + run_id="$RUN_ID" + + [CCIP.Env.Logging.LogStream] + log_targets=$log_targets + + [CCIP.Env.Logging.Loki] + tenant_id="$LOKI_TENANT_ID" + endpoint="$LOKI_ENDPOINT" + basic_auth_secret="$LOKI_BASIC_AUTH" + + [CCIP.Env.Logging.Grafana] + base_url="$GRAFANA_URL" + dashboard_url="$GRAFANA_DASHBOARD_URL" + $grafana_bearer_token + + [CCIP.Groups.load] + TestRunName = '$EXISTING_NAMESPACE' + + [CCIP.Groups.smoke] + TestRunName = '$EXISTING_NAMESPACE' + + EOF + + BASE64_CCIP_SECRETS_CONFIG=$(cat config.toml | base64 -w 0) + echo ::add-mask::$BASE64_CCIP_SECRETS_CONFIG + echo "BASE64_CCIP_SECRETS_CONFIG=$BASE64_CCIP_SECRETS_CONFIG" >> $GITHUB_ENV + echo "TEST_BASE64_CCIP_SECRETS_CONFIG=$BASE64_CCIP_SECRETS_CONFIG" >> $GITHUB_ENV diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml new file mode 100644 index 00000000000..6d1aaffb176 --- /dev/null +++ b/.github/workflows/ccip-chaos-tests.yml @@ -0,0 +1,252 @@ +name: CCIP Chaos Tests +on: + workflow_run: + workflows: [ CCIP Load Test ] + types: [ completed ] + branches: [ ccip-develop ] + workflow_dispatch: + + + +# Only run 1 of this workflow at a time per PR +concurrency: + group: chaos-ccip-tests-chainlink-${{ github.ref }} + cancel-in-progress: true + +env: + CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} + MOD_CACHE_VERSION: 1 + +jobs: + build-chainlink: + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + runs-on: ubuntu20.04-16cores-64GB + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Check if image exists + id: check-image + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + with: + repository: chainlink + tag: ${{ github.sha }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + - name: Build Image + if: steps.check-image.outputs.exists == 'false' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + env: + GH_TOKEN: ${{ github.token }} + with: + cl_repo: smartcontractkit/chainlink-ccip + cl_ref: ${{ github.sha }} + push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-chaos-tests-build-chainlink-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + + build-test-image: + environment: integration + permissions: + id-token: write + contents: read + name: Build Test Image + runs-on: ubuntu20.04-16cores-64GB + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-chaos-tests-build-test-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Test Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Build Test Image + uses: ./.github/actions/build-test-image + with: + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + + ccip-chaos-tests: + environment: integration + permissions: + issues: read + checks: write + pull-requests: write + id-token: write + contents: read + name: CCIP Chaos Tests + runs-on: ubuntu-latest + needs: [ build-chainlink, build-test-image ] + env: + TEST_SUITE: chaos + TEST_ARGS: -test.timeout 30m + CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_TRIGGERED_BY: ccip-cron-chaos-eth + TEST_LOG_LEVEL: debug + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + GH_TOKEN: ${{ github.token }} + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-chaos-tests + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: CCIP Chaos Tests + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + chainlinkVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + - name: Run Chaos Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + with: + test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci + test_download_vendor_packages_command: make gomod + cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + cl_image_tag: ${{ github.sha }} + artifacts_location: ./integration-tests/chaos/logs + publish_check_name: CCIP Chaos Test Results + publish_report_paths: ./tests-chaos-report.xml + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + CGO_ENABLED: "1" + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + ## Notify in slack if the job fails + - name: Notify Slack + if: failure() && github.event_name != 'workflow_dispatch' + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + env: + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + with: + channel-id: "#ccip-testing" + slack-message: ":x: :mild-panic-intensifies: CCIP chaos tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" + ## Run Cleanup if the job succeeds + - name: cleanup + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + with: + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + + ccip-chaos-with-load-tests: + environment: integration + permissions: + issues: read + checks: write + pull-requests: write + id-token: write + contents: read + name: CCIP Load With Chaos Tests + if: false # Disabled until CCIP-2555 is resolved + runs-on: ubuntu-latest + needs: [ build-chainlink, build-test-image ] + env: + TEST_SUITE: load + TEST_ARGS: -test.timeout 1h + CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_TRIGGERED_BY: ccip-cron-chaos-and-load-eth + TEST_LOG_LEVEL: debug + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + GH_TOKEN: ${{ github.token }} + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-chaos-tests-with-load-test + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: CCIP load with chaos test + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Prepare Base64 TOML override for CCIP secrests + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + chainlinkVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" + - name: Run Load With Chaos Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + with: + test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt + test_download_vendor_packages_command: make gomod + cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + cl_image_tag: ${{ github.sha }} + artifacts_location: ./integration-tests/load/logs + publish_check_name: CCIP Chaos With Load Test Results + publish_report_paths: ./tests-chaos-with-load-report.xml + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + CGO_ENABLED: "1" + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + ## Notify in slack if the job fails + - name: Notify Slack + if: failure() && github.event_name != 'workflow_dispatch' + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + env: + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + with: + channel-id: "#ccip-testing" + slack-message: ":x: :mild-panic-intensifies: CCIP chaos with load tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" + ## Run Cleanup if the job succeeds + - name: cleanup + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + with: + triggered_by: ${{ env.TEST_TRIGGERED_BY }} diff --git a/.github/workflows/ccip-client-compatibility-tests.yml b/.github/workflows/ccip-client-compatibility-tests.yml new file mode 100644 index 00000000000..42daf280ca7 --- /dev/null +++ b/.github/workflows/ccip-client-compatibility-tests.yml @@ -0,0 +1,742 @@ +name: CCIP Client Compatibility Tests +on: + schedule: + - cron: "30 5 * * TUE,FRI" # Run every Tuesday and Friday at midnight + 30min EST + push: + tags: + - "*" + merge_group: + pull_request: + workflow_dispatch: + inputs: + chainlinkVersion: + description: commit SHA or tag of the Chainlink version to test + required: true + type: string + evmImplementations: + description: comma separated list of EVM implementations to test (ignored if base64TestList is used) + required: true + type: string + default: "geth,besu,nethermind,erigon" + latestVersionsNumber: + description: how many of latest images of EVM implementations to test with (ignored if base64TestList is used) + required: true + type: number + default: 3 + base64TestList: + description: base64 encoded list of tests to run (same as base64-ed output of testlistgenerator tool) + required: false + type: string + +env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/ccip + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + MOD_CACHE_VERSION: 2 + +jobs: + # Build Test Dependencies + + check-dependency-bump: + name: Check for go-ethereum dependency bump + if: github.event_name == 'pull_request' || github.event_name == 'merge_queue' + runs-on: ubuntu-latest + outputs: + dependency_changed: ${{ steps.changes.outputs.dependency_changed }} + steps: + - name: Checkout code + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + repository: smartcontractkit/ccip + fetch-depth: 0 + - name: Check for go.mod changes + id: changes + run: | + git fetch origin ${{ github.base_ref }} + # if no match is found then grep exits with code 1, but if there is a match it exits with code 0 + # this will return a match if there are any changes on that corresponding line, for example if spacing was changed + DEPENDENCY_CHANGED=$(git diff -U0 origin/${{ github.base_ref }}...HEAD -- go.mod | grep -q 'github.com/ethereum/go-ethereum'; echo $?) + PR_VERSION=$(grep 'github.com/ethereum/go-ethereum' go.mod | awk '{print $2}') + + # here 0 means a match was found, 1 means no match was found + if [ "$DEPENDENCY_CHANGED" -eq 0 ]; then + # Dependency was changed in the PR, now compare with the base branch + git fetch origin ${{ github.base_ref }} + BASE_VERSION=$(git show origin/${{ github.base_ref }}:go.mod | grep 'github.com/ethereum/go-ethereum' | awk '{print $2}') + + echo "Base branch version: $BASE_VERSION" + echo "PR branch version: $PR_VERSION" + + echo "Dependency version changed in the PR compared to the base branch." + echo "dependency_changed=true" >> $GITHUB_OUTPUT + else + echo "No changes to ethereum/go-ethereum dependency in the PR." + echo "PR branch version: $PR_VERSION" + echo "dependency_changed=false" >> $GITHUB_OUTPUT + fi + + should-run: + if: always() + name: Check if the job should run + needs: check-dependency-bump + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.should-run.outputs.should_run }} + eth_implementations : ${{ steps.should-run.outputs.eth_implementations }} + env: + GITHUB_REF_TYPE: ${{ github.ref_type }} + steps: + - name: Check if the job should run + id: should-run + run: | + if [ "${{ needs.check-dependency-bump.outputs.dependency_changed }}" == "true" ]; then + echo "Will run tests, because go-ethereum dependency was bumped" + echo "should_run=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "Will run tests, because trigger event was $GITHUB_EVENT_NAME" + echo "should_run=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Will run tests, because trigger event was $GITHUB_EVENT_NAME" + echo "should_run=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_REF_TYPE" = "tag" ]; then + echo "Will run tests, because new tag was created" + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "Will not run tests" + echo "should_run=false" >> $GITHUB_OUTPUT + fi + + select-versions: + if: always() && needs.should-run.outputs.should_run == 'true' + name: Select Versions + needs: should-run + runs-on: ubuntu-latest + env: + RELEASED_DAYS_AGO: 4 + GITHUB_REF_TYPE: ${{ github.ref_type }} + outputs: + evm_implementations : ${{ steps.select-implementations.outputs.evm_implementations }} + chainlink_version: ${{ steps.select-chainlink-version.outputs.chainlink_version }} + latest_image_count: ${{ steps.get-image-count.outputs.image_count }} + steps: + # ghlatestreleasechecker is a tool to check if new release is available for a given repo + - name: Set Up ghlatestreleasechecker + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ghlatestreleasechecker@v1.0.0 + - name: Select EVM implementations to test + id: select-implementations + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "Checking for new releases" + implementations_arr=() + new_geth=$(ghlatestreleasechecker "ethereum/go-ethereum" $RELEASED_DAYS_AGO) + if [ "$new_geth" != "none" ]; then + echo "New geth release found: $new_geth" + implementations_arr+=("geth") + fi + new_besu=$(ghlatestreleasechecker "hyperledger/besu" $RELEASED_DAYS_AGO) + if [ "new_besu" != "none" ]; then + echo "New besu release found: $new_besu" + implementations_arr+=("besu") + fi + new_erigon=$(ghlatestreleasechecker "ledgerwatch/erigon" $RELEASED_DAYS_AGO) + if [ "new_erigon" != "none" ]; then + echo "New erigon release found: $new_erigon" + implementations_arr+=("erigon") + fi + new_nethermind=$(ghlatestreleasechecker "nethermindEth/nethermind" $RELEASED_DAYS_AGO) + if [ "new_nethermind" != "none" ]; then + echo "New nethermind release found: $new_nethermind" + implementations_arr+=("nethermind") + fi + IFS=',' + eth_implementations="${implementations_arr[*]}" + echo "Found new releases for: $eth_implementations" + echo "evm_implementations=$eth_implementations" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + if [ -n "${{ github.event.inputs.base64TestList }}" ]; then + echo "Base64-ed Test Input provided, ignoring EVM implementations" + else + echo "Will test following EVM implementations: ${{ github.event.inputs.evmImplementations }}" + echo "evm_implementations=${{ github.event.inputs.evmImplementations }}" >> $GITHUB_OUTPUT + fi + else + echo "Will test all EVM implementations" + echo "evm_implementations=geth,besu,nethermind,erigon" >> $GITHUB_OUTPUT + fi + - name: Select Chainlink CCIP version + id: select-chainlink-version + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "Fetching latest Chainlink CCIP stable version" + implementations_arr=() + # we use 100 days since we really want the latest one, and it's highly improbable there won't be a release in last 100 days + chainlink_version=$(ghlatestreleasechecker "smartcontractkit/ccip" 100) + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Fetching Chainlink version from input" + if [ -n "${{ github.event.inputs.chainlinkVersion }}" ]; then + echo "Chainlink version provided in input" + chainlink_version="${{ github.event.inputs.chainlinkVersion }}" + else + echo "Chainlink version not provided in input. Using latest commit SHA." + chainlink_version=${{ github.sha }} + fi + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "Fetching Chainlink version from PR's head commit" + chainlink_version="${{ github.event.pull_request.head.sha }}" + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "merge_queue" ]; then + echo "Fetching Chainlink version from merge queue's head commit" + chainlink_version="${{ github.event.merge_group.head_sha }}" + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_REF_TYPE" = "tag" ]; then + echo "Fetching Chainlink version from tag" + chainlink_version="${{ github.ref_name }}" + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + else + echo "Unsupported trigger event. It's probably an issue with the pipeline definition. Please reach out to the Test Tooling team." + exit 1 + fi + echo "Will use following Chainlink version: $chainlink_version" + - name: Get image count + id: get-image-count + run: | + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Fetching latest image count from input" + if [ -n "${{ github.event.inputs.base64TestList }}" ]; then + echo "Base64-ed Test Input provided, ignoring latest image count" + else + image_count="${{ github.event.inputs.latestVersionsNumber }}" + echo "image_count=$image_count" >> $GITHUB_OUTPUT + fi + else + echo "Fetching default latest image count" + image_count=3 + echo "image_count=$image_count" >> $GITHUB_OUTPUT + fi + echo "Will use following latest image count: $image_count" + + check-ecr-images-exist: + name: Check images used as test dependencies exist in ECR + if: always() && needs.should-run.outputs.should_run == 'true' + environment: integration + permissions: + id-token: write + contents: read + needs: [should-run] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mirror: + - name: ethereum/client-go + expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$' + - name: hyperledger/besu + expression: '^[0-9]+\.[0-9]+(\.[0-9]+)?$' + page_size: 300 + - name: thorax/erigon + expression: '^v[0-9]+\.[0-9]+\.[0-9]+$' + - name: nethermind/nethermind + expression: '^[0-9]+\.[0-9]+\.[0-9]+$' + - name: tofelb/ethereum-genesis-generator + expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?' + steps: + - name: Update internal ECR if the latest Ethereum client image does not exist + uses: smartcontractkit/chainlink-testing-framework/.github/actions/update-internal-mirrors@5eea86ee4f7742b4e944561a570a6b268e712d9e # v1.30.3 + with: + aws_region: ${{ secrets.QA_AWS_REGION }} + role_to_assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + aws_account_number: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + image_name: ${{matrix.mirror.name}} + expression: ${{matrix.mirror.expression}} + page_size: ${{matrix.mirror.page_size}} + + build-chainlink: + if: always() && needs.should-run.outputs.should_run == 'true' + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + needs: [should-run, select-versions] + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: client-compatablility-build-chainlink + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + repository: smartcontractkit/ccip + ref: ${{ needs.select-versions.outputs.chainlink_version }} + - name: Build Chainlink Image + uses: ./.github/actions/build-chainlink-image + with: + tag_suffix: "" + dockerfile: core/chainlink.Dockerfile + git_commit_sha: ${{ needs.select-versions.outputs.chainlink_version }} + check_image_exists: 'true' + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + get-latest-available-images: + name: Get Latest EVM Implementation's Images + if: always() && needs.should-run.outputs.should_run == 'true' + environment: integration + runs-on: ubuntu-latest + needs: [check-ecr-images-exist, should-run, select-versions] + permissions: + id-token: write + contents: read + env: + LATEST_IMAGE_COUNT: ${{ needs.select-versions.outputs.latest_image_count }} + outputs: + geth_images: ${{ env.GETH_IMAGES }} + nethermind_images: ${{ env.NETHERMIND_IMAGES }} + besu_images: ${{ env.BESU_IMAGES }} + erigon_images: ${{ env.ERIGON_IMAGES }} + steps: + # Setup AWS creds + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + aws-region: ${{ secrets.QA_AWS_REGION }} + role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + role-duration-seconds: 3600 + # Login to ECR + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + with: + mask-password: "true" + env: + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + # ecrimagefetcher is a tool to get latest images from ECR + - name: Set Up ecrimagefetcher + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1 + - name: Get latest docker images from ECR + if: ${{ github.event.inputs.base64TestList == '' }} + env: + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + ETH_IMPLEMENTATIONS: ${{ needs.select-versions.outputs.evm_implementations }} + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + if [[ "$ETH_IMPLEMENTATIONS" == *"geth"* ]]; then + geth_images=$(ecrimagefetcher 'ethereum/client-go' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }}) + echo "GETH_IMAGES=$geth_images" >> $GITHUB_ENV + echo "Geth latest images: $geth_images" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"nethermind"* ]]; then + nethermind_images=$(ecrimagefetcher 'nethermind/nethermind' '^[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }}) + echo "NETHERMIND_IMAGES=$nethermind_images" >> $GITHUB_ENV + echo "Nethermind latest images: $nethermind_images" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"besu"* ]]; then + # 24.3.3 is ignored as it doesn't support data & input fields in eth_call + besu_images=$(ecrimagefetcher 'hyperledger/besu' '^[0-9]+\.[0-9]+(\.[0-9]+)?$' ${{ env.LATEST_IMAGE_COUNT }} ">=24.5.1") + echo "BESU_IMAGES=$besu_images" >> $GITHUB_ENV + echo "Besu latest images: $besu_images" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then + # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node + erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "> $GITHUB_ENV + echo "Erigon latest images: $erigon_images" + fi + + # End Build Test Dependencies + + prepare-compatibility-matrix: + name: Prepare Compatibility Matrix + if: always() && needs.should-run.outputs.should_run == 'true' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + needs: [get-latest-available-images,should-run,select-versions] + runs-on: ubuntu-latest + env: + ETH_IMPLEMENTATIONS: ${{ needs.select-versions.outputs.evm_implementations }} + BASE64_TEST_LIST: ${{ github.event.inputs.base64TestList }} + outputs: + matrix: ${{ env.JOB_MATRIX_JSON }} + steps: + - name: Decode Base64 Test List Input if Set + if: env.BASE64_TEST_LIST != '' + run: | + echo "Decoding base64 tests list from the input" + DECODED_BASE64_TEST_LIST=$(echo $BASE64_TEST_LIST | base64 -d) + echo "Decoded input:" + echo "$DECODED_BASE64_TEST_LIST" + is_valid=$(echo "$DECODED_BASE64_TEST_LIST" | jq . > /dev/null 2>&1; echo $?) + if [ "$is_valid" -ne 0 ]; then + echo "Invalid base64 input. Please provide a valid base64 encoded JSON list of tests." + echo "Here is an example of valid JSON:" + cat <> $GITHUB_ENV + # testlistgenerator is a tool that builds a matrix of tests to run + - name: Set Up testlistgenerator + if: env.BASE64_TEST_LIST == '' + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/testlistgenerator@v1.1.0 + - name: Prepare matrix input + if: env.BASE64_TEST_LIST == '' + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + if [[ "$ETH_IMPLEMENTATIONS" == *"geth"* ]]; then + echo "Will test compatibility with geth" + testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "ccip-geth-compatibility-test" -w "SIMULATED_1,SIMULATED_2" -c 1337,2337 -n ubuntu-latest + else + echo "Will not test compatibility with geth" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"besu"* ]]; then + echo "Will test compatibility with besu" + testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "ccip-besu-compatibility-test" -w "SIMULATED_BESU_NONDEV_1,SIMULATED_BESU_NONDEV_2" -c 1337,2337 -n ubuntu-latest + else + echo "Will not test compatibility with besu" + fi + + # TODO: Waiting for CCIP-2255 to be resolved + if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then + echo "Will test compatibility with erigon" + testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "ccip-erigon-compatibility-test" -w "SIMULATED_1,SIMULATED_2" -c 1337,2337 -n ubuntu-latest + else + echo "Will not test compatibility with erigon" + fi + + # TODO: uncomment when nethermind flake reason is addressed + if [[ "$ETH_IMPLEMENTATIONS" == *"nethermind"* ]]; then + echo "Will not test compatibility with nethermind due to flakiness" + # echo "Will test compatibility with nethermind" + # testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "ccip-nethermind-compatibility-test" -w "SIMULATED_1,SIMULATED_2" -c 1337,2337 -n ubuntu-latest + else + echo "Will not test compatibility with nethermind" + fi + + jq . compatibility_test_list.json + echo "Adding human-readable name" + jq 'map(. + {visible_name: (.docker_image | split(",")[0] | split("=")[1])})' compatibility_test_list.json > compatibility_test_list_modified.json + jq . compatibility_test_list_modified.json + JOB_MATRIX_JSON=$(jq -c . compatibility_test_list_modified.json) + echo "JOB_MATRIX_JSON=${JOB_MATRIX_JSON}" >> $GITHUB_ENV + + run-client-compatibility-matrix: + name: CCIP Compatibility with ${{ matrix.evm_node.visible_name }} + if: always() && needs.should-run.outputs.should_run == 'true' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + needs: [build-chainlink, prepare-compatibility-matrix, should-run, select-versions] + env: + CHAINLINK_COMMIT_SHA: ${{ needs.select-versions.outputs.chainlink_version }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_LOG_LEVEL: debug + strategy: + fail-fast: false + matrix: + evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}} + runs-on: ubuntu-latest + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + repository: smartcontractkit/ccip + ref: ${{ needs.select-versions.outputs.chainlink_version }} + - name: Prepare Base64 TOML override + uses: ./.github/actions/setup-create-base64-config + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + selectedNetworks: ${{ matrix.evm_node.networks }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} + pyroscopeServer: ${{ !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + pyroscopeEnvironment: ci-ccip-bidirectional-lane-${{ matrix.evm_node.name }} + pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} + lokiEndpoint: ${{ secrets.LOKI_URL_CI }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + selectedNetworks: ${{ matrix.evm_node.networks }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} + lokiEndpoint: ${{ secrets.LOKI_URL_CI }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + customEvmNodes: ${{ matrix.evm_node.docker_image }} + evmNodeLogLevel: "trace" + - name: Prepare test log name + run: | + replace_special_chars() { + if [ -z "$1" ]; then + echo "Please provide a string as an argument." + return 1 + fi + + local input_string="$1" + + # Replace '/' with '-' + local modified_string="${input_string//\//-}" + + # Replace ':' with '-' + modified_string="${modified_string//:/-}" + + # Replace '.' with '-' + modified_string="${modified_string//./-}" + + echo "$modified_string" + } + echo "TEST_LOG_NAME=$(replace_special_chars "ccip-${{ matrix.evm_node.name }}-test-logs")" >> $GITHUB_ENV + - name: Print Test details - ${{ matrix.evm_node.docker_image }} + run: | + echo "EVM Implementation Docker Image: ${{ matrix.evm_node.docker_image }}" + echo "EVM Implementation Networks: ${{ matrix.evm_node.networks }}" + echo "Test identifier: ${{ matrix.evm_node.name }}" + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 + with: + test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=2 ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_name: ${{ env.TEST_LOG_NAME }} + artifacts_location: | + ./integration-tests/smoke/logs/ + ./integration-tests/ccip-tests/smoke/logs/* + /tmp/gotest.log + publish_check_name: ${{ matrix.evm_node.name }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: "" + should_tidy: "false" + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@1587f59bfd626b668d303abbc90fee41b12397e6 # v2.3.23 + with: + test_directories: ./integration-tests/smoke/,./integration-tests/ccip-tests/smoke/ + + start-slack-thread: + name: Start Slack Thread + if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' }} + environment: integration + outputs: + thread_ts: ${{ steps.slack.outputs.thread_ts }} + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [run-client-compatibility-matrix,should-run,select-versions] + steps: + - name: Debug Result + run: echo ${{ join(needs.*.result, ',') }} + - name: Main Slack Notification + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + id: slack + with: + channel-id: ${{ secrets.QA_CCIP_SLACK_CHANNEL }} + payload: | + { + "attachments": [ + { + "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "CCIP Compatibility Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", + "emoji": true + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed! Notifying ' || 'All Good!' }}" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.select-versions.outputs.chainlink_version }}|${{ needs.select-versions.outputs.chainlink_version }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" + } + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + + parse-test-results: + name: Parse Test Results + if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [run-client-compatibility-matrix,should-run] + outputs: + base64_parsed_results: ${{ steps.get-test-results.outputs.base64_parsed_results }} + steps: + # workflowresultparser is a tool to get job results from a workflow run + - name: Set Up workflowresultparser + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/workflowresultparser@v1.0.0 + - name: Get and parse Test Results + shell: bash + id: get-test-results + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^CCIP Compatibility with (.*)$" -namedKey="CCIP" -outputFile=output.json + + echo "base64_parsed_results=$(base64 -w 0 output.json)" >> $GITHUB_OUTPUT + + display-test-results: + name: Aggregated test results + if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' && needs.parse-test-results.result == 'success' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [start-slack-thread, should-run, select-versions, parse-test-results] + steps: + # asciitable is a tool that prints results in a nice ASCII table + - name: Set Up asciitable + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/asciitable@v1.0.2 + - name: Print aggregated test results + shell: bash + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + raw_results="$(echo ${{ needs.parse-test-results.outputs.base64_parsed_results }} | base64 -d)" + echo $raw_results > input.json + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "CCIP" --namedKey "CCIP" + + echo + echo "AGGREGATED RESULTS" + cat output.txt + + echo "## Aggregated EVM Implementations compatibility results summary" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + post-test-results-to-slack: + name: Post Test Results + if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' }} + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [start-slack-thread,should-run,select-versions] + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + ref: ${{ needs.select-versions.outputs.chainlink_version }} + - name: Get test results for CCIP + id: get-product-results + shell: bash + run: | + raw_results="$(echo ${{ needs.parse-test-results.outputs.base64_parsed_results }} | base64 -d)" + product_result=$(echo "$raw_results" | jq -c "select(has(\"CCIP\")) | .CCIP[]") + if [ -n "$product_result" ]; then + base64_result=$(echo $product_result | base64 -w 0) + echo "base64_result=$base64_result" >> $GITHUB_OUTPUT + else + echo "No results found for CCIP" + echo "base64_result=" >> $GITHUB_OUTPUT + fi + - name: Post Test Results to Slack + uses: ./.github/actions/notify-slack-jobs-result + with: + github_token: ${{ github.token }} + github_repository: ${{ github.repository }} + workflow_run_id: ${{ github.run_id }} + github_job_name_regex: ^CCIP Compatibility with (.*?)$ + message_title: CCIP Compatibility Test Results + slack_channel_id: ${{ secrets.QA_CCIP_SLACK_CHANNEL }} + slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} + slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} + base64_parsed_results: ${{ steps.get-product-results.outputs.base64_result }} diff --git a/.github/workflows/ccip-live-network-tests.yml b/.github/workflows/ccip-live-network-tests.yml new file mode 100644 index 00000000000..75fc218c5ee --- /dev/null +++ b/.github/workflows/ccip-live-network-tests.yml @@ -0,0 +1,305 @@ +name: CCIP On-Demand Live Network Tests +on: +# Enable this when CCIP-2875 is addressed +# schedule: +# - cron: '0 */6 * * *' + workflow_dispatch: + inputs: + base64_test_input : # base64 encoded toml for test input + description: 'Base64 encoded toml test input' + required: false + slackMemberID: + description: 'Slack member ID to notify' + required: false + test_type: + description: 'Type of test to run' + required: false + type: choice + options: + - 'load' + - 'smoke' + +# Only run 1 of this workflow at a time per PR +concurrency: + group: live-testnet-tests + cancel-in-progress: true + +env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + CHAINLINK_VERSION: ${{ github.sha}} + CHAINLINK_TEST_VERSION: ${{ github.sha}} + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws + +jobs: + build-chainlink: + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + runs-on: ubuntu20.04-16cores-64GB + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Check if image exists + id: check-image + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + with: + repository: chainlink + tag: ${{ env.CHAINLINK_VERSION }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + - name: Build Image + if: steps.check-image.outputs.exists == 'false' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + env: + GH_TOKEN: ${{ github.token }} + with: + cl_repo: smartcontractkit/chainlink-ccip + cl_ref: ${{ env.CHAINLINK_VERSION }} + push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-on-demand-live-testnet-tests-build-chainlink-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + + build-test-image: + environment: integration + permissions: + id-token: write + contents: read + name: Build Test Image + runs-on: ubuntu20.04-16cores-64GB + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-on-demand-live-testnet-tests-build-test-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Test Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Build Test Image + uses: ./.github/actions/build-test-image + with: + tag: ${{ env.CHAINLINK_TEST_VERSION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + + ccip-load-test: + name: CCIP Load Test + environment: integration + runs-on: ubuntu-latest + strategy: + matrix: + config: [mainnet.toml] + needs: [ build-chainlink, build-test-image ] + # if the event is a scheduled event or the test type is load and no previous job failed + if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }} + permissions: + issues: read + checks: write + pull-requests: write + id-token: write + contents: read + env: + CHAINLINK_ENV_USER: ${{ github.actor }} + SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} + SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} + TEST_LOG_LEVEL: info + REF_NAME: ${{ github.head_ref || github.ref_name }} + ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests + BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} + + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-on-demand-live-testnet-tests-load-tests + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: CCIP Load Test + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + ref: ${{ env.REF_NAME }} + - name: Prepare Base64 TOML override + shell: bash + run: | + SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) + echo ::add-mask::$SLACK_USER + echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) + echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE + echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + fi + if [[ "${{ github.event_name }}" == "schedule" ]]; then + BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }}) + echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE + echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV + fi + - name: step summary + shell: bash + run: | + echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + env: + TEST_SUITE: load + TEST_ARGS: -test.timeout 900h + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + RR_MEM: 8Gi + RR_CPU: 4 + DETACH_RUNNER: true + TEST_TRIGGERED_BY: ccip-load-test-ci + with: + test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ env.CHAINLINK_VERSION }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + artifacts_location: ./integration-tests/load/logs/payload_ccip.json + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + should_cleanup: false + + ccip-smoke-test: + name: CCIP smoke Test + environment: integration + runs-on: ubuntu-latest + needs: [ build-chainlink, build-test-image ] + # if the event is a scheduled event or the test type is load and no previous job failed + if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }} + permissions: + issues: read + checks: write + pull-requests: write + id-token: write + contents: read + env: + CHAINLINK_ENV_USER: ${{ github.actor }} + SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} + SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} + TEST_LOG_LEVEL: info + REF_NAME: ${{ github.head_ref || github.ref_name }} + ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests + BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} + + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-on-demand-live-testnet-tests-smoke-test + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: CCIP Smoke Test + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + ref: ${{ env.REF_NAME }} + - name: Prepare Base64 TOML override + shell: bash + run: | + SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) + echo ::add-mask::$SLACK_USER + echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) + echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE + echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + fi + - name: step summary + shell: bash + run: | + echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + env: + TEST_SUITE: smoke + TEST_ARGS: -test.timeout 900h + DETACH_RUNNER: true + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + RR_MEM: 8Gi + RR_CPU: 4 + TEST_TRIGGERED_BY: ccip-smoke-test-ci + with: + test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ env.CHAINLINK_VERSION }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + artifacts_location: ./integration-tests/smoke/logs/payload_ccip.json + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-smoke-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + should_cleanup: false \ No newline at end of file diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml new file mode 100644 index 00000000000..9f6a2b0878f --- /dev/null +++ b/.github/workflows/ccip-load-tests.yml @@ -0,0 +1,288 @@ +name: CCIP Load Test +on: + push: + branches: + - ccip-develop + tags: + - '*' + workflow_dispatch: + inputs: + base64_test_input: # base64 encoded toml for test input + description: 'Base64 encoded toml test input' + required: false + +# Only run 1 of this workflow at a time per PR +concurrency: + group: load-ccip-tests-chainlink-${{ github.ref }} + cancel-in-progress: true + +env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + CHAINLINK_VERSION: ${{ github.sha}} + INPUT_CHAINLINK_TEST_VERSION: ${{ github.sha}} + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws + MOD_CACHE_VERSION: 1 + +jobs: + build-chainlink: + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + runs-on: ubuntu20.04-16cores-64GB + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Check if image exists + id: check-image + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + with: + repository: chainlink + tag: ${{ env.CHAINLINK_VERSION }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + - name: Build Image + if: steps.check-image.outputs.exists == 'false' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + env: + GH_TOKEN: ${{ github.token }} + with: + cl_repo: smartcontractkit/chainlink-ccip + cl_ref: ${{ env.CHAINLINK_VERSION }} + push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-load-test-build-chainlink-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + + build-test-image: + environment: integration + permissions: + id-token: write + contents: read + name: Build Test Image + runs-on: ubuntu20.04-16cores-64GB + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-load-test-build-test-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Test Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Build Test Image + uses: ./.github/actions/build-test-image + with: + tag: ${{ env.INPUT_CHAINLINK_TEST_VERSION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + + ccip-load-test: + environment: integration + needs: [ build-chainlink, build-test-image ] + if: ${{ always() && !contains(needs.*.result, 'failure') }} + permissions: + issues: read + checks: write + pull-requests: write + id-token: write + contents: read + env: + CHAINLINK_ENV_USER: ${{ github.actor }} + SLACK_USER: ${{ inputs.slackMemberID }} + SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} + SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} + TEST_LOG_LEVEL: info + REF_NAME: ${{ github.head_ref || github.ref_name }} + BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} + strategy: + fail-fast: false + matrix: + type: + - name: stable-load + run: ^TestLoadCCIPStableRPS$ + os: ubuntu-latest + - name: load-with-arm-curse-uncurse + run: ^TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ + config_path: ./integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml + os: ubuntu-latest + runs-on: ${{ matrix.type.os }} + name: CCIP ${{ matrix.type.name }} + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-load-test-${{ matrix.type.name }} + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: CCIP ${{ matrix.type.name }} + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + ref: ${{ env.REF_NAME }} + - name: Sets env vars + shell: bash + run: | + # if the matrix.type.config_path is set, use it as the override config + if [ -n "${{ matrix.type.config_path }}" ]; then + echo "BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})" >> $GITHUB_ENV + echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})" >> $GITHUB_ENV + fi + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) + echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE + if [ -n "${BASE64_CCIP_CONFIG_OVERRIDE}" && "$BASE64_CCIP_CONFIG_OVERRIDE" != "null"]; then + echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV + fi + fi + - name: step summary + shell: bash + run: | + echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + env: + TEST_SUITE: load + TEST_ARGS: -test.timeout 900h + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + RR_MEM: 8Gi + RR_CPU: 4 + TEST_TRIGGERED_BY: ccip-load-test-ci-${{ matrix.type.name }} + with: + test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ${{ matrix.type.run }} ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ env.CHAINLINK_VERSION }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + publish_check_name: ${{ matrix.type.name }} + artifacts_location: ./integration-tests/load/logs/payload_ccip.json + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + should_cleanup: "true" + + # Reporting Jobs + start-slack-thread: + name: Start Slack Thread + if: ${{ failure() && needs.ccip-load-test.result != 'skipped' && needs.ccip-load-test.result != 'cancelled' }} + environment: integration + outputs: + thread_ts: ${{ steps.slack.outputs.thread_ts }} + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [ccip-load-test] + steps: + - name: Debug Result + run: echo ${{ join(needs.*.result, ',') }} + - name: Main Slack Notification + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + id: slack + with: + channel-id: "#ccip-testing" + payload: | + { + "attachments": [ + { + "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "CCIP load tests results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", + "emoji": true + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/${{contains(github.ref_name, 'release') && 'releases/tag' || 'tree'}}/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" + } + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + + post-test-results-to-slack: + name: Post Test Results + if: ${{ failure() && needs.start-slack-thread.result != 'skipped' && needs.start-slack-thread.result != 'cancelled' }} + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: start-slack-thread + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Post Test Results + uses: ./.github/actions/notify-slack-jobs-result + with: + github_token: ${{ github.token }} + github_repository: ${{ github.repository }} + workflow_run_id: ${{ github.run_id }} + github_job_name_regex: ^CCIP (.*)$ + message_title: CCIP Jobs + slack_channel_id: "#ccip-testing" + slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} + slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} + + # End Reporting Jobs diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml new file mode 100644 index 00000000000..50e02dfc55e --- /dev/null +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -0,0 +1,390 @@ +name: CCIP Offchain Upgrade Compatibility Tests +on: + merge_group: + pull_request: + push: + tags: + - "*" + workflow_dispatch: + +concurrency: + group: upgrade-tests-ccip-${{ github.ref }} + cancel-in-progress: true + +env: + # for run-test variables and environment + ECR_TAG: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:ccip-develop + ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + MOD_CACHE_VERSION: 2 + AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws + +jobs: + # Build Test Dependencies + changes: + environment: integration + name: Check Paths That Require Tests To Run + runs-on: ubuntu-latest + # We don't directly merge dependabot PRs, so let's not waste the resources + if: github.actor != 'dependabot[bot]' + steps: + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: smartcontractkit/ccip + ref: ${{ inputs.cl_ref }} + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + with: + filters: | + src: + - '**/*.go' + - '**/*go.sum' + - '**/*go.mod' + - '.github/workflows/integration-tests.yml' + - '**/*Dockerfile' + - 'core/**/config/**/*.toml' + - 'integration-tests/**/*.toml' + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-offchain-upgrade-compatibility-tests-changes + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Check Paths That Require Tests To Run + continue-on-error: true + outputs: + src: ${{ inputs.set_changes_output || steps.changes.outputs.src }} + + build-chainlink: + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + needs: [ changes ] + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-offchain-upgrade-compatibility-tests-build-chainlink + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: smartcontractkit/ccip + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Chainlink Image + uses: ./.github/actions/build-chainlink-image + with: + tag_suffix: "" + dockerfile: core/chainlink.Dockerfile + git_commit_sha: ${{ github.sha }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + build-test-image: + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + needs: [ changes ] + environment: integration + permissions: + id-token: write + contents: read + name: Build Test Image + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-offchain-upgrade-compatibility-tests-build-test-image + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Test Image with Current Sha + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: smartcontractkit/ccip + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Build Test Image + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + uses: ./.github/actions/build-test-image + with: + # we just want to build the load tests + suites: ccip-tests/load ccip-tests/smoke + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + + last-release-info: + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + needs: [ changes ] + environment: integration + permissions: + id-token: write + contents: read + name: Fetch Info for Last Release + runs-on: ubuntu-latest + outputs: + release_name: ${{ steps.fetch_last_release.outputs.release_name }} + release_sha: ${{ steps.fetch_last_release.outputs.sha_ref }} + release_tag: ${{ steps.fetch_last_release.outputs.release_tag_name }} + steps: + - name: Collect Metrics + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-offchain-upgrade-compatibility-tests-last-release-info + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Test Image for Last Release + continue-on-error: true + - name: find last release + id: fetch_last_release + shell: bash + run: | + release_name=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases | jq -r --arg SUFFIX "release" '.[] | select(.tag_name | endswith("release")) | .tag_name' | sort -V | tail -n 1) + release_tag_name="${release_name:1}" + echo "release_tag_name=${release_tag_name}" >> $GITHUB_OUTPUT + echo "release_name=${release_name}" >> $GITHUB_OUTPUT + sha_ref=$(curl -s --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/git/refs/tags/${release_name} | jq -r '.object.sha' | sort -V | tail -n 1) + echo "sha_ref=${sha_ref}" >> $GITHUB_OUTPUT + # End Build Test Dependencies + + # run test with previous image + run-test-with-last-release: + environment: integration + permissions: + actions: read + checks: write + pull-requests: write + id-token: write + contents: read + needs: [ changes, last-release-info, build-test-image ] + outputs: + existing_namespace: ${{ steps.fetch_namespace.outputs.existing_namespace }} + triggered_by: ${{ steps.fetch_namespace.outputs.triggered_by }} + strategy: + fail-fast: false + matrix: + product: + - name: ccip-smoke-with-last-release + os: ubuntu-latest + run: ^TestSmokeCCIPForBidirectionalLane$ + config_path: ./integration-tests/ccip-tests/testconfig/tomls/node-pre-upgrade-compatibility.toml + runs-on: ubuntu-latest + name: CCIP Deployment with ${{ needs.last-release-info.outputs.release_tag }} - ${{ matrix.product.name }} + env: + RELEASE_TAG: ${{ needs.last-release-info.outputs.release_tag }} + RELEASE_SHA: ${{ needs.last-release-info.outputs.release_sha }} + RELEASE_NAME: ${{ needs.last-release-info.outputs.release_name }} + TEST_TRIGGERED_BY: ${{ matrix.product.name }}-${{ github.run_id }} + steps: + - name: Collect Metrics + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-offchain-upgrade-compatibility-tests-run-tests-with-last-release + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: CCIP Deployment with ${{ needs.last-release-info.outputs.release_tag }} - ${{ matrix.product.name }} + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: smartcontractkit/ccip + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Set Override Config + id: set_override_config + run: | + # if the matrix.product.config_path is set, use it as the override config + if [ "${{ matrix.product.config_path }}" != "" ]; then + echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT" + fi + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + selectedNetworks: SIMULATED_1,SIMULATED_2 + chainlinkImage: ${{ env.AWS_ECR_REPO_PUBLIC_REGISTRY }}/w0i8p0z9/chainlink-ccip + chainlinkVersion: ${{ env.RELEASE_TAG }} + lokiEndpoint: ${{ secrets.LOKI_URL_CI }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + - name: Run Tests + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + env: + BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }} + TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }} + ENV_JOB_IMAGE: ${{ env.ENV_JOB_IMAGE_BASE }}:${{ github.sha }} + TEST_SUITE: smoke + TEST_ARGS: -test.timeout 30m + TEST_LOG_LEVEL: info + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + RR_MEM: 8Gi + RR_CPU: 4 + BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} + with: + test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 30m -count=1 -json -run ${{ matrix.product.run }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.AWS_ECR_REPO_PUBLIC_REGISTRY }}/w0i8p0z9/chainlink-ccip # releases are published to public registry + cl_image_tag: ${{ env.RELEASE_TAG }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs + artifacts_location: | + ./integration-tests/ccip-tests/smoke/logs/* + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + cache_key_id: ccip-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + triggered_by: ${{ env.TEST_TRIGGERED_BY }} + should_tidy: "false" + should_cleanup: "false" + + - name: store laneconfig in artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: lane-config + path: ./integration-tests/ccip-tests/smoke/tmp_*.json + if-no-files-found: error + + - name: fetch namespace + id: fetch_namespace + run: | + echo "looking for namespaces" + ITEMS=$(kubectl get ns -l=triggered-by=${{ env.TEST_TRIGGERED_BY }}-${{ github.event.pull_request.number || github.run_id }} -o jsonpath='{.items}') + COUNT=$(echo "${ITEMS}" | jq '. | length') + echo "found ${COUNT} namespaces. will set the env var with first one" + for ((i=0;i<${COUNT};i++)); do + name=$(echo "${ITEMS}" | jq -r ".[${i}].metadata.name") + echo "setting output var with namespace: ${name}" + echo "existing_namespace=${name}" >> $GITHUB_OUTPUT + break + done + echo "triggered_by=${{ env.TEST_TRIGGERED_BY }}" >> $GITHUB_OUTPUT + echo "completed env var set up" + + # run load test with current image + run-test-with-current-sha: + environment: integration + permissions: + actions: read + checks: write + pull-requests: write + id-token: write + contents: read + needs: [ build-chainlink, changes, build-test-image, run-test-with-last-release ] + strategy: + fail-fast: false + matrix: + product: + - name: ccip-load-after-upgrade + os: ubuntu-latest + run: ^TestLoadCCIPWithUpgradeNodeVersion$ + config_path: ./integration-tests/ccip-tests/testconfig/tomls/node-post-upgrade-compatibility.toml + runs-on: ubuntu-latest + name: Upgrade Nodes with Current SHA and Run ${{ matrix.product.name }} + steps: + - name: Collect Metrics + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: ccip-offchain-upgrade-compatibility-tests-run-test-image-current-sha + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Upgrade Nodes with Current SHA and Run ${{ matrix.product.name }} + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: smartcontractkit/ccip + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Download LaneConfig From Last Release + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 + with: + name: lane-config + path: ./integration-tests/ccip-tests/load/lane-config + + - name: Set Override Config + id: set_override_config + run: | + # if the matrix.product.config_path is set, use it as the override config + if [ "${{ matrix.product.config_path }}" != "" ]; then + echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT" + fi + - name: Prepare Base64 TOML override for CCIP secrets + uses: ./.github/actions/setup-create-base64-config-ccip + with: + runId: ${{ github.run_id }} + existingNamespace: ${{ needs.run-test-with-last-release.outputs.existing_namespace }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ github.sha }} + upgradeImage: ${{ env.CHAINLINK_IMAGE }} + upgradeVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" + - name: Run Tests + if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + env: + BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }} + TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }} + ENV_JOB_IMAGE: ${{ env.ENV_JOB_IMAGE_BASE }}:${{ github.sha }} + TEST_SUITE: load + TEST_ARGS: -test.timeout 1h + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable + RR_MEM: 8Gi + RR_CPU: 4 + with: + test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 1h -count=1 -json -run ${{ matrix.product.run }} ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ github.sha }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs + artifacts_location: | + ./integration-tests/ccip-tests/load/tmp_*.json + ./integration-tests/ccip-tests/load/logs/* + publish_check_name: ${{ matrix.product.name }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + should_tidy: "false" + should_cleanup: "true" + triggered_by: ${{ needs.run-test-with-last-release.outputs.triggered_by }} \ No newline at end of file diff --git a/.github/workflows/chain-selectors-check.yml b/.github/workflows/chain-selectors-check.yml new file mode 100644 index 00000000000..633388b319d --- /dev/null +++ b/.github/workflows/chain-selectors-check.yml @@ -0,0 +1,39 @@ +name: Chain Selectors Version Check + +on: + push: + branches: + - ccip-develop + - release/* + tags: + - v* + pull_request: + branches: + - release/* + + +jobs: + verify-version: + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + only-modules: true + go-version-file: "go.mod" + + - name: Get chain-selectors version + id: get-chain-selectors-version + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + current_chain_selector_version=$(go list -m -f '{{.Version}}' github.com/smartcontractkit/chain-selectors) + latest_chain_selector_version=$(gh release view -R smartcontractkit/chain-selectors --json tagName --jq '.tagName') + if [[ "$current_chain_selector_version" != "$latest_chain_selector_version" ]]; then + echo "::error:: Chain Selectors version mismatch. Current version: $current_chain_selector_version, Latest version: $latest_chain_selector_version" + exit 1 + fi