diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 3b91bd251a1..d1e373b55a5 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -1160,9 +1160,10 @@ runner-test-matrix: - E2E CCIP Chaos Tests test_cmd: cd integration-tests/ccip-tests/chaos && DETACH_RUNNER=false go test ccip_test.go -v -test.parallel=11 -timeout 60m -count=1 -json test_env_vars: - TEST_SUITE: chaos + TEST_SUITE: ccip-chaos TEST_TRIGGERED_BY: ccip-cron-chaos-eth TEST_LOG_LEVEL: debug + test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/ccip-chaos.toml - id: ccip-tests/load/ccip_test.go:^TestLoadCCIPStableWithPodChaosDiffCommitAndExec path: integration-tests/ccip-tests/load/ccip_test.go @@ -1173,7 +1174,7 @@ runner-test-matrix: # - E2E CCIP Chaos Tests test_cmd: cd integration-tests/ccip-tests/load && DETACH_RUNNER=false go test -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' -v -test.parallel=4 -timeout 120m -count=1 -json test_env_vars: - TEST_SUITE: chaos + TEST_SUITE: ccip-chaos TEST_TRIGGERED_BY: ccip-cron-chaos-eth TEST_LOG_LEVEL: debug E2E_TEST_GRAFANA_DASHBOARD_URL: /d/6vjVx-1V8/ccip-long-running-tests diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml index a27c100eb16..3a07ecefaa0 100644 --- a/.github/workflows/automation-benchmark-tests.yml +++ b/.github/workflows/automation-benchmark-tests.yml @@ -24,7 +24,7 @@ on: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml test_ids: '${{ inputs.testType }}/automation_test.go:TestAutomationBenchmark' diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml index f538d7b3385..71bfbfa029e 100644 --- a/.github/workflows/automation-load-tests.yml +++ b/.github/workflows/automation-load-tests.yml @@ -19,7 +19,7 @@ on: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml test_ids: 'load/automationv2_1/automationv2_1_test.go:TestLogTrigger' diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml index 7fa25632207..222eac2e75b 100644 --- a/.github/workflows/automation-nightly-tests.yml +++ b/.github/workflows/automation-nightly-tests.yml @@ -10,7 +10,7 @@ on: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml test_trigger: Automation Nightly Tests diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index f94b9f3488a..f85a1330e6e 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -153,7 +153,7 @@ jobs: call-run-e2e-tests-workflow: name: Run E2E Tests needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml test_list: ${{ needs.set-tests-to-run.outputs.test_list }} diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml index 742d1561e1e..cf101ce4132 100644 --- a/.github/workflows/ccip-chaos-tests.yml +++ b/.github/workflows/ccip-chaos-tests.yml @@ -1,5 +1,6 @@ name: CCIP Chaos Tests on: + pull_request: # DEBUG: Checking why we're not running properly workflow_run: workflows: [ CCIP Load Test ] types: [ completed ] @@ -14,16 +15,16 @@ concurrency: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 # v0.2.0 with: test_path: .github/e2e-tests.yml - chainlink_version: ${{ github.sha }} - require_chainlink_image_versions_in_qa_ecr: ${{ github.sha }} test_trigger: E2E CCIP Chaos Tests + chainlink_version: ${{ github.sha }} test_log_level: debug - slack_notification_after_tests: on_failure + slack_notification_after_tests: never # DEBUG: avoid spam slack_notification_after_tests_channel_id: '#ccip-testing' slack_notification_after_tests_name: CCIP Chaos E2E Tests + test_image_suites: ccip-chaos secrets: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml index a549cddd55f..af06571909b 100644 --- a/.github/workflows/ccip-load-tests.yml +++ b/.github/workflows/ccip-load-tests.yml @@ -31,7 +31,7 @@ concurrency: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml test_trigger: E2E CCIP Load Tests diff --git a/.github/workflows/integration-chaos-tests.yml b/.github/workflows/integration-chaos-tests.yml index b7a695b887b..3be161005f8 100644 --- a/.github/workflows/integration-chaos-tests.yml +++ b/.github/workflows/integration-chaos-tests.yml @@ -10,7 +10,7 @@ on: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml chainlink_version: ${{ github.sha }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6aeb9698136..e70e9df621a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -210,7 +210,7 @@ jobs: contents: read needs: [build-chainlink, changes] if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true') - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: workflow_name: Run Core E2E Tests For PR chainlink_version: ${{ inputs.evm-ref || github.sha }} @@ -251,7 +251,7 @@ jobs: contents: read needs: [build-chainlink, changes] if: github.event_name == 'merge_group' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true') - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: workflow_name: Run Core E2E Tests For Merge Queue chainlink_version: ${{ inputs.evm-ref || github.sha }} @@ -296,7 +296,7 @@ jobs: contents: read needs: [build-chainlink, changes] if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true') - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: workflow_name: Run CCIP E2E Tests For PR chainlink_version: ${{ inputs.evm-ref || github.sha }} @@ -337,7 +337,7 @@ jobs: contents: read needs: [build-chainlink, changes] if: github.event_name == 'merge_group' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true') - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: workflow_name: Run CCIP E2E Tests For Merge Queue chainlink_version: ${{ inputs.evm-ref || github.sha }} diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml index b0ec88ff30b..a96c000fa44 100644 --- a/.github/workflows/on-demand-ocr-soak-test.yml +++ b/.github/workflows/on-demand-ocr-soak-test.yml @@ -39,7 +39,7 @@ on: jobs: run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: test_path: .github/e2e-tests.yml test_ids: ${{ inputs.testToRun}} diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml index e4f8665400a..0a9cec3db93 100644 --- a/.github/workflows/on-demand-vrfv2-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2-performance-test.yml @@ -67,7 +67,7 @@ jobs: run-e2e-tests-workflow: name: Run E2E Tests needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }} chainlink_version: ${{ inputs.chainlink_version }} diff --git a/.github/workflows/on-demand-vrfv2-smoke-tests.yml b/.github/workflows/on-demand-vrfv2-smoke-tests.yml index 7169b0e4ab3..fcb083df708 100644 --- a/.github/workflows/on-demand-vrfv2-smoke-tests.yml +++ b/.github/workflows/on-demand-vrfv2-smoke-tests.yml @@ -70,7 +70,7 @@ jobs: run-e2e-tests-workflow: name: Run E2E Tests needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }} chainlink_version: ${{ inputs.chainlink_version }} diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml index 8dbf325ced9..534aea3d27d 100644 --- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml @@ -67,7 +67,7 @@ jobs: run-e2e-tests-workflow: name: Run E2E Tests needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }} chainlink_version: ${{ inputs.chainlink_version }} diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml index 7cb7b8400da..9137cc64d98 100644 --- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml +++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml @@ -70,7 +70,7 @@ jobs: run-e2e-tests-workflow: name: Run E2E Tests needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }} chainlink_version: ${{ inputs.chainlink_version }} diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml index 25305955b3d..0637363ca76 100644 --- a/.github/workflows/run-nightly-e2e-tests.yml +++ b/.github/workflows/run-nightly-e2e-tests.yml @@ -20,7 +20,7 @@ on: jobs: call-run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: chainlink_version: ${{ inputs.chainlink_version || 'develop' }} test_path: .github/e2e-tests.yml diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml index 754e821c14a..b53bc756f4d 100644 --- a/.github/workflows/run-selected-e2e-tests.yml +++ b/.github/workflows/run-selected-e2e-tests.yml @@ -35,7 +35,7 @@ run-name: ${{ inputs.workflow_run_name }} jobs: call-run-e2e-tests-workflow: name: Run E2E Tests - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d + uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 with: chainlink_version: ${{ github.event.inputs.chainlink_version }} test_path: .github/e2e-tests.yml diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go index da2e870f3ec..cdd85ce9876 100644 --- a/integration-tests/ccip-tests/actions/ccip_helpers.go +++ b/integration-tests/ccip-tests/actions/ccip_helpers.go @@ -3651,7 +3651,7 @@ func (lane *CCIPLane) DeployNewCCIPLane( execNodes := clNodes[env.ExecNodeStartIndex : env.ExecNodeStartIndex+env.NumOfExecNodes] if !commitAndExecOnSameDON { if len(clNodes) < 11 { - return fmt.Errorf("not enough CL nodes for separate commit and execution nodes") + return fmt.Errorf("not enough CL nodes for separate commit and execution nodes, need at least 11 CL nodes, found %d", len(clNodes)) } bootstrapExec = clNodes[1] // for a set-up of different commit and execution nodes second node is the bootstrapper for execution nodes } diff --git a/integration-tests/ccip-tests/testconfig/tomls/ccip-chaos.toml b/integration-tests/ccip-tests/testconfig/tomls/ccip-chaos.toml new file mode 100644 index 00000000000..da50ff07c77 --- /dev/null +++ b/integration-tests/ccip-tests/testconfig/tomls/ccip-chaos.toml @@ -0,0 +1,4 @@ +[CCIP] +[CCIP.Env] +[CCIP.Env.NewCLCluster] +NoOfNodes = 12 diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index eee424d50d1..3f5b6a0699d 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -1363,19 +1363,19 @@ func (o *CCIPTestSetUpOutputs) CreateEnvironment( ccipEnv.NumOfCommitNodes = testConfig.TestGroupInput.NoOfCommitNodes ccipEnv.NumOfExecNodes = ccipEnv.NumOfCommitNodes if !pointer.GetBool(testConfig.TestGroupInput.CommitAndExecuteOnSameDON) { - if len(ccipEnv.CLNodesWithKeys) < 11 { - return fmt.Errorf("not enough CL nodes for separate commit and execution nodes") + if len(ccipEnv.CLNodesWithKeys[chains[0].GetChainID().String()]) < 11 { + return fmt.Errorf("not enough CL nodes for separate commit and execution nodes, need at least 11 CL nodes, found %d", len(ccipEnv.CLNodesWithKeys)) } if testConfig.TestGroupInput.NoOfCommitNodes >= totalNodes { - return fmt.Errorf("number of commit nodes can not be greater than total number of nodes in DON") + return fmt.Errorf("number of commit nodes can not be greater than total number of nodes in DON, found %d commit nodes and %d total nodes", testConfig.TestGroupInput.NoOfCommitNodes, totalNodes) + } + if ccipEnv.NumOfExecNodes < 4 { + return fmt.Errorf("insufficient number of exec nodes, need at least 4 exec nodes, found %d", ccipEnv.NumOfExecNodes) } // first two nodes are reserved for bootstrap commit and bootstrap exec ccipEnv.CommitNodeStartIndex = 2 ccipEnv.ExecNodeStartIndex = 2 + testConfig.TestGroupInput.NoOfCommitNodes ccipEnv.NumOfExecNodes = totalNodes - (2 + testConfig.TestGroupInput.NoOfCommitNodes) - if ccipEnv.NumOfExecNodes < 4 { - return fmt.Errorf("insufficient number of exec nodes") - } } ccipEnv.NumOfAllowedFaultyExec = (ccipEnv.NumOfExecNodes - 1) / 3 ccipEnv.NumOfAllowedFaultyCommit = (ccipEnv.NumOfCommitNodes - 1) / 3 diff --git a/integration-tests/scripts/buildTests b/integration-tests/scripts/buildTests index 7be2d0c5a7e..947b1502b61 100755 --- a/integration-tests/scripts/buildTests +++ b/integration-tests/scripts/buildTests @@ -30,6 +30,11 @@ do pushd "./ccip-tests/load" && go test -c -tags embed -o ../ ./... mv ../load.test ../ccip-load.test # rename the binary to match the suite name popd + elif [ "$x" = "ccip-chaos" ]; then + echo "Changing directory and executing go test -c ./... for 'ccip-chaos' package" + pushd "./ccip-tests/chaos" && go test -c -tags embed -o ../ ./... + mv ../chaos.test ../ccip-chaos.test # rename the binary to match the suite name + popd else go test -c -tags embed ./"${x}" fi diff --git a/integration-tests/test.Dockerfile b/integration-tests/test.Dockerfile index 6252cfdd426..39c2ec8e164 100644 --- a/integration-tests/test.Dockerfile +++ b/integration-tests/test.Dockerfile @@ -12,7 +12,7 @@ RUN cd integration-tests/load && go mod download COPY . . -ARG SUITES=chaos soak benchmark load ccip-load +ARG SUITES=chaos soak benchmark load ccip-load ccip-chaos RUN /go/testdir/integration-tests/scripts/buildTests "${SUITES}"