[TT-1771] Proper CCIP Chaos Tests

.github/e2e-tests.yml

@@ -1160,9 +1160,10 @@ runner-test-matrix:
       - E2E CCIP Chaos Tests
     test_cmd: cd integration-tests/ccip-tests/chaos && DETACH_RUNNER=false go test ccip_test.go -v -test.parallel=11 -timeout 60m -count=1 -json
     test_env_vars:
-      TEST_SUITE: chaos
+      TEST_SUITE: ccip-chaos
       TEST_TRIGGERED_BY: ccip-cron-chaos-eth
       TEST_LOG_LEVEL: debug
+    test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/ccip-chaos.toml
 
   - id: ccip-tests/load/ccip_test.go:^TestLoadCCIPStableWithPodChaosDiffCommitAndExec
     path: integration-tests/ccip-tests/load/ccip_test.go
@@ -1173,7 +1174,7 @@ runner-test-matrix:
       # - E2E CCIP Chaos Tests
     test_cmd: cd integration-tests/ccip-tests/load && DETACH_RUNNER=false go test -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' -v -test.parallel=4 -timeout 120m -count=1 -json
     test_env_vars:
-      TEST_SUITE: chaos
+      TEST_SUITE: ccip-chaos
       TEST_TRIGGERED_BY: ccip-cron-chaos-eth
       TEST_LOG_LEVEL: debug
       E2E_TEST_GRAFANA_DASHBOARD_URL: /d/6vjVx-1V8/ccip-long-running-tests

.github/workflows/automation-benchmark-tests.yml

@@ -24,7 +24,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       test_ids: '${{ inputs.testType }}/automation_test.go:TestAutomationBenchmark'

.github/workflows/automation-load-tests.yml

@@ -19,7 +19,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       test_ids: 'load/automationv2_1/automationv2_1_test.go:TestLogTrigger'

.github/workflows/automation-nightly-tests.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       test_trigger: Automation Nightly Tests

.github/workflows/automation-ondemand-tests.yml

@@ -153,7 +153,7 @@ jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       test_list: ${{ needs.set-tests-to-run.outputs.test_list }}

.github/workflows/ccip-chaos-tests.yml

@@ -1,5 +1,6 @@
 name: CCIP Chaos Tests
 on:
+  pull_request: # DEBUG: Checking why we're not running properly
   workflow_run:
     workflows: [ CCIP Load Test ]
     types: [ completed ]
@@ -14,16 +15,16 @@ concurrency:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229 # v0.2.0
     with:
       test_path: .github/e2e-tests.yml
-      chainlink_version: ${{ github.sha }}
-      require_chainlink_image_versions_in_qa_ecr: ${{ github.sha }}
       test_trigger: E2E CCIP Chaos Tests
+      chainlink_version: ${{ github.sha }}
       test_log_level: debug
-      slack_notification_after_tests: on_failure
+      slack_notification_after_tests: never # DEBUG: avoid spam
       slack_notification_after_tests_channel_id: '#ccip-testing'
       slack_notification_after_tests_name: CCIP Chaos E2E Tests
+      test_image_suites: ccip-chaos
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

.github/workflows/ccip-load-tests.yml

@@ -31,7 +31,7 @@ concurrency:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       test_trigger: E2E CCIP Load Tests

.github/workflows/integration-chaos-tests.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       chainlink_version: ${{ github.sha }}

.github/workflows/integration-tests.yml

@@ -210,7 +210,7 @@ jobs:
       contents: read
     needs: [build-chainlink, changes]
     if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       workflow_name: Run Core E2E Tests For PR
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
@@ -251,7 +251,7 @@ jobs:
       contents: read
     needs: [build-chainlink, changes]
     if: github.event_name == 'merge_group' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       workflow_name: Run Core E2E Tests For Merge Queue
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
@@ -296,7 +296,7 @@ jobs:
       contents: read
     needs: [build-chainlink, changes]
     if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       workflow_name: Run CCIP E2E Tests For PR
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
@@ -337,7 +337,7 @@ jobs:
       contents: read
     needs: [build-chainlink, changes]
     if: github.event_name == 'merge_group' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       workflow_name: Run CCIP E2E Tests For Merge Queue
       chainlink_version: ${{ inputs.evm-ref || github.sha }}

.github/workflows/on-demand-ocr-soak-test.yml

@@ -39,7 +39,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       test_path: .github/e2e-tests.yml
       test_ids: ${{ inputs.testToRun}}

.github/workflows/on-demand-vrfv2-performance-test.yml

@@ -67,7 +67,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

.github/workflows/on-demand-vrfv2-smoke-tests.yml

@@ -70,7 +70,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

.github/workflows/on-demand-vrfv2plus-performance-test.yml

@@ -67,7 +67,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

.github/workflows/on-demand-vrfv2plus-smoke-tests.yml

@@ -70,7 +70,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

.github/workflows/run-nightly-e2e-tests.yml

@@ -20,7 +20,7 @@ on:
 jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       chainlink_version: ${{ inputs.chainlink_version || 'develop' }}
       test_path: .github/e2e-tests.yml

.github/workflows/run-selected-e2e-tests.yml

@@ -35,7 +35,7 @@ run-name: ${{ inputs.workflow_run_name }}
 jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@827d85b57065211ddd254457a7cf21581d85092d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f1f2dac0a20f0e02408eb7f528c768fe95c39229
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
       test_path: .github/e2e-tests.yml

integration-tests/ccip-tests/actions/ccip_helpers.go

@@ -3651,7 +3651,7 @@ func (lane *CCIPLane) DeployNewCCIPLane(
 	execNodes := clNodes[env.ExecNodeStartIndex : env.ExecNodeStartIndex+env.NumOfExecNodes]
 	if !commitAndExecOnSameDON {
 		if len(clNodes) < 11 {
-			return fmt.Errorf("not enough CL nodes for separate commit and execution nodes")
+			return fmt.Errorf("not enough CL nodes for separate commit and execution nodes, need at least 11 CL nodes, found %d", len(clNodes))
 		}
 		bootstrapExec = clNodes[1] // for a set-up of different commit and execution nodes second node is the bootstrapper for execution nodes
 	}

integration-tests/ccip-tests/testconfig/tomls/ccip-chaos.toml

@@ -0,0 +1,4 @@
+[CCIP]
+[CCIP.Env]
+[CCIP.Env.NewCLCluster]
+NoOfNodes = 12

integration-tests/ccip-tests/testsetups/ccip.go

@@ -1363,19 +1363,19 @@ func (o *CCIPTestSetUpOutputs) CreateEnvironment(
 			ccipEnv.NumOfCommitNodes = testConfig.TestGroupInput.NoOfCommitNodes
 			ccipEnv.NumOfExecNodes = ccipEnv.NumOfCommitNodes
 			if !pointer.GetBool(testConfig.TestGroupInput.CommitAndExecuteOnSameDON) {
-				if len(ccipEnv.CLNodesWithKeys) < 11 {
-					return fmt.Errorf("not enough CL nodes for separate commit and execution nodes")
+				if len(ccipEnv.CLNodesWithKeys[chains[0].GetChainID().String()]) < 11 {
+					return fmt.Errorf("not enough CL nodes for separate commit and execution nodes, need at least 11 CL nodes, found %d", len(ccipEnv.CLNodesWithKeys))
 				}
 				if testConfig.TestGroupInput.NoOfCommitNodes >= totalNodes {
-					return fmt.Errorf("number of commit nodes can not be greater than total number of nodes in DON")
+					return fmt.Errorf("number of commit nodes can not be greater than total number of nodes in DON, found %d commit nodes and %d total nodes", testConfig.TestGroupInput.NoOfCommitNodes, totalNodes)
+				}
+				if ccipEnv.NumOfExecNodes < 4 {
+					return fmt.Errorf("insufficient number of exec nodes, need at least 4 exec nodes, found %d", ccipEnv.NumOfExecNodes)
 				}
 				// first two nodes are reserved for bootstrap commit and bootstrap exec
 				ccipEnv.CommitNodeStartIndex = 2
 				ccipEnv.ExecNodeStartIndex = 2 + testConfig.TestGroupInput.NoOfCommitNodes
 				ccipEnv.NumOfExecNodes = totalNodes - (2 + testConfig.TestGroupInput.NoOfCommitNodes)
-				if ccipEnv.NumOfExecNodes < 4 {
-					return fmt.Errorf("insufficient number of exec nodes")
-				}
 			}
 			ccipEnv.NumOfAllowedFaultyExec = (ccipEnv.NumOfExecNodes - 1) / 3
 			ccipEnv.NumOfAllowedFaultyCommit = (ccipEnv.NumOfCommitNodes - 1) / 3

integration-tests/scripts/buildTests

@@ -30,6 +30,11 @@ do
       pushd "./ccip-tests/load" && go test -c -tags embed -o ../ ./...
       mv ../load.test ../ccip-load.test # rename the binary to match the suite name
       popd
+    elif [ "$x" = "ccip-chaos" ]; then
+      echo "Changing directory and executing go test -c ./... for 'ccip-chaos' package"
+      pushd "./ccip-tests/chaos" && go test -c -tags embed -o ../ ./...
+      mv ../chaos.test ../ccip-chaos.test # rename the binary to match the suite name
+      popd
     else
       go test -c -tags embed ./"${x}"
     fi

integration-tests/test.Dockerfile

@@ -12,7 +12,7 @@ RUN cd integration-tests/load && go mod download
 
 COPY . .
 
-ARG SUITES=chaos soak benchmark load ccip-load
+ARG SUITES=chaos soak benchmark load ccip-load ccip-chaos
 
 RUN /go/testdir/integration-tests/scripts/buildTests "${SUITES}"