[AUTO-11176] fix automation chaos test (#13904)

* fix automation chaos test

* fix ondemand workflow for chaos tests

* use standard automation test functions

* update automation testconfig

* lint

* lint

.github/workflows/automation-ondemand-tests.yml

@@ -146,7 +146,7 @@ jobs:
           - name: chaos
             id: chaos
             suite: chaos
-            nodes: 15
+            nodes: 20
             os: ubuntu-latest
             enabled: ${{ inputs.enableChaos }}
             pyroscope_env: ci-automation-on-demand-chaos

integration-tests/chaos/automation_chaos_test.go

@@ -6,6 +6,11 @@ import (
 	"testing"
 	"time"
 
+	ocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper"
+
+	ocr2keepers30config "github.com/smartcontractkit/chainlink-automation/pkg/v3/config"
+	"github.com/smartcontractkit/chainlink/integration-tests/actions/automationv2"
+
 	seth_utils "github.com/smartcontractkit/chainlink-testing-framework/utils/seth"
 
 	"github.com/onsi/gomega"
@@ -77,22 +82,6 @@ ListenAddresses = ["0.0.0.0:6690"]`
 			},
 		},
 	}
-
-	defaultOCRRegistryConfig = contracts.KeeperRegistrySettings{
-		PaymentPremiumPPB:    uint32(200000000),
-		FlatFeeMicroLINK:     uint32(0),
-		BlockCountPerTurn:    big.NewInt(10),
-		CheckGasLimit:        uint32(2500000),
-		StalenessSeconds:     big.NewInt(90000),
-		GasCeilingMultiplier: uint16(1),
-		MinUpkeepSpend:       big.NewInt(0),
-		MaxPerformGas:        uint32(5000000),
-		FallbackGasPrice:     big.NewInt(2e11),
-		FallbackLinkPrice:    big.NewInt(2e18),
-		MaxCheckDataSize:     uint32(5000),
-		MaxPerformDataSize:   uint32(5000),
-		MaxRevertDataSize:    uint32(5000),
-	}
 )
 
 func getDefaultAutomationSettings(config *tc.TestConfig) map[string]interface{} {
@@ -127,6 +116,7 @@ func TestAutomationChaos(t *testing.T) {
 		"registry_2_0": eth_contracts.RegistryVersion_2_0,
 		"registry_2_1": eth_contracts.RegistryVersion_2_1,
 		"registry_2_2": eth_contracts.RegistryVersion_2_2,
+		"registry_2_3": eth_contracts.RegistryVersion_2_3,
 	}
 
 	for name, registryVersion := range registryVersions {
@@ -253,57 +243,76 @@ func TestAutomationChaos(t *testing.T) {
 						require.NoError(t, err, "Error tearing down environment")
 					})
 
-					txCost, err := actions.EstimateCostForChainlinkOperations(l, chainClient, network, 1000)
-					require.NoError(t, err, "Error estimating cost for Chainlink Operations")
-					err = actions.FundChainlinkNodesFromRootAddress(l, chainClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes), txCost)
-					require.NoError(t, err, "Error funding Chainlink nodes")
-
-					linkToken, err := contracts.DeployLinkTokenContract(l, chainClient)
-					require.NoError(t, err, "Error deploying LINK token")
-
-					wethToken, err := contracts.DeployWETHTokenContract(l, chainClient)
-					require.NoError(t, err, "Error deploying weth token contract")
-
-					// This feed is used for both eth/usd and link/usd
-					ethUSDFeed, err := contracts.DeployMockETHUSDFeed(chainClient, defaultOCRRegistryConfig.FallbackLinkPrice)
-					require.NoError(t, err, "Error deploying eth usd feed contract")
-
-					registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar(
-						t,
-						chainClient,
-						rv,
-						defaultOCRRegistryConfig,
-						linkToken,
-						wethToken,
-						ethUSDFeed,
-					)
-
-					// Fund the registry with LINK
-					err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps))))
-					require.NoError(t, err, "Funding keeper registry contract shouldn't fail")
-
-					actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, rv)
-					nodesWithoutBootstrap := chainlinkNodes[1:]
-					defaultOCRRegistryConfig.RegistryVersion = rv
-					ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, defaultOCRRegistryConfig, registrar.Address(), 30*time.Second, registry.ChainModuleAddress(), registry.ReorgProtectionEnabled(), linkToken, wethToken, ethUSDFeed)
-					require.NoError(t, err, "Error building OCR config vars")
-
-					if rv == eth_contracts.RegistryVersion_2_0 {
-						err = registry.SetConfig(defaultOCRRegistryConfig, ocrConfig)
-					} else {
-						err = registry.SetConfigTypeSafe(ocrConfig)
+					a := automationv2.NewAutomationTestK8s(l, chainClient, chainlinkNodes)
+					a.SetMercuryCredentialName("cred1")
+					conf := config.Automation.AutomationConfig
+					a.RegistrySettings = contracts.KeeperRegistrySettings{
+						PaymentPremiumPPB:    *conf.RegistrySettings.PaymentPremiumPPB,
+						FlatFeeMicroLINK:     *conf.RegistrySettings.FlatFeeMicroLINK,
+						CheckGasLimit:        *conf.RegistrySettings.CheckGasLimit,
+						StalenessSeconds:     conf.RegistrySettings.StalenessSeconds,
+						GasCeilingMultiplier: *conf.RegistrySettings.GasCeilingMultiplier,
+						MaxPerformGas:        *conf.RegistrySettings.MaxPerformGas,
+						MinUpkeepSpend:       conf.RegistrySettings.MinUpkeepSpend,
+						FallbackGasPrice:     conf.RegistrySettings.FallbackGasPrice,
+						FallbackLinkPrice:    conf.RegistrySettings.FallbackLinkPrice,
+						MaxCheckDataSize:     *conf.RegistrySettings.MaxCheckDataSize,
+						MaxPerformDataSize:   *conf.RegistrySettings.MaxPerformDataSize,
+						MaxRevertDataSize:    *conf.RegistrySettings.MaxRevertDataSize,
+						RegistryVersion:      rv,
+					}
+					a.RegistrarSettings = contracts.KeeperRegistrarSettings{
+						AutoApproveConfigType: uint8(2),
+						AutoApproveMaxAllowed: 1000,
+						MinLinkJuels:          big.NewInt(0),
+					}
+					plCfg := config.GetAutomationConfig().AutomationConfig.PluginConfig
+					a.PluginConfig = ocr2keepers30config.OffchainConfig{
+						TargetProbability:    *plCfg.TargetProbability,
+						TargetInRounds:       *plCfg.TargetInRounds,
+						PerformLockoutWindow: *plCfg.PerformLockoutWindow,
+						GasLimitPerReport:    *plCfg.GasLimitPerReport,
+						GasOverheadPerUpkeep: *plCfg.GasOverheadPerUpkeep,
+						MinConfirmations:     *plCfg.MinConfirmations,
+						MaxUpkeepBatchSize:   *plCfg.MaxUpkeepBatchSize,
+					}
+					pubCfg := config.GetAutomationConfig().AutomationConfig.PublicConfig
+					a.PublicConfig = ocr3.PublicConfig{
+						DeltaProgress:                           *pubCfg.DeltaProgress,
+						DeltaResend:                             *pubCfg.DeltaResend,
+						DeltaInitial:                            *pubCfg.DeltaInitial,
+						DeltaRound:                              *pubCfg.DeltaRound,
+						DeltaGrace:                              *pubCfg.DeltaGrace,
+						DeltaCertifiedCommitRequest:             *pubCfg.DeltaCertifiedCommitRequest,
+						DeltaStage:                              *pubCfg.DeltaStage,
+						RMax:                                    *pubCfg.RMax,
+						MaxDurationQuery:                        *pubCfg.MaxDurationQuery,
+						MaxDurationObservation:                  *pubCfg.MaxDurationObservation,
+						MaxDurationShouldAcceptAttestedReport:   *pubCfg.MaxDurationShouldAcceptAttestedReport,
+						MaxDurationShouldTransmitAcceptedReport: *pubCfg.MaxDurationShouldTransmitAcceptedReport,
+						F:                                       *pubCfg.F,
 					}
-					require.NoError(t, err, "Error setting OCR config")
 
-					consumersConditional, upkeepidsConditional := actions.DeployConsumers(t, chainClient, registry, registrar, linkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, false, false, false, nil)
-					consumersLogtrigger, upkeepidsLogtrigger := actions.DeployConsumers(t, chainClient, registry, registrar, linkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, true, false, false, nil)
+					a.SetupAutomationDeployment(t)
 
-					consumers := append(consumersConditional, consumersLogtrigger...)
-					upkeepIDs := append(upkeepidsConditional, upkeepidsLogtrigger...)
+					err = actions.FundChainlinkNodesFromRootAddress(l, a.ChainClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes[1:]), big.NewFloat(*config.Common.ChainlinkNodeFunding))
+					require.NoError(t, err, "Error funding Chainlink nodes")
+
+					var consumersLogTrigger, consumersConditional []contracts.KeeperConsumer
+					var upkeepidsConditional, upkeepidsLogTrigger []*big.Int
+					consumersConditional, upkeepidsConditional = actions.DeployConsumers(t, a.ChainClient, a.Registry, a.Registrar, a.LinkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, false, false, false, nil)
+					consumers := consumersConditional
+					upkeepIDs := upkeepidsConditional
+					if rv >= eth_contracts.RegistryVersion_2_1 {
+						consumersLogTrigger, upkeepidsLogTrigger = actions.DeployConsumers(t, a.ChainClient, a.Registry, a.Registrar, a.LinkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, true, false, false, nil)
 
-					for _, c := range consumersLogtrigger {
-						err = c.Start()
-						require.NoError(t, err, "Error starting consumer")
+						consumers = append(consumersConditional, consumersLogTrigger...)
+						upkeepIDs = append(upkeepidsConditional, upkeepidsLogTrigger...)
+
+						for _, c := range consumersLogTrigger {
+							err = c.Start()
+							require.NoError(t, err, "Error starting consumer")
+						}
 					}
 
 					l.Info().Msg("Waiting for all upkeeps to be performed")
@@ -324,6 +333,13 @@ func TestAutomationChaos(t *testing.T) {
 					_, err = testEnvironment.Chaos.Run(testCase.chaosFunc(testEnvironment.Cfg.Namespace, testCase.chaosProps))
 					require.NoError(t, err)
 
+					if rv >= eth_contracts.RegistryVersion_2_1 {
+						for _, c := range consumersLogTrigger {
+							err = c.Start()
+							require.NoError(t, err, "Error starting consumer")
+						}
+					}
+
 					gom.Eventually(func(g gomega.Gomega) {
 						// Check if the upkeeps are performing multiple times by analyzing their counters and checking they are greater than 10
 						for i := 0; i < len(upkeepIDs); i++ {
@@ -334,7 +350,7 @@ func TestAutomationChaos(t *testing.T) {
 							g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
 								"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
 						}
-					}, "3m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~2m for performing each upkeep 5 times, ~2m buffer
+					}, "5m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~2m for performing each upkeep 5 times, ~2m buffer
 				})
 			}
 

integration-tests/testconfig/automation/automation.toml

@@ -152,6 +152,57 @@ max_check_data_size=5_000
 max_perform_data_size=5_000
 max_revert_data_size=5_000
 
+# chaos test specific overrides
+[Chaos.Common]
+chainlink_node_funding = 2.0
+
+[Chaos.Automation]
+
+[Chaos.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Chaos.Automation.AutomationConfig.PublicConfig]
+delta_progress=10_000_000_000
+delta_resend=15_000_000_000
+delta_initial=500_000_000
+delta_round=1_000_000_000
+delta_grace=200_000_000
+delta_certified_commit_request=300_000_000
+delta_stage=30_000_000_000
+r_max=24
+f=1
+max_duration_query=20_000_000
+max_duration_observation=20_000_000
+max_duration_should_accept_attested_report=1_200_000_000
+max_duration_should_transmit_accepted_report=20_000_000
+
+[Chaos.Automation.AutomationConfig.PluginConfig]
+perform_lockout_window=3_600_000
+target_probability="0.999"
+target_in_rounds=1
+min_confirmations=0
+gas_limit_per_report=10_300_000
+gas_overhead_per_upkeep=300_000
+max_upkeep_batch_size=10
+
+[Chaos.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Chaos.Automation.AutomationConfig.RegistrySettings]
+payment_premium_ppb=200_000_000
+flat_fee_micro_link=0
+check_gas_limit=2_500_000
+staleness_seconds=90000
+gas_ceiling_multiplier=1
+max_perform_gas=5_000_000
+min_upkeep_spend=0
+fallback_gas_price=200_000_000_000
+fallback_link_price=2_000_000_000_000_000_000
+max_check_data_size=5_000
+max_perform_data_size=5_000
+max_revert_data_size=5_000
+
 # load test specific overrides
 [Load.Seth]
 ephemeral_addresses_number = 100