From 2207a38326f515e398c3b615f535da53b7498e7d Mon Sep 17 00:00:00 2001 From: Anirudh Warrier <12178754+anirudhwarrier@users.noreply.github.com> Date: Fri, 19 Jul 2024 21:16:05 +0530 Subject: [PATCH] [AUTO-11176] fix automation chaos test (#13904) * fix automation chaos test * fix ondemand workflow for chaos tests * use standard automation test functions * update automation testconfig * lint * lint --- .../workflows/automation-ondemand-tests.yml | 2 +- .../chaos/automation_chaos_test.go | 144 ++++++++++-------- .../testconfig/automation/automation.toml | 51 +++++++ 3 files changed, 132 insertions(+), 65 deletions(-) diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index 324d5b04b9b..7514743fa88 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -146,7 +146,7 @@ jobs: - name: chaos id: chaos suite: chaos - nodes: 15 + nodes: 20 os: ubuntu-latest enabled: ${{ inputs.enableChaos }} pyroscope_env: ci-automation-on-demand-chaos diff --git a/integration-tests/chaos/automation_chaos_test.go b/integration-tests/chaos/automation_chaos_test.go index 79513b2e8fb..467f371ad18 100644 --- a/integration-tests/chaos/automation_chaos_test.go +++ b/integration-tests/chaos/automation_chaos_test.go @@ -6,6 +6,11 @@ import ( "testing" "time" + ocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + + ocr2keepers30config "github.com/smartcontractkit/chainlink-automation/pkg/v3/config" + "github.com/smartcontractkit/chainlink/integration-tests/actions/automationv2" + seth_utils "github.com/smartcontractkit/chainlink-testing-framework/utils/seth" "github.com/onsi/gomega" @@ -77,22 +82,6 @@ ListenAddresses = ["0.0.0.0:6690"]` }, }, } - - defaultOCRRegistryConfig = contracts.KeeperRegistrySettings{ - PaymentPremiumPPB: uint32(200000000), - FlatFeeMicroLINK: uint32(0), - BlockCountPerTurn: big.NewInt(10), - CheckGasLimit: uint32(2500000), - StalenessSeconds: big.NewInt(90000), - GasCeilingMultiplier: uint16(1), - MinUpkeepSpend: big.NewInt(0), - MaxPerformGas: uint32(5000000), - FallbackGasPrice: big.NewInt(2e11), - FallbackLinkPrice: big.NewInt(2e18), - MaxCheckDataSize: uint32(5000), - MaxPerformDataSize: uint32(5000), - MaxRevertDataSize: uint32(5000), - } ) func getDefaultAutomationSettings(config *tc.TestConfig) map[string]interface{} { @@ -127,6 +116,7 @@ func TestAutomationChaos(t *testing.T) { "registry_2_0": eth_contracts.RegistryVersion_2_0, "registry_2_1": eth_contracts.RegistryVersion_2_1, "registry_2_2": eth_contracts.RegistryVersion_2_2, + "registry_2_3": eth_contracts.RegistryVersion_2_3, } for name, registryVersion := range registryVersions { @@ -253,57 +243,76 @@ func TestAutomationChaos(t *testing.T) { require.NoError(t, err, "Error tearing down environment") }) - txCost, err := actions.EstimateCostForChainlinkOperations(l, chainClient, network, 1000) - require.NoError(t, err, "Error estimating cost for Chainlink Operations") - err = actions.FundChainlinkNodesFromRootAddress(l, chainClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes), txCost) - require.NoError(t, err, "Error funding Chainlink nodes") - - linkToken, err := contracts.DeployLinkTokenContract(l, chainClient) - require.NoError(t, err, "Error deploying LINK token") - - wethToken, err := contracts.DeployWETHTokenContract(l, chainClient) - require.NoError(t, err, "Error deploying weth token contract") - - // This feed is used for both eth/usd and link/usd - ethUSDFeed, err := contracts.DeployMockETHUSDFeed(chainClient, defaultOCRRegistryConfig.FallbackLinkPrice) - require.NoError(t, err, "Error deploying eth usd feed contract") - - registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar( - t, - chainClient, - rv, - defaultOCRRegistryConfig, - linkToken, - wethToken, - ethUSDFeed, - ) - - // Fund the registry with LINK - err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps)))) - require.NoError(t, err, "Funding keeper registry contract shouldn't fail") - - actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, rv) - nodesWithoutBootstrap := chainlinkNodes[1:] - defaultOCRRegistryConfig.RegistryVersion = rv - ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, defaultOCRRegistryConfig, registrar.Address(), 30*time.Second, registry.ChainModuleAddress(), registry.ReorgProtectionEnabled(), linkToken, wethToken, ethUSDFeed) - require.NoError(t, err, "Error building OCR config vars") - - if rv == eth_contracts.RegistryVersion_2_0 { - err = registry.SetConfig(defaultOCRRegistryConfig, ocrConfig) - } else { - err = registry.SetConfigTypeSafe(ocrConfig) + a := automationv2.NewAutomationTestK8s(l, chainClient, chainlinkNodes) + a.SetMercuryCredentialName("cred1") + conf := config.Automation.AutomationConfig + a.RegistrySettings = contracts.KeeperRegistrySettings{ + PaymentPremiumPPB: *conf.RegistrySettings.PaymentPremiumPPB, + FlatFeeMicroLINK: *conf.RegistrySettings.FlatFeeMicroLINK, + CheckGasLimit: *conf.RegistrySettings.CheckGasLimit, + StalenessSeconds: conf.RegistrySettings.StalenessSeconds, + GasCeilingMultiplier: *conf.RegistrySettings.GasCeilingMultiplier, + MaxPerformGas: *conf.RegistrySettings.MaxPerformGas, + MinUpkeepSpend: conf.RegistrySettings.MinUpkeepSpend, + FallbackGasPrice: conf.RegistrySettings.FallbackGasPrice, + FallbackLinkPrice: conf.RegistrySettings.FallbackLinkPrice, + MaxCheckDataSize: *conf.RegistrySettings.MaxCheckDataSize, + MaxPerformDataSize: *conf.RegistrySettings.MaxPerformDataSize, + MaxRevertDataSize: *conf.RegistrySettings.MaxRevertDataSize, + RegistryVersion: rv, + } + a.RegistrarSettings = contracts.KeeperRegistrarSettings{ + AutoApproveConfigType: uint8(2), + AutoApproveMaxAllowed: 1000, + MinLinkJuels: big.NewInt(0), + } + plCfg := config.GetAutomationConfig().AutomationConfig.PluginConfig + a.PluginConfig = ocr2keepers30config.OffchainConfig{ + TargetProbability: *plCfg.TargetProbability, + TargetInRounds: *plCfg.TargetInRounds, + PerformLockoutWindow: *plCfg.PerformLockoutWindow, + GasLimitPerReport: *plCfg.GasLimitPerReport, + GasOverheadPerUpkeep: *plCfg.GasOverheadPerUpkeep, + MinConfirmations: *plCfg.MinConfirmations, + MaxUpkeepBatchSize: *plCfg.MaxUpkeepBatchSize, + } + pubCfg := config.GetAutomationConfig().AutomationConfig.PublicConfig + a.PublicConfig = ocr3.PublicConfig{ + DeltaProgress: *pubCfg.DeltaProgress, + DeltaResend: *pubCfg.DeltaResend, + DeltaInitial: *pubCfg.DeltaInitial, + DeltaRound: *pubCfg.DeltaRound, + DeltaGrace: *pubCfg.DeltaGrace, + DeltaCertifiedCommitRequest: *pubCfg.DeltaCertifiedCommitRequest, + DeltaStage: *pubCfg.DeltaStage, + RMax: *pubCfg.RMax, + MaxDurationQuery: *pubCfg.MaxDurationQuery, + MaxDurationObservation: *pubCfg.MaxDurationObservation, + MaxDurationShouldAcceptAttestedReport: *pubCfg.MaxDurationShouldAcceptAttestedReport, + MaxDurationShouldTransmitAcceptedReport: *pubCfg.MaxDurationShouldTransmitAcceptedReport, + F: *pubCfg.F, } - require.NoError(t, err, "Error setting OCR config") - consumersConditional, upkeepidsConditional := actions.DeployConsumers(t, chainClient, registry, registrar, linkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, false, false, false, nil) - consumersLogtrigger, upkeepidsLogtrigger := actions.DeployConsumers(t, chainClient, registry, registrar, linkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, true, false, false, nil) + a.SetupAutomationDeployment(t) - consumers := append(consumersConditional, consumersLogtrigger...) - upkeepIDs := append(upkeepidsConditional, upkeepidsLogtrigger...) + err = actions.FundChainlinkNodesFromRootAddress(l, a.ChainClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes[1:]), big.NewFloat(*config.Common.ChainlinkNodeFunding)) + require.NoError(t, err, "Error funding Chainlink nodes") + + var consumersLogTrigger, consumersConditional []contracts.KeeperConsumer + var upkeepidsConditional, upkeepidsLogTrigger []*big.Int + consumersConditional, upkeepidsConditional = actions.DeployConsumers(t, a.ChainClient, a.Registry, a.Registrar, a.LinkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, false, false, false, nil) + consumers := consumersConditional + upkeepIDs := upkeepidsConditional + if rv >= eth_contracts.RegistryVersion_2_1 { + consumersLogTrigger, upkeepidsLogTrigger = actions.DeployConsumers(t, a.ChainClient, a.Registry, a.Registrar, a.LinkToken, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, true, false, false, nil) - for _, c := range consumersLogtrigger { - err = c.Start() - require.NoError(t, err, "Error starting consumer") + consumers = append(consumersConditional, consumersLogTrigger...) + upkeepIDs = append(upkeepidsConditional, upkeepidsLogTrigger...) + + for _, c := range consumersLogTrigger { + err = c.Start() + require.NoError(t, err, "Error starting consumer") + } } l.Info().Msg("Waiting for all upkeeps to be performed") @@ -324,6 +333,13 @@ func TestAutomationChaos(t *testing.T) { _, err = testEnvironment.Chaos.Run(testCase.chaosFunc(testEnvironment.Cfg.Namespace, testCase.chaosProps)) require.NoError(t, err) + if rv >= eth_contracts.RegistryVersion_2_1 { + for _, c := range consumersLogTrigger { + err = c.Start() + require.NoError(t, err, "Error starting consumer") + } + } + gom.Eventually(func(g gomega.Gomega) { // Check if the upkeeps are performing multiple times by analyzing their counters and checking they are greater than 10 for i := 0; i < len(upkeepIDs); i++ { @@ -334,7 +350,7 @@ func TestAutomationChaos(t *testing.T) { g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) } - }, "3m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~2m for performing each upkeep 5 times, ~2m buffer + }, "5m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~2m for performing each upkeep 5 times, ~2m buffer }) } diff --git a/integration-tests/testconfig/automation/automation.toml b/integration-tests/testconfig/automation/automation.toml index 7e10af62cfa..26b9f05597d 100644 --- a/integration-tests/testconfig/automation/automation.toml +++ b/integration-tests/testconfig/automation/automation.toml @@ -152,6 +152,57 @@ max_check_data_size=5_000 max_perform_data_size=5_000 max_revert_data_size=5_000 +# chaos test specific overrides +[Chaos.Common] +chainlink_node_funding = 2.0 + +[Chaos.Automation] + +[Chaos.Automation.AutomationConfig] +use_log_buffer_v1=false + +[Chaos.Automation.AutomationConfig.PublicConfig] +delta_progress=10_000_000_000 +delta_resend=15_000_000_000 +delta_initial=500_000_000 +delta_round=1_000_000_000 +delta_grace=200_000_000 +delta_certified_commit_request=300_000_000 +delta_stage=30_000_000_000 +r_max=24 +f=1 +max_duration_query=20_000_000 +max_duration_observation=20_000_000 +max_duration_should_accept_attested_report=1_200_000_000 +max_duration_should_transmit_accepted_report=20_000_000 + +[Chaos.Automation.AutomationConfig.PluginConfig] +perform_lockout_window=3_600_000 +target_probability="0.999" +target_in_rounds=1 +min_confirmations=0 +gas_limit_per_report=10_300_000 +gas_overhead_per_upkeep=300_000 +max_upkeep_batch_size=10 + +[Chaos.Automation.AutomationConfig.PluginConfig.LogProviderConfig] +block_rate=1 +log_limit=2 + +[Chaos.Automation.AutomationConfig.RegistrySettings] +payment_premium_ppb=200_000_000 +flat_fee_micro_link=0 +check_gas_limit=2_500_000 +staleness_seconds=90000 +gas_ceiling_multiplier=1 +max_perform_gas=5_000_000 +min_upkeep_spend=0 +fallback_gas_price=200_000_000_000 +fallback_link_price=2_000_000_000_000_000_000 +max_check_data_size=5_000 +max_perform_data_size=5_000 +max_revert_data_size=5_000 + # load test specific overrides [Load.Seth] ephemeral_addresses_number = 100