From f74c15a7175bd0d696422cfc630f4149490a9e55 Mon Sep 17 00:00:00 2001 From: Lei Date: Mon, 16 Oct 2023 15:45:57 -0700 Subject: [PATCH] reorg for v21 (#10381) --- .../reorg/automation_reorg_test.go | 272 +++++++++--------- 1 file changed, 142 insertions(+), 130 deletions(-) diff --git a/integration-tests/reorg/automation_reorg_test.go b/integration-tests/reorg/automation_reorg_test.go index 2f8db75f749..66da801333e 100644 --- a/integration-tests/reorg/automation_reorg_test.go +++ b/integration-tests/reorg/automation_reorg_test.go @@ -9,9 +9,6 @@ import ( "time" "github.com/onsi/gomega" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" - "github.com/smartcontractkit/chainlink-env/environment" "github.com/smartcontractkit/chainlink-env/pkg/cdk8s/blockscout" "github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink" @@ -20,6 +17,8 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" "github.com/smartcontractkit/chainlink-testing-framework/utils" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/client" @@ -124,134 +123,147 @@ const ( * normal pace after the event. */ func TestAutomationReorg(t *testing.T) { + t.Parallel() l := logging.GetTestLogger(t) - network := networks.SelectedNetwork - defaultAutomationSettings["replicas"] = numberOfNodes - cd := chainlink.New(0, defaultAutomationSettings) - - testEnvironment := environment. - New(&environment.Config{ - NamespacePrefix: fmt.Sprintf("automation-reorg-%d", automationReorgBlocks), - TTL: time.Hour * 1, - Test: t}). - AddHelm(reorg.New(defaultReorgEthereumSettings)). - AddChart(blockscout.New(&blockscout.Props{ - Name: "geth-blockscout", - WsURL: activeEVMNetwork.URL, - HttpURL: activeEVMNetwork.HTTPURLs[0]})). - AddHelm(cd) - err := testEnvironment.Run() - require.NoError(t, err, "Error setting up test environment") - - if testEnvironment.WillUseRemoteRunner() { - return - } - chainClient, err := blockchain.NewEVMClient(network, testEnvironment, l) - require.NoError(t, err, "Error connecting to blockchain") - contractDeployer, err := contracts.NewContractDeployer(chainClient, l) - require.NoError(t, err, "Error building contract deployer") - chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) - require.NoError(t, err, "Error connecting to Chainlink nodes") - chainClient.ParallelTransactions(true) - - // Register cleanup for any test - t.Cleanup(func() { - err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, zapcore.PanicLevel, chainClient) - require.NoError(t, err, "Error tearing down environment") - }) - - txCost, err := chainClient.EstimateCostForChainlinkOperations(1000) - require.NoError(t, err, "Error estimating cost for Chainlink Operations") - err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost) - require.NoError(t, err, "Error funding Chainlink nodes") - - linkToken, err := contractDeployer.DeployLinkTokenContract() - require.NoError(t, err, "Error deploying LINK token") - - registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar( - t, - ethereum.RegistryVersion_2_0, - defaultOCRRegistryConfig, - linkToken, - contractDeployer, - chainClient, - ) - - // Fund the registry with LINK - err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps)))) - require.NoError(t, err, "Funding keeper registry contract shouldn't fail") - - actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, ethereum.RegistryVersion_2_0) - nodesWithoutBootstrap := chainlinkNodes[1:] - ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, defaultOCRRegistryConfig, registrar.Address(), 30*time.Second) - require.NoError(t, err, "OCR2 config should be built successfully") - err = registry.SetConfig(defaultOCRRegistryConfig, ocrConfig) - require.NoError(t, err, "Registry config should be be set successfully") - require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set") - - consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, false) - - l.Info().Msg("Waiting for all upkeeps to be performed") - - gom := gomega.NewGomegaWithT(t) - gom.Eventually(func(g gomega.Gomega) { - // Check if the upkeeps are performing multiple times by analyzing their counters and checking they are greater than 5 - for i := 0; i < len(upkeepIDs); i++ { - counter, err := consumers[i].Counter(context.Background()) - require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i) - expect := 5 - l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed") - g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), - "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) - } - }, "7m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~3m for performing each upkeep 5 times, ~3m buffer - - l.Info().Msg("All upkeeps performed under happy path. Starting reorg") - - rc, err := NewReorgController( - &ReorgConfig{ - FromPodLabel: reorg.TXNodesAppLabel, - ToPodLabel: reorg.MinerNodesAppLabel, - Network: chainClient, - Env: testEnvironment, - BlockConsensusThreshold: 3, - Timeout: 1800 * time.Second, - }, - ) - - require.NoError(t, err, "Error getting reorg controller") - rc.ReOrg(automationReorgBlocks) - rc.WaitReorgStarted() - - l.Info().Msg("Reorg started. Expecting chain to become unstable and upkeeps to still getting performed") - - gom.Eventually(func(g gomega.Gomega) { - // Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 10 - for i := 0; i < len(upkeepIDs); i++ { - counter, err := consumers[i].Counter(context.Background()) - require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i) - expect := 10 - l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed") - g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), - "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) - } - }, "5m", "1s").Should(gomega.Succeed()) - - l.Info().Msg("Upkeep performed during unstable chain, waiting for reorg to finish") - rc.WaitDepthReached() - - l.Info().Msg("Reorg finished, chain should be stable now. Expecting upkeeps to keep getting performed") - gom.Eventually(func(g gomega.Gomega) { - // Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 20 - for i := 0; i < len(upkeepIDs); i++ { - counter, err := consumers[i].Counter(context.Background()) - require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i) - expect := 20 - l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed") - g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), - "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) - } - }, "10m", "1s").Should(gomega.Succeed()) + registryVersions := map[string]ethereum.KeeperRegistryVersion{ + "registry_2_0": ethereum.RegistryVersion_2_0, + "registry_2_1_conditional": ethereum.RegistryVersion_2_1, + "registry_2_1_logtrigger": ethereum.RegistryVersion_2_1, + } + for name, registryVersion := range registryVersions { + t.Run(name, func(t *testing.T) { + t.Parallel() + network := networks.SelectedNetwork + + defaultAutomationSettings["replicas"] = numberOfNodes + cd := chainlink.New(0, defaultAutomationSettings) + testEnvironment := environment. + New(&environment.Config{ + NamespacePrefix: fmt.Sprintf("automation-reorg-%d", automationReorgBlocks), + TTL: time.Hour * 1, + Test: t}). + AddHelm(reorg.New(defaultReorgEthereumSettings)). + AddChart(blockscout.New(&blockscout.Props{ + Name: "geth-blockscout", + WsURL: activeEVMNetwork.URL, + HttpURL: activeEVMNetwork.HTTPURLs[0]})). + AddHelm(cd) + err := testEnvironment.Run() + require.NoError(t, err, "Error setting up test environment") + + if testEnvironment.WillUseRemoteRunner() { + return + } + + chainClient, err := blockchain.NewEVMClient(network, testEnvironment, l) + require.NoError(t, err, "Error connecting to blockchain") + contractDeployer, err := contracts.NewContractDeployer(chainClient, l) + require.NoError(t, err, "Error building contract deployer") + chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) + require.NoError(t, err, "Error connecting to Chainlink nodes") + chainClient.ParallelTransactions(true) + + // Register cleanup for any test + t.Cleanup(func() { + err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, zapcore.PanicLevel, chainClient) + require.NoError(t, err, "Error tearing down environment") + }) + + txCost, err := chainClient.EstimateCostForChainlinkOperations(1000) + require.NoError(t, err, "Error estimating cost for Chainlink Operations") + err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost) + require.NoError(t, err, "Error funding Chainlink nodes") + + linkToken, err := contractDeployer.DeployLinkTokenContract() + require.NoError(t, err, "Error deploying LINK token") + + registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar( + t, + registryVersion, + defaultOCRRegistryConfig, + linkToken, + contractDeployer, + chainClient, + ) + // Fund the registry with LINK + err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps)))) + require.NoError(t, err, "Funding keeper registry contract shouldn't fail") + + actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, registryVersion) + nodesWithoutBootstrap := chainlinkNodes[1:] + ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, defaultOCRRegistryConfig, registrar.Address(), 5*time.Second) + require.NoError(t, err, "OCR2 config should be built successfully") + err = registry.SetConfig(defaultOCRRegistryConfig, ocrConfig) + require.NoError(t, err, "Registry config should be be set successfully") + require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set") + + // Use the name to determine if this is a log trigger or not + isLogTrigger := name == "registry_2_1_logtrigger" + consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, isLogTrigger) + + l.Info().Msg("Waiting for all upkeeps to be performed") + + gom := gomega.NewGomegaWithT(t) + gom.Eventually(func(g gomega.Gomega) { + // Check if the upkeeps are performing multiple times by analyzing their counters and checking they are greater than 5 + for i := 0; i < len(upkeepIDs); i++ { + counter, err := consumers[i].Counter(context.Background()) + require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i) + expect := 5 + l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed") + g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), + "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) + } + }, "7m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~3m for performing each upkeep 5 times, ~3m buffer + + l.Info().Msg("All upkeeps performed under happy path. Starting reorg") + + rc, err := NewReorgController( + &ReorgConfig{ + FromPodLabel: reorg.TXNodesAppLabel, + ToPodLabel: reorg.MinerNodesAppLabel, + Network: chainClient, + Env: testEnvironment, + BlockConsensusThreshold: 3, + Timeout: 1800 * time.Second, + }, + ) + + require.NoError(t, err, "Error getting reorg controller") + rc.ReOrg(automationReorgBlocks) + rc.WaitReorgStarted() + + l.Info().Msg("Reorg started. Expecting chain to become unstable and upkeeps to still getting performed") + + gom.Eventually(func(g gomega.Gomega) { + // Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 10 + for i := 0; i < len(upkeepIDs); i++ { + counter, err := consumers[i].Counter(context.Background()) + require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i) + expect := 10 + l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed") + g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), + "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) + } + }, "5m", "1s").Should(gomega.Succeed()) + + l.Info().Msg("Upkeep performed during unstable chain, waiting for reorg to finish") + rc.WaitDepthReached() + + l.Info().Msg("Reorg finished, chain should be stable now. Expecting upkeeps to keep getting performed") + gom.Eventually(func(g gomega.Gomega) { + // Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 20 + for i := 0; i < len(upkeepIDs); i++ { + counter, err := consumers[i].Counter(context.Background()) + require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i) + expect := 20 + l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed") + g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)), + "Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64()) + } + }, "10m", "1s").Should(gomega.Succeed()) + }) + } }