Skip to content

Commit

Permalink
reorg for v21 (#10381)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shileiwill
shileiwill authored Oct 16, 2023
1 parent 458c18c commit f74c15a
Showing 1 changed file with 142 additions and 130 deletions.
272 changes: 142 additions & 130 deletions integration-tests/reorg/automation_reorg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ import (
"time"

"github.com/onsi/gomega"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zapcore"

"github.com/smartcontractkit/chainlink-env/environment"
"github.com/smartcontractkit/chainlink-env/pkg/cdk8s/blockscout"
"github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink"
Expand All @@ -20,6 +17,8 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/logging"
"github.com/smartcontractkit/chainlink-testing-framework/networks"
"github.com/smartcontractkit/chainlink-testing-framework/utils"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zapcore"

"github.com/smartcontractkit/chainlink/integration-tests/actions"
"github.com/smartcontractkit/chainlink/integration-tests/client"
Expand Down Expand Up @@ -124,134 +123,147 @@ const (
* normal pace after the event.
*/
func TestAutomationReorg(t *testing.T) {
t.Parallel()
l := logging.GetTestLogger(t)
network := networks.SelectedNetwork
defaultAutomationSettings["replicas"] = numberOfNodes
cd := chainlink.New(0, defaultAutomationSettings)

testEnvironment := environment.
New(&environment.Config{
NamespacePrefix: fmt.Sprintf("automation-reorg-%d", automationReorgBlocks),
TTL: time.Hour * 1,
Test: t}).
AddHelm(reorg.New(defaultReorgEthereumSettings)).
AddChart(blockscout.New(&blockscout.Props{
Name: "geth-blockscout",
WsURL: activeEVMNetwork.URL,
HttpURL: activeEVMNetwork.HTTPURLs[0]})).
AddHelm(cd)
err := testEnvironment.Run()
require.NoError(t, err, "Error setting up test environment")

if testEnvironment.WillUseRemoteRunner() {
return
}

chainClient, err := blockchain.NewEVMClient(network, testEnvironment, l)
require.NoError(t, err, "Error connecting to blockchain")
contractDeployer, err := contracts.NewContractDeployer(chainClient, l)
require.NoError(t, err, "Error building contract deployer")
chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment)
require.NoError(t, err, "Error connecting to Chainlink nodes")
chainClient.ParallelTransactions(true)

// Register cleanup for any test
t.Cleanup(func() {
err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, zapcore.PanicLevel, chainClient)
require.NoError(t, err, "Error tearing down environment")
})

txCost, err := chainClient.EstimateCostForChainlinkOperations(1000)
require.NoError(t, err, "Error estimating cost for Chainlink Operations")
err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost)
require.NoError(t, err, "Error funding Chainlink nodes")

linkToken, err := contractDeployer.DeployLinkTokenContract()
require.NoError(t, err, "Error deploying LINK token")

registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar(
t,
ethereum.RegistryVersion_2_0,
defaultOCRRegistryConfig,
linkToken,
contractDeployer,
chainClient,
)

// Fund the registry with LINK
err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps))))
require.NoError(t, err, "Funding keeper registry contract shouldn't fail")

actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, ethereum.RegistryVersion_2_0)
nodesWithoutBootstrap := chainlinkNodes[1:]
ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, defaultOCRRegistryConfig, registrar.Address(), 30*time.Second)
require.NoError(t, err, "OCR2 config should be built successfully")
err = registry.SetConfig(defaultOCRRegistryConfig, ocrConfig)
require.NoError(t, err, "Registry config should be be set successfully")
require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set")

consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, false)

l.Info().Msg("Waiting for all upkeeps to be performed")

gom := gomega.NewGomegaWithT(t)
gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they are greater than 5
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(context.Background())
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 5
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "7m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~3m for performing each upkeep 5 times, ~3m buffer

l.Info().Msg("All upkeeps performed under happy path. Starting reorg")

rc, err := NewReorgController(
&ReorgConfig{
FromPodLabel: reorg.TXNodesAppLabel,
ToPodLabel: reorg.MinerNodesAppLabel,
Network: chainClient,
Env: testEnvironment,
BlockConsensusThreshold: 3,
Timeout: 1800 * time.Second,
},
)

require.NoError(t, err, "Error getting reorg controller")
rc.ReOrg(automationReorgBlocks)
rc.WaitReorgStarted()

l.Info().Msg("Reorg started. Expecting chain to become unstable and upkeeps to still getting performed")

gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 10
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(context.Background())
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 10
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "5m", "1s").Should(gomega.Succeed())

l.Info().Msg("Upkeep performed during unstable chain, waiting for reorg to finish")
rc.WaitDepthReached()

l.Info().Msg("Reorg finished, chain should be stable now. Expecting upkeeps to keep getting performed")
gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 20
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(context.Background())
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 20
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "10m", "1s").Should(gomega.Succeed())
registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1_conditional": ethereum.RegistryVersion_2_1,
"registry_2_1_logtrigger": ethereum.RegistryVersion_2_1,
}

for name, registryVersion := range registryVersions {
t.Run(name, func(t *testing.T) {
t.Parallel()
network := networks.SelectedNetwork

defaultAutomationSettings["replicas"] = numberOfNodes
cd := chainlink.New(0, defaultAutomationSettings)
testEnvironment := environment.
New(&environment.Config{
NamespacePrefix: fmt.Sprintf("automation-reorg-%d", automationReorgBlocks),
TTL: time.Hour * 1,
Test: t}).
AddHelm(reorg.New(defaultReorgEthereumSettings)).
AddChart(blockscout.New(&blockscout.Props{
Name: "geth-blockscout",
WsURL: activeEVMNetwork.URL,
HttpURL: activeEVMNetwork.HTTPURLs[0]})).
AddHelm(cd)
err := testEnvironment.Run()
require.NoError(t, err, "Error setting up test environment")

if testEnvironment.WillUseRemoteRunner() {
return
}

chainClient, err := blockchain.NewEVMClient(network, testEnvironment, l)
require.NoError(t, err, "Error connecting to blockchain")
contractDeployer, err := contracts.NewContractDeployer(chainClient, l)
require.NoError(t, err, "Error building contract deployer")
chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment)
require.NoError(t, err, "Error connecting to Chainlink nodes")
chainClient.ParallelTransactions(true)

// Register cleanup for any test
t.Cleanup(func() {
err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, zapcore.PanicLevel, chainClient)
require.NoError(t, err, "Error tearing down environment")
})

txCost, err := chainClient.EstimateCostForChainlinkOperations(1000)
require.NoError(t, err, "Error estimating cost for Chainlink Operations")
err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost)
require.NoError(t, err, "Error funding Chainlink nodes")

linkToken, err := contractDeployer.DeployLinkTokenContract()
require.NoError(t, err, "Error deploying LINK token")

registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar(
t,
registryVersion,
defaultOCRRegistryConfig,
linkToken,
contractDeployer,
chainClient,
)
// Fund the registry with LINK
err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps))))
require.NoError(t, err, "Funding keeper registry contract shouldn't fail")

actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, registryVersion)
nodesWithoutBootstrap := chainlinkNodes[1:]
ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, defaultOCRRegistryConfig, registrar.Address(), 5*time.Second)
require.NoError(t, err, "OCR2 config should be built successfully")
err = registry.SetConfig(defaultOCRRegistryConfig, ocrConfig)
require.NoError(t, err, "Registry config should be be set successfully")
require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set")

// Use the name to determine if this is a log trigger or not
isLogTrigger := name == "registry_2_1_logtrigger"
consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, isLogTrigger)

l.Info().Msg("Waiting for all upkeeps to be performed")

gom := gomega.NewGomegaWithT(t)
gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they are greater than 5
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(context.Background())
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 5
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "7m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~3m for performing each upkeep 5 times, ~3m buffer

l.Info().Msg("All upkeeps performed under happy path. Starting reorg")

rc, err := NewReorgController(
&ReorgConfig{
FromPodLabel: reorg.TXNodesAppLabel,
ToPodLabel: reorg.MinerNodesAppLabel,
Network: chainClient,
Env: testEnvironment,
BlockConsensusThreshold: 3,
Timeout: 1800 * time.Second,
},
)

require.NoError(t, err, "Error getting reorg controller")
rc.ReOrg(automationReorgBlocks)
rc.WaitReorgStarted()

l.Info().Msg("Reorg started. Expecting chain to become unstable and upkeeps to still getting performed")

gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 10
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(context.Background())
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 10
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "5m", "1s").Should(gomega.Succeed())

l.Info().Msg("Upkeep performed during unstable chain, waiting for reorg to finish")
rc.WaitDepthReached()

l.Info().Msg("Reorg finished, chain should be stable now. Expecting upkeeps to keep getting performed")
gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 20
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(context.Background())
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 20
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "10m", "1s").Should(gomega.Succeed())
})
}
}

0 comments on commit f74c15a

Please sign in to comment.