Skip to content

Commit

Permalink
TT-1174 Migrate automation reorg test to GethSetHead (#13463)
Browse files Browse the repository at this point in the history 
* wip

* change depth settings

* Fix reorg test

1. Fund nodes with 2 eth
2. Start re-org below finality depth

* Remove blockscout chain explorer app from helm deployment

blockscout image cannot be found

* Enable other reorg tests and update test description

* Disable failing tests

* require reorg block count below finality depth

* Fix log trigger test

* Enable all reorg tests

* Do not log node config
  • Loading branch information
@lukaszcl
lukaszcl authored Jun 7, 2024
1 parent 8c75b1a commit 85044f9
Showing 1 changed file with 61 additions and 95 deletions.
156 changes: 61 additions & 95 deletions integration-tests/reorg/automation_reorg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@ import (
"go.uber.org/zap/zapcore"

"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
ctf_client "github.com/smartcontractkit/chainlink-testing-framework/client"
ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
"github.com/smartcontractkit/chainlink-testing-framework/k8s/environment"
"github.com/smartcontractkit/chainlink-testing-framework/k8s/pkg/cdk8s/blockscout"
"github.com/smartcontractkit/chainlink-testing-framework/k8s/pkg/helm/chainlink"
"github.com/smartcontractkit/chainlink-testing-framework/k8s/pkg/helm/reorg"
"github.com/smartcontractkit/chainlink-testing-framework/logging"
"github.com/smartcontractkit/chainlink-testing-framework/networks"
"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"

geth_helm "github.com/smartcontractkit/chainlink-testing-framework/k8s/pkg/helm/ethereum"
"github.com/smartcontractkit/chainlink/integration-tests/actions"
"github.com/smartcontractkit/chainlink/integration-tests/client"
"github.com/smartcontractkit/chainlink/integration-tests/contracts"
Expand All @@ -29,7 +29,8 @@ import (
)

var (
baseTOML = `[Feature]
baseTOML = `
[Feature]
LogPoller = true
[OCR2]
Expand All @@ -38,18 +39,27 @@ Enabled = true
[P2P]
[P2P.V2]
AnnounceAddresses = ["0.0.0.0:6690"]
ListenAddresses = ["0.0.0.0:6690"]`
networkTOML = `Enabled = true
FinalityDepth = 200
LogPollInterval = '1s'
ListenAddresses = ["0.0.0.0:6690"]
`
finalityDepth = 20
historyDepth = 30
reorgBlockCount = 10 // Number of blocks to reorg (less than finalityDepth)
networkTOML = fmt.Sprintf(`
Enabled = true
FinalityDepth = %d
[EVM.HeadTracker]
HistoryDepth = 400
HistoryDepth = %d
[EVM.GasEstimator]
Mode = 'FixedPrice'
LimitDefault = 5_000_000`

LimitDefault = 5_000_000
`, finalityDepth, historyDepth)
upkeepCount = 2
nodeCount = 6
nodeFundsAmount = new(big.Float).SetFloat64(2) // Each node will have 2 ETH
defaultUpkeepGasLimit = uint32(2500000)
defaultLinkFunds = int64(9e18)
defaultAutomationSettings = map[string]interface{}{
"toml": "",
"db": map[string]interface{}{
Expand All @@ -67,22 +77,6 @@ LimitDefault = 5_000_000`
},
},
}

defaultReorgEthereumSettings = &reorg.Props{
NetworkName: "",
NetworkType: "geth-reorg",
Values: map[string]interface{}{
"geth": map[string]interface{}{
"genesis": map[string]interface{}{
"networkId": "1337",
},
"miner": map[string]interface{}{
"replicas": 2,
},
},
},
}

defaultOCRRegistryConfig = contracts.KeeperRegistrySettings{
PaymentPremiumPPB: uint32(200000000),
FlatFeeMicroLINK: uint32(0),
Expand All @@ -100,40 +94,26 @@ LimitDefault = 5_000_000`
}
)

const (
defaultUpkeepGasLimit = uint32(2500000)
defaultLinkFunds = int64(9e18)
numberOfUpkeeps = 2
automationReorgBlocks = 50
numberOfNodes = 6
)

/*
* This test verifies that conditional upkeeps automatically recover from chain reorgs
* The blockchain is configured to have two separate miners and one geth node. The test starts
* with happy path where the two miners remain in sync and upkeeps are expected to be performed.
* Then reorg starts and the connection between the two geth miners is severed. This makes the
* chain unstable, however all the CL nodes get the same view of the unstable chain through the
* same geth node.
* This test verifies that conditional upkeeps automatically recover from chain reorgs.
*
* Upkeeps are expected to be performed during the reorg as there are only two versions of the
* the chain, on average 1/2 performUpkeeps should go through.
* The test starts with happy path where upkeeps are expected to be performed.
* Then reorg below finality depth happens which makes the chain unstable.
*
* The miner nodes are synced back after automationReorgBlocks. The syncing event can cause a
* large reorg from CL node perspective, causing existing performUpkeeps to become staleUpkeeps.
* Automation should be able to recover from this and upkeeps should continue to occur at a
* normal pace after the event.
* Upkeeps are expected to be performed during the reorg.
*/
func TestAutomationReorg(t *testing.T) {
require.Less(t, reorgBlockCount, finalityDepth, "Reorg block count should be less than finality depth")

t.Parallel()
l := logging.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1_conditional": ethereum.RegistryVersion_2_1,
"registry_2_1_logtrigger": ethereum.RegistryVersion_2_1,
"registry_2_2_conditional": ethereum.RegistryVersion_2_2,
"registry_2_2_logtrigger": ethereum.RegistryVersion_2_2,
"registry_2_2_conditional": ethereum.RegistryVersion_2_2, // Works only on Chainlink Node v2.10.0 or greater
"registry_2_2_logtrigger": ethereum.RegistryVersion_2_2, // Works only on Chainlink Node v2.10.0 or greater
}

for n, rv := range registryVersions {
Expand All @@ -148,7 +128,7 @@ func TestAutomationReorg(t *testing.T) {

network := networks.MustGetSelectedNetworkConfig(config.Network)[0]

defaultAutomationSettings["replicas"] = numberOfNodes
defaultAutomationSettings["replicas"] = nodeCount
defaultAutomationSettings["toml"] = networks.AddNetworkDetailedConfig(baseTOML, config.Pyroscope, networkTOML, network)

var overrideFn = func(_ interface{}, target interface{}) {
Expand All @@ -158,26 +138,27 @@ func TestAutomationReorg(t *testing.T) {

cd := chainlink.NewWithOverride(0, defaultAutomationSettings, config.ChainlinkImage, overrideFn)

ethSetting := defaultReorgEthereumSettings
ethSetting.NetworkName = network.Name

testEnvironment := environment.
New(&environment.Config{
NamespacePrefix: fmt.Sprintf("automation-reorg-%d", automationReorgBlocks),
NamespacePrefix: fmt.Sprintf("automation-reorg-%d", reorgBlockCount),
TTL: time.Hour * 1,
Test: t}).
AddHelm(reorg.New(ethSetting)).
AddChart(blockscout.New(&blockscout.Props{
Name: "geth-blockscout",
WsURL: network.URL,
HttpURL: network.HTTPURLs[0]})).
// Use Geth blockchain to simulate reorgs
AddHelm(geth_helm.New(&geth_helm.Props{
NetworkName: network.Name,
Simulated: true,
WsURLs: network.URLs,
})).
AddHelm(cd)
err = testEnvironment.Run()
require.NoError(t, err, "Error setting up test environment")

if testEnvironment.WillUseRemoteRunner() {
return
}
gethURL := testEnvironment.URLs["Simulated Geth_http"][0]
require.NotEmpty(t, gethURL, "Geth URL should not be empty")
gethRPCClient := ctf_client.NewRPCClient(gethURL)

chainClient, err := blockchain.NewEVMClient(network, testEnvironment, l)
require.NoError(t, err, "Error connecting to blockchain")
Expand All @@ -193,9 +174,7 @@ func TestAutomationReorg(t *testing.T) {
require.NoError(t, err, "Error tearing down environment")
})

txCost, err := chainClient.EstimateCostForChainlinkOperations(1000)
require.NoError(t, err, "Error estimating cost for Chainlink Operations")
err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost)
err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, nodeFundsAmount)
require.NoError(t, err, "Error funding Chainlink nodes")

linkToken, err := contractDeployer.DeployLinkTokenContract()
Expand All @@ -210,7 +189,7 @@ func TestAutomationReorg(t *testing.T) {
chainClient,
)
// Fund the registry with LINK
err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(numberOfUpkeeps))))
err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(upkeepCount))))
require.NoError(t, err, "Funding keeper registry contract shouldn't fail")

actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0, registryVersion)
Expand All @@ -227,8 +206,18 @@ func TestAutomationReorg(t *testing.T) {
require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set")

// Use the name to determine if this is a log trigger or not
isLogTrigger := name == "registry_2_1_logtrigger"
consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, numberOfUpkeeps, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, isLogTrigger, false)
isLogTrigger := name == "registry_2_1_logtrigger" || name == "registry_2_2_logtrigger"
consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, upkeepCount, big.NewInt(defaultLinkFunds), defaultUpkeepGasLimit, isLogTrigger, false)

if isLogTrigger {
for i := 0; i < len(upkeepIDs); i++ {
if err := consumers[i].Start(); err != nil {
l.Error().Msg("Error when starting consumer")
return
}
l.Info().Int("Consumer index", i).Msg("Consumer started")
}
}

l.Info().Msg("Waiting for all upkeeps to be performed")

Expand All @@ -247,20 +236,14 @@ func TestAutomationReorg(t *testing.T) {

l.Info().Msg("All upkeeps performed under happy path. Starting reorg")

rc, err := NewReorgController(
&ReorgConfig{
FromPodLabel: reorg.TXNodesAppLabel,
ToPodLabel: reorg.MinerNodesAppLabel,
Network: chainClient,
Env: testEnvironment,
BlockConsensusThreshold: 3,
Timeout: 1800 * time.Second,
},
)

require.NoError(t, err, "Error getting reorg controller")
rc.ReOrg(automationReorgBlocks)
rc.WaitReorgStarted()
l.Info().
Str("URL", gethRPCClient.URL).
Int("BlocksBack", reorgBlockCount).
Int("FinalityDepth", finalityDepth).
Int("HistoryDepth", historyDepth).
Msg("Rewinding blocks on chain below finality depth")
err = gethRPCClient.GethSetHead(reorgBlockCount)
require.NoError(t, err, "Error rewinding blocks on chain")

l.Info().Msg("Reorg started. Expecting chain to become unstable and upkeeps to still getting performed")

Expand All @@ -275,23 +258,6 @@ func TestAutomationReorg(t *testing.T) {
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "5m", "1s").Should(gomega.Succeed())

l.Info().Msg("Upkeep performed during unstable chain, waiting for reorg to finish")
err = rc.WaitDepthReached()
require.NoError(t, err)

l.Info().Msg("Reorg finished, chain should be stable now. Expecting upkeeps to keep getting performed")
gom.Eventually(func(g gomega.Gomega) {
// Check if the upkeeps are performing multiple times by analyzing their counters and checking they reach 20
for i := 0; i < len(upkeepIDs); i++ {
counter, err := consumers[i].Counter(testcontext.Get(t))
require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
expect := 20
l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
}
}, "10m", "1s").Should(gomega.Succeed())
})
}
}

0 comments on commit 85044f9

Please sign in to comment.