From 15adf2e71bb1c24226b4f481918166dc6f57822e Mon Sep 17 00:00:00 2001 From: Lukasz <120112546+lukaszcl@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:19:59 +0200 Subject: [PATCH] TT-1155 Allow to run OCR soak test with reorg on Geth blockchain (#13483) * Allow to run OCR soak test with Simulated Geth reorg * wip * Fix nil in reorg timer * Bump CTF * Revert "wip" This reverts commit 008c90445e86fd7491dedd7268a621aa7dd83cd5. * Bump CTF * Fix when using remote runner * Bump CTF --- integration-tests/go.mod | 4 +- integration-tests/go.sum | 8 ++-- integration-tests/load/go.mod | 4 +- integration-tests/load/go.sum | 8 ++-- integration-tests/testsetups/ocr.go | 71 +++++++++++++++++++++++++++-- 5 files changed, 78 insertions(+), 17 deletions(-) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index da441083906..83fbad89e23 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -19,7 +19,7 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/montanaflynn/stats v0.7.1 github.com/onsi/gomega v1.30.0 - github.com/pelletier/go-toml/v2 v2.1.1 + github.com/pelletier/go-toml/v2 v2.2.2 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.30.0 github.com/scylladb/go-reflectx v1.0.1 @@ -28,7 +28,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240607135320-c9bc0a2ac0ce - github.com/smartcontractkit/chainlink-testing-framework v1.30.2 + github.com/smartcontractkit/chainlink-testing-framework v1.30.3 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 560734651c2..d53326cba3d 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1354,8 +1354,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.2 h1:PKAg0nVT6clcTYebm3Q0rhTId+5xE9wYyvFxxSg4+lI= -github.com/smartcontractkit/chainlink-testing-framework v1.30.2/go.mod h1:oEIggLGWyWfLkjWvuXLol8inUT4YbBb06fJx/S60gQ4= +github.com/smartcontractkit/chainlink-testing-framework v1.30.3 h1:3mL0dBI/+8nTSh9GdxYyhNIAPTjSBThPGJnjcjjDaIg= +github.com/smartcontractkit/chainlink-testing-framework v1.30.3/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index ce37cee57d4..4ef564b289c 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -11,13 +11,13 @@ require ( github.com/K-Phoen/grabana v0.22.1 github.com/ethereum/go-ethereum v1.13.8 github.com/go-resty/resty/v2 v2.11.0 - github.com/pelletier/go-toml/v2 v2.1.1 + github.com/pelletier/go-toml/v2 v2.2.2 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.30.0 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240607135320-c9bc0a2ac0ce - github.com/smartcontractkit/chainlink-testing-framework v1.30.2 + github.com/smartcontractkit/chainlink-testing-framework v1.30.3 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 9a84616195b..45b5e8fdc7f 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1344,8 +1344,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.2 h1:PKAg0nVT6clcTYebm3Q0rhTId+5xE9wYyvFxxSg4+lI= -github.com/smartcontractkit/chainlink-testing-framework v1.30.2/go.mod h1:oEIggLGWyWfLkjWvuXLol8inUT4YbBb06fJx/S60gQ4= +github.com/smartcontractkit/chainlink-testing-framework v1.30.3 h1:3mL0dBI/+8nTSh9GdxYyhNIAPTjSBThPGJnjcjjDaIg= +github.com/smartcontractkit/chainlink-testing-framework v1.30.3/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index 2c3452b77d5..02f4db9c51b 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -27,7 +27,7 @@ import ( "github.com/smartcontractkit/libocr/gethwrappers2/ocr2aggregator" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" - ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client" + ctf_client "github.com/smartcontractkit/chainlink-testing-framework/client" ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" "github.com/smartcontractkit/chainlink-testing-framework/k8s/environment" "github.com/smartcontractkit/chainlink-testing-framework/k8s/pkg/helm/chainlink" @@ -71,7 +71,7 @@ type OCRSoakTest struct { log zerolog.Logger bootstrapNode *client.ChainlinkK8sClient workerNodes []*client.ChainlinkK8sClient - mockServer *ctfClient.MockserverClient + mockServer *ctf_client.MockserverClient filterQuery geth.FilterQuery ocrRoundStates []*testreporters.OCRRoundState @@ -83,7 +83,8 @@ type OCRSoakTest struct { ocrV2Instances []contracts.OffchainAggregatorV2 ocrV2InstanceMap map[string]contracts.OffchainAggregatorV2 // address : instance - rpcNetwork blockchain.EVMNetwork // network configuration for the blockchain node + rpcNetwork blockchain.EVMNetwork // network configuration for the blockchain node + reorgHappened bool // flag to indicate if a reorg happened during the test } // NewOCRSoakTest creates a new OCR soak test to setup and run @@ -193,6 +194,12 @@ func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTe o.testEnvironment = testEnv o.namespace = testEnv.Cfg.Namespace + // If the test is using the remote runner, we don't need to set the network URLs + // as the remote runner will handle that + if o.Environment().WillUseRemoteRunner() { + return + } + o.rpcNetwork = nodeNetwork if o.rpcNetwork.Simulated && o.rpcNetwork.Name == "Anvil" { if testEnv.Cfg.InsideK8s { @@ -202,6 +209,19 @@ func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTe // Test is running locally, set forwarded URL of Anvil blockchain node o.rpcNetwork.URLs = []string{anvilChart.ForwardedWSURL} } + } else if o.rpcNetwork.Simulated && o.rpcNetwork.Name == blockchain.SimulatedEVMNetwork.Name { + if testEnv.Cfg.InsideK8s { + // Test is running inside K8s + o.rpcNetwork.URLs = blockchain.SimulatedEVMNetwork.URLs + } else { + // Test is running locally, set forwarded URL of Geth blockchain node + wsURLs := o.testEnvironment.URLs[blockchain.SimulatedEVMNetwork.Name+"_internal"] + httpURLs := o.testEnvironment.URLs[blockchain.SimulatedEVMNetwork.Name+"_internal_http"] + require.NotEmpty(o.t, wsURLs, "Forwarded Geth URLs should not be empty") + require.NotEmpty(o.t, httpURLs, "Forwarded Geth URLs should not be empty") + o.rpcNetwork.URLs = wsURLs + o.rpcNetwork.HTTPURLs = httpURLs + } } } @@ -218,7 +238,7 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { nodes, err := client.ConnectChainlinkNodes(o.testEnvironment) require.NoError(o.t, err, "Connecting to chainlink nodes shouldn't fail") o.bootstrapNode, o.workerNodes = nodes[0], nodes[1:] - o.mockServer, err = ctfClient.ConnectMockServer(o.testEnvironment) + o.mockServer, err = ctf_client.ConnectMockServer(o.testEnvironment) require.NoError(o.t, err, "Creating mockserver clients shouldn't fail") linkContract, err := contracts.DeployLinkTokenContract(o.log, seth) @@ -379,6 +399,7 @@ type OCRSoakTestState struct { BootStrapNodeURL string `toml:"bootstrapNodeURL"` WorkerNodeURLs []string `toml:"workerNodeURLs"` ChainURL string `toml:"chainURL"` + ReorgHappened bool `toml:"reorgHappened"` MockServerURL string `toml:"mockServerURL"` } @@ -404,6 +425,7 @@ func (o *OCRSoakTest) SaveState() error { MockServerURL: "http://mockserver:1080", // TODO: Make this dynamic BootStrapNodeURL: o.bootstrapNode.URL(), WorkerNodeURLs: workerNodeURLs, + ReorgHappened: o.reorgHappened, } data, err := toml.Marshal(testState) if err != nil { @@ -454,6 +476,7 @@ func (o *OCRSoakTest) LoadState() error { o.timeLeft = testState.TestDuration - testState.TimeRunning o.startTime = testState.StartTime o.startingBlockNum = testState.StartingBlockNum + o.reorgHappened = testState.ReorgHappened o.Config.OCR.Soak.OCRVersion = &testState.OCRVersion o.bootstrapNode, err = client.ConnectChainlinkNodeURL(testState.BootStrapNodeURL) @@ -485,7 +508,7 @@ func (o *OCRSoakTest) LoadState() error { } } - o.mockServer, err = ctfClient.ConnectMockServerURL(testState.MockServerURL) + o.mockServer, err = ctf_client.ConnectMockServerURL(testState.MockServerURL) if err != nil { return err } @@ -562,6 +585,22 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { err := o.observeOCREvents() require.NoError(o.t, err, "Error subscribing to OCR events") + // Schedule blockchain re-org if needed + // Reorg only avaible for Simulated Geth + var reorgCh <-chan time.Time + n := o.Config.GetNetworkConfig() + if n.IsSimulatedGethSelected() && n.GethReorgConfig.Enabled { + var reorgDelay time.Duration + if n.GethReorgConfig.DelayCreate.Duration > testDuration { + // This may happen when test is resumed and the reorg delay is longer than the time left + o.log.Warn().Msg("Reorg delay is longer than test duration, reorg scheduled immediately") + reorgDelay = 0 + } else { + reorgDelay = n.GethReorgConfig.DelayCreate.Duration + } + reorgCh = time.After(reorgDelay) + } + for { select { case <-interruption: @@ -595,6 +634,12 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { newValue = rand.Intn(256) + 1 // #nosec G404 - kudos to you if you actually find a way to exploit this } lastValue = newValue + + // Schedule blockchain re-org if needed + case <-reorgCh: + if !o.reorgHappened { + o.startBlockchainReorg(o.Config.GetNetworkConfig().GethReorgConfig.Depth) + } } } } @@ -610,6 +655,22 @@ func (o *OCRSoakTest) complete() { o.TestReporter.RecordEvents(o.ocrRoundStates, o.testIssues) } +func (o *OCRSoakTest) startBlockchainReorg(depth int) { + if !o.Config.GetNetworkConfig().IsSimulatedGethSelected() { + require.FailNow(o.t, "Reorg only available for Simulated Geth") + return + } + + client := ctf_client.NewRPCClient(o.rpcNetwork.HTTPURLs[0]) + o.log.Info(). + Str("URL", client.URL). + Int("depth", depth). + Msg("Starting blockchain reorg on Simulated Geth chain") + err := client.GethSetHead(depth) + require.NoError(o.t, err, "Error starting blockchain reorg on Simulated Geth chain") + o.reorgHappened = true +} + // setFilterQuery to look for all events that happened func (o *OCRSoakTest) setFilterQuery() { ocrAddresses := o.getContractAddresses()