TT-1155 Allow to run OCR soak test with reorg on Geth blockchain (#13483

)

* Allow to run OCR soak test with Simulated Geth reorg

* wip

* Fix nil in reorg timer

* Bump CTF

* Revert "wip"

This reverts commit 008c904.

* Bump CTF

* Fix when using remote runner

* Bump CTF

integration-tests/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/manifoldco/promptui v0.9.0
 	github.com/montanaflynn/stats v0.7.1
 	github.com/onsi/gomega v1.30.0
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.30.0
 	github.com/scylladb/go-reflectx v1.0.1
@@ -28,7 +28,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240607135320-c9bc0a2ac0ce
-	github.com/smartcontractkit/chainlink-testing-framework v1.30.2
+	github.com/smartcontractkit/chainlink-testing-framework v1.30.3
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c

integration-tests/go.sum

@@ -1354,8 +1354,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
@@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.2 h1:PKAg0nVT6clcTYebm3Q0rhTId+5xE9wYyvFxxSg4+lI=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.2/go.mod h1:oEIggLGWyWfLkjWvuXLol8inUT4YbBb06fJx/S60gQ4=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.3 h1:3mL0dBI/+8nTSh9GdxYyhNIAPTjSBThPGJnjcjjDaIg=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.3/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

integration-tests/load/go.mod

@@ -11,13 +11,13 @@ require (
 	github.com/K-Phoen/grabana v0.22.1
 	github.com/ethereum/go-ethereum v1.13.8
 	github.com/go-resty/resty/v2 v2.11.0
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.30.0
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240607135320-c9bc0a2ac0ce
-	github.com/smartcontractkit/chainlink-testing-framework v1.30.2
+	github.com/smartcontractkit/chainlink-testing-framework v1.30.3
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c

integration-tests/load/go.sum

@@ -1344,8 +1344,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
@@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.2 h1:PKAg0nVT6clcTYebm3Q0rhTId+5xE9wYyvFxxSg4+lI=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.2/go.mod h1:oEIggLGWyWfLkjWvuXLol8inUT4YbBb06fJx/S60gQ4=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.3 h1:3mL0dBI/+8nTSh9GdxYyhNIAPTjSBThPGJnjcjjDaIg=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.3/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

integration-tests/testsetups/ocr.go

@@ -27,7 +27,7 @@ import (
 	"github.com/smartcontractkit/libocr/gethwrappers2/ocr2aggregator"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
-	ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client"
+	ctf_client "github.com/smartcontractkit/chainlink-testing-framework/client"
 	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s/environment"
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s/pkg/helm/chainlink"
@@ -71,7 +71,7 @@ type OCRSoakTest struct {
 	log              zerolog.Logger
 	bootstrapNode    *client.ChainlinkK8sClient
 	workerNodes      []*client.ChainlinkK8sClient
-	mockServer       *ctfClient.MockserverClient
+	mockServer       *ctf_client.MockserverClient
 	filterQuery      geth.FilterQuery
 
 	ocrRoundStates []*testreporters.OCRRoundState
@@ -83,7 +83,8 @@ type OCRSoakTest struct {
 	ocrV2Instances   []contracts.OffchainAggregatorV2
 	ocrV2InstanceMap map[string]contracts.OffchainAggregatorV2 // address : instance
 
-	rpcNetwork blockchain.EVMNetwork // network configuration for the blockchain node
+	rpcNetwork    blockchain.EVMNetwork // network configuration for the blockchain node
+	reorgHappened bool                  // flag to indicate if a reorg happened during the test
 }
 
 // NewOCRSoakTest creates a new OCR soak test to setup and run
@@ -193,6 +194,12 @@ func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTe
 	o.testEnvironment = testEnv
 	o.namespace = testEnv.Cfg.Namespace
 
+	// If the test is using the remote runner, we don't need to set the network URLs
+	// as the remote runner will handle that
+	if o.Environment().WillUseRemoteRunner() {
+		return
+	}
+
 	o.rpcNetwork = nodeNetwork
 	if o.rpcNetwork.Simulated && o.rpcNetwork.Name == "Anvil" {
 		if testEnv.Cfg.InsideK8s {
@@ -202,6 +209,19 @@ func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTe
 			// Test is running locally, set forwarded URL of Anvil blockchain node
 			o.rpcNetwork.URLs = []string{anvilChart.ForwardedWSURL}
 		}
+	} else if o.rpcNetwork.Simulated && o.rpcNetwork.Name == blockchain.SimulatedEVMNetwork.Name {
+		if testEnv.Cfg.InsideK8s {
+			// Test is running inside K8s
+			o.rpcNetwork.URLs = blockchain.SimulatedEVMNetwork.URLs
+		} else {
+			// Test is running locally, set forwarded URL of Geth blockchain node
+			wsURLs := o.testEnvironment.URLs[blockchain.SimulatedEVMNetwork.Name+"_internal"]
+			httpURLs := o.testEnvironment.URLs[blockchain.SimulatedEVMNetwork.Name+"_internal_http"]
+			require.NotEmpty(o.t, wsURLs, "Forwarded Geth URLs should not be empty")
+			require.NotEmpty(o.t, httpURLs, "Forwarded Geth URLs should not be empty")
+			o.rpcNetwork.URLs = wsURLs
+			o.rpcNetwork.HTTPURLs = httpURLs
+		}
 	}
 }
 
@@ -218,7 +238,7 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) {
 	nodes, err := client.ConnectChainlinkNodes(o.testEnvironment)
 	require.NoError(o.t, err, "Connecting to chainlink nodes shouldn't fail")
 	o.bootstrapNode, o.workerNodes = nodes[0], nodes[1:]
-	o.mockServer, err = ctfClient.ConnectMockServer(o.testEnvironment)
+	o.mockServer, err = ctf_client.ConnectMockServer(o.testEnvironment)
 	require.NoError(o.t, err, "Creating mockserver clients shouldn't fail")
 
 	linkContract, err := contracts.DeployLinkTokenContract(o.log, seth)
@@ -379,6 +399,7 @@ type OCRSoakTestState struct {
 	BootStrapNodeURL string   `toml:"bootstrapNodeURL"`
 	WorkerNodeURLs   []string `toml:"workerNodeURLs"`
 	ChainURL         string   `toml:"chainURL"`
+	ReorgHappened    bool     `toml:"reorgHappened"`
 	MockServerURL    string   `toml:"mockServerURL"`
 }
 
@@ -404,6 +425,7 @@ func (o *OCRSoakTest) SaveState() error {
 		MockServerURL:    "http://mockserver:1080", // TODO: Make this dynamic
 		BootStrapNodeURL: o.bootstrapNode.URL(),
 		WorkerNodeURLs:   workerNodeURLs,
+		ReorgHappened:    o.reorgHappened,
 	}
 	data, err := toml.Marshal(testState)
 	if err != nil {
@@ -454,6 +476,7 @@ func (o *OCRSoakTest) LoadState() error {
 	o.timeLeft = testState.TestDuration - testState.TimeRunning
 	o.startTime = testState.StartTime
 	o.startingBlockNum = testState.StartingBlockNum
+	o.reorgHappened = testState.ReorgHappened
 	o.Config.OCR.Soak.OCRVersion = &testState.OCRVersion
 
 	o.bootstrapNode, err = client.ConnectChainlinkNodeURL(testState.BootStrapNodeURL)
@@ -485,7 +508,7 @@ func (o *OCRSoakTest) LoadState() error {
 		}
 	}
 
-	o.mockServer, err = ctfClient.ConnectMockServerURL(testState.MockServerURL)
+	o.mockServer, err = ctf_client.ConnectMockServerURL(testState.MockServerURL)
 	if err != nil {
 		return err
 	}
@@ -562,6 +585,22 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) {
 	err := o.observeOCREvents()
 	require.NoError(o.t, err, "Error subscribing to OCR events")
 
+	// Schedule blockchain re-org if needed
+	// Reorg only avaible for Simulated Geth
+	var reorgCh <-chan time.Time
+	n := o.Config.GetNetworkConfig()
+	if n.IsSimulatedGethSelected() && n.GethReorgConfig.Enabled {
+		var reorgDelay time.Duration
+		if n.GethReorgConfig.DelayCreate.Duration > testDuration {
+			// This may happen when test is resumed and the reorg delay is longer than the time left
+			o.log.Warn().Msg("Reorg delay is longer than test duration, reorg scheduled immediately")
+			reorgDelay = 0
+		} else {
+			reorgDelay = n.GethReorgConfig.DelayCreate.Duration
+		}
+		reorgCh = time.After(reorgDelay)
+	}
+
 	for {
 		select {
 		case <-interruption:
@@ -595,6 +634,12 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) {
 				newValue = rand.Intn(256) + 1 // #nosec G404 - kudos to you if you actually find a way to exploit this
 			}
 			lastValue = newValue
+
+		// Schedule blockchain re-org if needed
+		case <-reorgCh:
+			if !o.reorgHappened {
+				o.startBlockchainReorg(o.Config.GetNetworkConfig().GethReorgConfig.Depth)
+			}
 		}
 	}
 }
@@ -610,6 +655,22 @@ func (o *OCRSoakTest) complete() {
 	o.TestReporter.RecordEvents(o.ocrRoundStates, o.testIssues)
 }
 
+func (o *OCRSoakTest) startBlockchainReorg(depth int) {
+	if !o.Config.GetNetworkConfig().IsSimulatedGethSelected() {
+		require.FailNow(o.t, "Reorg only available for Simulated Geth")
+		return
+	}
+
+	client := ctf_client.NewRPCClient(o.rpcNetwork.HTTPURLs[0])
+	o.log.Info().
+		Str("URL", client.URL).
+		Int("depth", depth).
+		Msg("Starting blockchain reorg on Simulated Geth chain")
+	err := client.GethSetHead(depth)
+	require.NoError(o.t, err, "Error starting blockchain reorg on Simulated Geth chain")
+	o.reorgHappened = true
+}
+
 // setFilterQuery to look for all events that happened
 func (o *OCRSoakTest) setFilterQuery() {
 	ocrAddresses := o.getContractAddresses()