TT-1154 Add new OCR soak tests with gas simulations (#13576)

* Add TestOCRSoak_GasSpike test

* Refactor reorg start logic

* Start gas spike simulation after 1 min since test is 15 min long

* Add makefile

* Update CTF

* Add TestOCRSoak_ChangeBlockGasLimit test

* Bump CTF

integration-tests/Makefile

@@ -158,6 +158,16 @@ test_soak_ocr_reorg_2:
 	. ./scripts/check_base64_env_var.sh
 	go test -v -count=1 -run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$$ ./soak	
 
+.PHONY: test_soak_ocr_gas_spike
+test_soak_ocr_gas_spike:
+	. ./scripts/check_base64_env_var.sh
+	go test -v -count=1 -run ^TestOCRSoak_GasSpike$$ ./soak
+
+.PHONY: test_soak_ocr_gas_limit_change
+test_soak_ocr_gas_limit_change:
+	. ./scripts/check_base64_env_var.sh
+	go test -v -count=1 -run ^TestOCRSoak_ChangeBlockGasLimit$$ ./soak
+
 .PHONY: test_soak_forwarder_ocr
 test_soak_forwarder_ocr:
 	. ./scripts/check_base64_env_var.sh

integration-tests/go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240614134652-1fb0b48758af
-	github.com/smartcontractkit/chainlink-testing-framework v1.30.4
+	github.com/smartcontractkit/chainlink-testing-framework v1.30.9
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
@@ -50,7 +50,6 @@ require (
 exclude github.com/hashicorp/consul v1.2.1
 
 replace (
-	// Pin K8s versions as their updates are highly disruptive and go mod keeps wanting to update them
 	k8s.io/api => k8s.io/api v0.25.11
 	k8s.io/client-go => k8s.io/client-go v0.25.11
 	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d

integration-tests/go.sum

@@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.4 h1:kf6zRL6v5D047gynYNNqXGl9QBvnQSa4LMs1iHLRu64=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.9 h1:U8TqnPn2pfe+jSojeZL58sXBgfHFBHRsnm3izLthaYw=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.9/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

integration-tests/load/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240614134652-1fb0b48758af
-	github.com/smartcontractkit/chainlink-testing-framework v1.30.4
+	github.com/smartcontractkit/chainlink-testing-framework v1.30.9
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c

integration-tests/load/go.sum

@@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.4 h1:kf6zRL6v5D047gynYNNqXGl9QBvnQSa4LMs1iHLRu64=
-github.com/smartcontractkit/chainlink-testing-framework v1.30.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.9 h1:U8TqnPn2pfe+jSojeZL58sXBgfHFBHRsnm3izLthaYw=
+github.com/smartcontractkit/chainlink-testing-framework v1.30.9/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

integration-tests/soak/ocr_test.go

@@ -37,6 +37,19 @@ func TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled(t *testing.T) {
 	runOCRSoakTest(t, config, "")
 }
 
+func TestOCRSoak_GasSpike(t *testing.T) {
+	config, err := tc.GetConfig(t.Name(), tc.OCR)
+	require.NoError(t, err, "Error getting config")
+	runOCRSoakTest(t, config, "")
+}
+
+// TestOCRSoak_ChangeBlockGasLimit changes next block gas limit and sets it to percentage of last gasUsed in previous block creating congestion
+func TestOCRSoak_ChangeBlockGasLimit(t *testing.T) {
+	config, err := tc.GetConfig(t.Name(), tc.OCR)
+	require.NoError(t, err, "Error getting config")
+	runOCRSoakTest(t, config, "")
+}
+
 func runOCRSoakTest(t *testing.T, config tc.TestConfig, customNetworkTOML string) {
 	l := logging.GetTestLogger(t)
 

integration-tests/testconfig/ocr/ocr.toml

@@ -135,4 +135,40 @@ test_duration="15m"
 [TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled.OCR.Soak]
 ocr_version="1"
 number_of_contracts=2
-time_between_rounds="1m"
+time_between_rounds="1m"
+
+# OCR soak test configuration with gas spike on Anvil network
+[TestOCRSoak_GasSpike.Common]
+chainlink_node_funding = 100
+[TestOCRSoak_GasSpike.OCR.Common]
+test_duration="15m"
+[TestOCRSoak_GasSpike.OCR.Soak]
+ocr_version="1"
+number_of_contracts=2
+time_between_rounds="1m"
+[TestOCRSoak_GasSpike.Network]
+selected_networks=["Anvil"]
+[TestOCRSoak_GasSpike.Network.AnvilConfigs.anvil.GasSpikeSimulation]
+enabled = true
+start_gas_price = 2000000000
+gas_rise_percentage = 0.7
+gas_spike = true
+delay_create = "1m"
+duration = "3m"
+
+# OCR soak test configuration with change to gas limit on Anvil network
+[TestOCRSoak_ChangeBlockGasLimit.Common]
+chainlink_node_funding = 100
+[TestOCRSoak_ChangeBlockGasLimit.OCR.Common]
+test_duration="15m"
+[TestOCRSoak_ChangeBlockGasLimit.OCR.Soak]
+ocr_version="1"
+number_of_contracts=2
+time_between_rounds="1m"
+[TestOCRSoak_ChangeBlockGasLimit.Network]
+selected_networks=["Anvil"]
+[TestOCRSoak_ChangeBlockGasLimit.Network.AnvilConfigs.anvil.GasLimitSimulation]
+enabled = true
+next_gas_limit_percentage = 0.5
+delay_create = "1m"
+duration = "3m"

integration-tests/testsetups/ocr.go

@@ -83,8 +83,10 @@ type OCRSoakTest struct {
 	ocrV2Instances   []contracts.OffchainAggregatorV2
 	ocrV2InstanceMap map[string]contracts.OffchainAggregatorV2 // address : instance
 
-	rpcNetwork    blockchain.EVMNetwork // network configuration for the blockchain node
-	reorgHappened bool                  // flag to indicate if a reorg happened during the test
+	rpcNetwork                 blockchain.EVMNetwork // network configuration for the blockchain node
+	reorgHappened              bool                  // flag to indicate if a reorg happened during the test
+	gasSpikeSimulationHappened bool                  // flag to indicate if a gas spike simulation happened during the test
+	gasLimitSimulationHappened bool                  // flag to indicate if a gas limit simulation happened during the test
 }
 
 // NewOCRSoakTest creates a new OCR soak test to setup and run
@@ -208,6 +210,7 @@ func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTe
 		} else {
 			// Test is running locally, set forwarded URL of Anvil blockchain node
 			o.rpcNetwork.URLs = []string{anvilChart.ForwardedWSURL}
+			o.rpcNetwork.HTTPURLs = []string{anvilChart.ForwardedHTTPURL}
 		}
 	} else if o.rpcNetwork.Simulated && o.rpcNetwork.Name == blockchain.SimulatedEVMNetwork.Name {
 		if testEnv.Cfg.InsideK8s {
@@ -585,10 +588,10 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) {
 	err := o.observeOCREvents()
 	require.NoError(o.t, err, "Error subscribing to OCR events")
 
+	n := o.Config.GetNetworkConfig()
+
 	// Schedule blockchain re-org if needed
 	// Reorg only avaible for Simulated Geth
-	var reorgCh <-chan time.Time
-	n := o.Config.GetNetworkConfig()
 	if n.IsSimulatedGethSelected() && n.GethReorgConfig.Enabled {
 		var reorgDelay time.Duration
 		if n.GethReorgConfig.DelayCreate.Duration > testDuration {
@@ -598,7 +601,47 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) {
 		} else {
 			reorgDelay = n.GethReorgConfig.DelayCreate.Duration
 		}
-		reorgCh = time.After(reorgDelay)
+		time.AfterFunc(reorgDelay, func() {
+			if !o.reorgHappened {
+				o.startGethBlockchainReorg(o.rpcNetwork, n.GethReorgConfig)
+			}
+		})
+	}
+
+	// Schedule gas simulations if needed
+	// Gas simulation only available for Anvil
+	if o.rpcNetwork.Name == "Anvil" {
+		ac := o.Config.GetNetworkConfig().AnvilConfigs["ANVIL"]
+		if ac != nil && ac.GasSpikeSimulation.Enabled {
+			var delay time.Duration
+			if ac.GasSpikeSimulation.DelayCreate.Duration > testDuration {
+				// This may happen when test is resumed and the reorg delay is longer than the time left
+				o.log.Warn().Msg("Gas spike simulation delay is longer than test duration, gas simulation scheduled immediately")
+				delay = 0
+			} else {
+				delay = ac.GasSpikeSimulation.DelayCreate.Duration
+			}
+			time.AfterFunc(delay, func() {
+				if !o.gasSpikeSimulationHappened {
+					o.startAnvilGasSpikeSimulation(o.rpcNetwork, ac.GasSpikeSimulation)
+				}
+			})
+		}
+		if ac != nil && ac.GasLimitSimulation.Enabled {
+			var delay time.Duration
+			if ac.GasLimitSimulation.DelayCreate.Duration > testDuration {
+				// This may happen when test is resumed and the reorg delay is longer than the time left
+				o.log.Warn().Msg("Gas limit simulation delay is longer than test duration, gas simulation scheduled immediately")
+				delay = 0
+			} else {
+				delay = ac.GasLimitSimulation.DelayCreate.Duration
+			}
+			time.AfterFunc(delay, func() {
+				if !o.gasLimitSimulationHappened {
+					o.startAnvilGasLimitSimulation(o.rpcNetwork, ac.GasLimitSimulation)
+				}
+			})
+		}
 	}
 
 	for {
@@ -634,12 +677,6 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) {
 				newValue = rand.Intn(256) + 1 // #nosec G404 - kudos to you if you actually find a way to exploit this
 			}
 			lastValue = newValue
-
-		// Schedule blockchain re-org if needed
-		case <-reorgCh:
-			if !o.reorgHappened {
-				o.startBlockchainReorg(o.Config.GetNetworkConfig().GethReorgConfig.Depth)
-			}
 		}
 	}
 }
@@ -655,22 +692,54 @@ func (o *OCRSoakTest) complete() {
 	o.TestReporter.RecordEvents(o.ocrRoundStates, o.testIssues)
 }
 
-func (o *OCRSoakTest) startBlockchainReorg(depth int) {
-	if !o.Config.GetNetworkConfig().IsSimulatedGethSelected() {
-		require.FailNow(o.t, "Reorg only available for Simulated Geth")
-		return
-	}
-
-	client := ctf_client.NewRPCClient(o.rpcNetwork.HTTPURLs[0])
+func (o *OCRSoakTest) startGethBlockchainReorg(network blockchain.EVMNetwork, conf ctf_config.ReorgConfig) {
+	client := ctf_client.NewRPCClient(network.HTTPURLs[0])
 	o.log.Info().
 		Str("URL", client.URL).
-		Int("depth", depth).
+		Int("Depth", conf.Depth).
 		Msg("Starting blockchain reorg on Simulated Geth chain")
-	err := client.GethSetHead(depth)
+	err := client.GethSetHead(conf.Depth)
 	require.NoError(o.t, err, "Error starting blockchain reorg on Simulated Geth chain")
 	o.reorgHappened = true
 }
 
+func (o *OCRSoakTest) startAnvilGasSpikeSimulation(network blockchain.EVMNetwork, conf ctf_config.GasSpikeSimulationConfig) {
+	client := ctf_client.NewRPCClient(network.HTTPURLs[0])
+	o.log.Info().
+		Str("URL", client.URL).
+		Any("GasSpikeSimulationConfig", conf).
+		Msg("Starting gas spike simulation on Anvil chain")
+	err := client.ModulateBaseFeeOverDuration(o.log, conf.StartGasPrice, conf.GasRisePercentage, conf.Duration.Duration, conf.GasSpike)
+	require.NoError(o.t, err, "Error starting gas simulation on Anvil chain")
+	o.gasSpikeSimulationHappened = true
+}
+
+func (o *OCRSoakTest) startAnvilGasLimitSimulation(network blockchain.EVMNetwork, conf ctf_config.GasLimitSimulationConfig) {
+	client := ctf_client.NewRPCClient(network.HTTPURLs[0])
+	latestBlock, err := o.seth.Client.BlockByNumber(context.Background(), nil)
+	require.NoError(o.t, err)
+	newGasLimit := int64(math.Ceil(float64(latestBlock.GasUsed()) * conf.NextGasLimitPercentage))
+	o.log.Info().
+		Str("URL", client.URL).
+		Any("GasLimitSimulationConfig", conf).
+		Uint64("LatestBlock", latestBlock.Number().Uint64()).
+		Uint64("LatestGasUsed", latestBlock.GasUsed()).
+		Uint64("LatestGasLimit", latestBlock.GasLimit()).
+		Int64("NewGasLimit", newGasLimit).
+		Msg("Starting new gas limit simulation on Anvil chain")
+	err = client.AnvilSetBlockGasLimit([]interface{}{newGasLimit})
+	require.NoError(o.t, err, "Error starting gas simulation on Anvil chain")
+	time.Sleep(conf.Duration.Duration)
+	o.log.Info().
+		Str("URL", client.URL).
+		Any("GasLimitSimulationConfig", conf).
+		Uint64("LatestGasLimit", latestBlock.GasLimit()).
+		Msg("Returning to old gas limit simulation on Anvil chain")
+	err = client.AnvilSetBlockGasLimit([]interface{}{latestBlock.GasLimit()})
+	require.NoError(o.t, err, "Error starting gas simulation on Anvil chain")
+	o.gasLimitSimulationHappened = true
+}
+
 // setFilterQuery to look for all events that happened
 func (o *OCRSoakTest) setFilterQuery() {
 	ocrAddresses := o.getContractAddresses()