diff --git a/integration-tests/Makefile b/integration-tests/Makefile index af3307dd395..5b3b3a2aae2 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -158,6 +158,16 @@ test_soak_ocr_reorg_2: . ./scripts/check_base64_env_var.sh go test -v -count=1 -run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$$ ./soak +.PHONY: test_soak_ocr_gas_spike +test_soak_ocr_gas_spike: + . ./scripts/check_base64_env_var.sh + go test -v -count=1 -run ^TestOCRSoak_GasSpike$$ ./soak + +.PHONY: test_soak_ocr_gas_limit_change +test_soak_ocr_gas_limit_change: + . ./scripts/check_base64_env_var.sh + go test -v -count=1 -run ^TestOCRSoak_ChangeBlockGasLimit$$ ./soak + .PHONY: test_soak_forwarder_ocr test_soak_forwarder_ocr: . ./scripts/check_base64_env_var.sh diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 1265e54dc16..c3257029ace 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -28,7 +28,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240614134652-1fb0b48758af - github.com/smartcontractkit/chainlink-testing-framework v1.30.4 + github.com/smartcontractkit/chainlink-testing-framework v1.30.9 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c @@ -50,7 +50,6 @@ require ( exclude github.com/hashicorp/consul v1.2.1 replace ( - // Pin K8s versions as their updates are highly disruptive and go mod keeps wanting to update them k8s.io/api => k8s.io/api v0.25.11 k8s.io/client-go => k8s.io/client-go v0.25.11 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d diff --git a/integration-tests/go.sum b/integration-tests/go.sum index fc0614fdb9e..c7792ef3193 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.4 h1:kf6zRL6v5D047gynYNNqXGl9QBvnQSa4LMs1iHLRu64= -github.com/smartcontractkit/chainlink-testing-framework v1.30.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework v1.30.9 h1:U8TqnPn2pfe+jSojeZL58sXBgfHFBHRsnm3izLthaYw= +github.com/smartcontractkit/chainlink-testing-framework v1.30.9/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 1f102b9cc2c..6858bcc7fd9 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240614134652-1fb0b48758af - github.com/smartcontractkit/chainlink-testing-framework v1.30.4 + github.com/smartcontractkit/chainlink-testing-framework v1.30.9 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 3522cd16d2e..f27109b799e 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.4 h1:kf6zRL6v5D047gynYNNqXGl9QBvnQSa4LMs1iHLRu64= -github.com/smartcontractkit/chainlink-testing-framework v1.30.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework v1.30.9 h1:U8TqnPn2pfe+jSojeZL58sXBgfHFBHRsnm3izLthaYw= +github.com/smartcontractkit/chainlink-testing-framework v1.30.9/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/soak/ocr_test.go b/integration-tests/soak/ocr_test.go index f8120d2ae0f..789c7de8e3b 100644 --- a/integration-tests/soak/ocr_test.go +++ b/integration-tests/soak/ocr_test.go @@ -37,6 +37,19 @@ func TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled(t *testing.T) { runOCRSoakTest(t, config, "") } +func TestOCRSoak_GasSpike(t *testing.T) { + config, err := tc.GetConfig(t.Name(), tc.OCR) + require.NoError(t, err, "Error getting config") + runOCRSoakTest(t, config, "") +} + +// TestOCRSoak_ChangeBlockGasLimit changes next block gas limit and sets it to percentage of last gasUsed in previous block creating congestion +func TestOCRSoak_ChangeBlockGasLimit(t *testing.T) { + config, err := tc.GetConfig(t.Name(), tc.OCR) + require.NoError(t, err, "Error getting config") + runOCRSoakTest(t, config, "") +} + func runOCRSoakTest(t *testing.T, config tc.TestConfig, customNetworkTOML string) { l := logging.GetTestLogger(t) diff --git a/integration-tests/testconfig/ocr/ocr.toml b/integration-tests/testconfig/ocr/ocr.toml index 30fc9655bf9..6c5745807b4 100644 --- a/integration-tests/testconfig/ocr/ocr.toml +++ b/integration-tests/testconfig/ocr/ocr.toml @@ -135,4 +135,40 @@ test_duration="15m" [TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled.OCR.Soak] ocr_version="1" number_of_contracts=2 -time_between_rounds="1m" \ No newline at end of file +time_between_rounds="1m" + +# OCR soak test configuration with gas spike on Anvil network +[TestOCRSoak_GasSpike.Common] +chainlink_node_funding = 100 +[TestOCRSoak_GasSpike.OCR.Common] +test_duration="15m" +[TestOCRSoak_GasSpike.OCR.Soak] +ocr_version="1" +number_of_contracts=2 +time_between_rounds="1m" +[TestOCRSoak_GasSpike.Network] +selected_networks=["Anvil"] +[TestOCRSoak_GasSpike.Network.AnvilConfigs.anvil.GasSpikeSimulation] +enabled = true +start_gas_price = 2000000000 +gas_rise_percentage = 0.7 +gas_spike = true +delay_create = "1m" +duration = "3m" + +# OCR soak test configuration with change to gas limit on Anvil network +[TestOCRSoak_ChangeBlockGasLimit.Common] +chainlink_node_funding = 100 +[TestOCRSoak_ChangeBlockGasLimit.OCR.Common] +test_duration="15m" +[TestOCRSoak_ChangeBlockGasLimit.OCR.Soak] +ocr_version="1" +number_of_contracts=2 +time_between_rounds="1m" +[TestOCRSoak_ChangeBlockGasLimit.Network] +selected_networks=["Anvil"] +[TestOCRSoak_ChangeBlockGasLimit.Network.AnvilConfigs.anvil.GasLimitSimulation] +enabled = true +next_gas_limit_percentage = 0.5 +delay_create = "1m" +duration = "3m" \ No newline at end of file diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index 02f4db9c51b..fcff17874ec 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -83,8 +83,10 @@ type OCRSoakTest struct { ocrV2Instances []contracts.OffchainAggregatorV2 ocrV2InstanceMap map[string]contracts.OffchainAggregatorV2 // address : instance - rpcNetwork blockchain.EVMNetwork // network configuration for the blockchain node - reorgHappened bool // flag to indicate if a reorg happened during the test + rpcNetwork blockchain.EVMNetwork // network configuration for the blockchain node + reorgHappened bool // flag to indicate if a reorg happened during the test + gasSpikeSimulationHappened bool // flag to indicate if a gas spike simulation happened during the test + gasLimitSimulationHappened bool // flag to indicate if a gas limit simulation happened during the test } // NewOCRSoakTest creates a new OCR soak test to setup and run @@ -208,6 +210,7 @@ func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTe } else { // Test is running locally, set forwarded URL of Anvil blockchain node o.rpcNetwork.URLs = []string{anvilChart.ForwardedWSURL} + o.rpcNetwork.HTTPURLs = []string{anvilChart.ForwardedHTTPURL} } } else if o.rpcNetwork.Simulated && o.rpcNetwork.Name == blockchain.SimulatedEVMNetwork.Name { if testEnv.Cfg.InsideK8s { @@ -585,10 +588,10 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { err := o.observeOCREvents() require.NoError(o.t, err, "Error subscribing to OCR events") + n := o.Config.GetNetworkConfig() + // Schedule blockchain re-org if needed // Reorg only avaible for Simulated Geth - var reorgCh <-chan time.Time - n := o.Config.GetNetworkConfig() if n.IsSimulatedGethSelected() && n.GethReorgConfig.Enabled { var reorgDelay time.Duration if n.GethReorgConfig.DelayCreate.Duration > testDuration { @@ -598,7 +601,47 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { } else { reorgDelay = n.GethReorgConfig.DelayCreate.Duration } - reorgCh = time.After(reorgDelay) + time.AfterFunc(reorgDelay, func() { + if !o.reorgHappened { + o.startGethBlockchainReorg(o.rpcNetwork, n.GethReorgConfig) + } + }) + } + + // Schedule gas simulations if needed + // Gas simulation only available for Anvil + if o.rpcNetwork.Name == "Anvil" { + ac := o.Config.GetNetworkConfig().AnvilConfigs["ANVIL"] + if ac != nil && ac.GasSpikeSimulation.Enabled { + var delay time.Duration + if ac.GasSpikeSimulation.DelayCreate.Duration > testDuration { + // This may happen when test is resumed and the reorg delay is longer than the time left + o.log.Warn().Msg("Gas spike simulation delay is longer than test duration, gas simulation scheduled immediately") + delay = 0 + } else { + delay = ac.GasSpikeSimulation.DelayCreate.Duration + } + time.AfterFunc(delay, func() { + if !o.gasSpikeSimulationHappened { + o.startAnvilGasSpikeSimulation(o.rpcNetwork, ac.GasSpikeSimulation) + } + }) + } + if ac != nil && ac.GasLimitSimulation.Enabled { + var delay time.Duration + if ac.GasLimitSimulation.DelayCreate.Duration > testDuration { + // This may happen when test is resumed and the reorg delay is longer than the time left + o.log.Warn().Msg("Gas limit simulation delay is longer than test duration, gas simulation scheduled immediately") + delay = 0 + } else { + delay = ac.GasLimitSimulation.DelayCreate.Duration + } + time.AfterFunc(delay, func() { + if !o.gasLimitSimulationHappened { + o.startAnvilGasLimitSimulation(o.rpcNetwork, ac.GasLimitSimulation) + } + }) + } } for { @@ -634,12 +677,6 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { newValue = rand.Intn(256) + 1 // #nosec G404 - kudos to you if you actually find a way to exploit this } lastValue = newValue - - // Schedule blockchain re-org if needed - case <-reorgCh: - if !o.reorgHappened { - o.startBlockchainReorg(o.Config.GetNetworkConfig().GethReorgConfig.Depth) - } } } } @@ -655,22 +692,54 @@ func (o *OCRSoakTest) complete() { o.TestReporter.RecordEvents(o.ocrRoundStates, o.testIssues) } -func (o *OCRSoakTest) startBlockchainReorg(depth int) { - if !o.Config.GetNetworkConfig().IsSimulatedGethSelected() { - require.FailNow(o.t, "Reorg only available for Simulated Geth") - return - } - - client := ctf_client.NewRPCClient(o.rpcNetwork.HTTPURLs[0]) +func (o *OCRSoakTest) startGethBlockchainReorg(network blockchain.EVMNetwork, conf ctf_config.ReorgConfig) { + client := ctf_client.NewRPCClient(network.HTTPURLs[0]) o.log.Info(). Str("URL", client.URL). - Int("depth", depth). + Int("Depth", conf.Depth). Msg("Starting blockchain reorg on Simulated Geth chain") - err := client.GethSetHead(depth) + err := client.GethSetHead(conf.Depth) require.NoError(o.t, err, "Error starting blockchain reorg on Simulated Geth chain") o.reorgHappened = true } +func (o *OCRSoakTest) startAnvilGasSpikeSimulation(network blockchain.EVMNetwork, conf ctf_config.GasSpikeSimulationConfig) { + client := ctf_client.NewRPCClient(network.HTTPURLs[0]) + o.log.Info(). + Str("URL", client.URL). + Any("GasSpikeSimulationConfig", conf). + Msg("Starting gas spike simulation on Anvil chain") + err := client.ModulateBaseFeeOverDuration(o.log, conf.StartGasPrice, conf.GasRisePercentage, conf.Duration.Duration, conf.GasSpike) + require.NoError(o.t, err, "Error starting gas simulation on Anvil chain") + o.gasSpikeSimulationHappened = true +} + +func (o *OCRSoakTest) startAnvilGasLimitSimulation(network blockchain.EVMNetwork, conf ctf_config.GasLimitSimulationConfig) { + client := ctf_client.NewRPCClient(network.HTTPURLs[0]) + latestBlock, err := o.seth.Client.BlockByNumber(context.Background(), nil) + require.NoError(o.t, err) + newGasLimit := int64(math.Ceil(float64(latestBlock.GasUsed()) * conf.NextGasLimitPercentage)) + o.log.Info(). + Str("URL", client.URL). + Any("GasLimitSimulationConfig", conf). + Uint64("LatestBlock", latestBlock.Number().Uint64()). + Uint64("LatestGasUsed", latestBlock.GasUsed()). + Uint64("LatestGasLimit", latestBlock.GasLimit()). + Int64("NewGasLimit", newGasLimit). + Msg("Starting new gas limit simulation on Anvil chain") + err = client.AnvilSetBlockGasLimit([]interface{}{newGasLimit}) + require.NoError(o.t, err, "Error starting gas simulation on Anvil chain") + time.Sleep(conf.Duration.Duration) + o.log.Info(). + Str("URL", client.URL). + Any("GasLimitSimulationConfig", conf). + Uint64("LatestGasLimit", latestBlock.GasLimit()). + Msg("Returning to old gas limit simulation on Anvil chain") + err = client.AnvilSetBlockGasLimit([]interface{}{latestBlock.GasLimit()}) + require.NoError(o.t, err, "Error starting gas simulation on Anvil chain") + o.gasLimitSimulationHappened = true +} + // setFilterQuery to look for all events that happened func (o *OCRSoakTest) setFilterQuery() { ocrAddresses := o.getContractAddresses()