From 3f83f9e8e66029c78a52e2c1eeb5dfb95a615f55 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Wed, 18 Sep 2024 03:40:08 -0400 Subject: [PATCH] Enables OOO Execution for E2E Tests (#14392) * Enables OOO Execution for E2E Tests * Out of order false by default * Port over CCIP fix * Changeset * Fix changeset * Fix lint --- .changeset/rare-crabs-tell.md | 5 ++ .../ccip/testhelpers/ccip_contracts.go | 2 + .../ocr2/plugins/ccip/testhelpers/config.go | 1 + .../testhelpers_1_4_0/ccip_contracts_1_4_0.go | 2 + .../testhelpers_1_4_0/config_1_4_0.go | 1 + .../ccip-tests/actions/ccip_helpers.go | 66 +++++++++++++------ .../ccip-tests/contracts/contract_deployer.go | 7 +- .../ccip-tests/contracts/contract_models.go | 54 ++++++++++++--- .../contracts/laneconfig/parse_contracts.go | 1 + .../ccip-tests/load/ccip_loadgen.go | 24 ++++++- integration-tests/ccip-tests/load/helper.go | 5 +- .../ccip-tests/smoke/ccip_test.go | 30 +++------ .../ccip-tests/testconfig/README.md | 4 ++ .../ccip-tests/testconfig/ccip.go | 1 + .../testconfig/tomls/ccip-default.toml | 10 ++- 15 files changed, 153 insertions(+), 60 deletions(-) create mode 100644 .changeset/rare-crabs-tell.md diff --git a/.changeset/rare-crabs-tell.md b/.changeset/rare-crabs-tell.md new file mode 100644 index 00000000000..9bd98213aec --- /dev/null +++ b/.changeset/rare-crabs-tell.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#added Adds the ability to use out of order execution transactions in CCIP E2E tests diff --git a/core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go b/core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go index 7b3351ce062..08020282c39 100644 --- a/core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go +++ b/core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go @@ -150,6 +150,7 @@ func NewExecOffchainConfig( RelativeBoostPerWaitHour float64, InflightCacheExpiry config.Duration, RootSnoozeTime config.Duration, + BatchingStrategyID uint32, ) ExecOffchainConfig { return ExecOffchainConfig{v1_2_0.JSONExecOffchainConfig{ DestOptimisticConfirmations: DestOptimisticConfirmations, @@ -157,6 +158,7 @@ func NewExecOffchainConfig( RelativeBoostPerWaitHour: RelativeBoostPerWaitHour, InflightCacheExpiry: InflightCacheExpiry, RootSnoozeTime: RootSnoozeTime, + BatchingStrategyID: BatchingStrategyID, }} } diff --git a/core/services/ocr2/plugins/ccip/testhelpers/config.go b/core/services/ocr2/plugins/ccip/testhelpers/config.go index c9d1ca5a126..4dcb627347f 100644 --- a/core/services/ocr2/plugins/ccip/testhelpers/config.go +++ b/core/services/ocr2/plugins/ccip/testhelpers/config.go @@ -70,6 +70,7 @@ func (c *CCIPContracts) createExecOffchainConfig(t *testing.T, inflightCacheExpi 0.07, *config.MustNewDuration(inflightCacheExpiry), *config.MustNewDuration(rootSnoozeTime), + uint32(0), ).Encode() require.NoError(t, err) return config diff --git a/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/ccip_contracts_1_4_0.go b/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/ccip_contracts_1_4_0.go index b8db2dfff7f..9906a7b365a 100644 --- a/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/ccip_contracts_1_4_0.go +++ b/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/ccip_contracts_1_4_0.go @@ -157,6 +157,7 @@ func NewExecOffchainConfig( RelativeBoostPerWaitHour float64, InflightCacheExpiry config.Duration, RootSnoozeTime config.Duration, + BatchingStrategyID uint32, ) ExecOffchainConfig { return ExecOffchainConfig{v1_2_0.JSONExecOffchainConfig{ DestOptimisticConfirmations: DestOptimisticConfirmations, @@ -164,6 +165,7 @@ func NewExecOffchainConfig( RelativeBoostPerWaitHour: RelativeBoostPerWaitHour, InflightCacheExpiry: InflightCacheExpiry, RootSnoozeTime: RootSnoozeTime, + BatchingStrategyID: BatchingStrategyID, }} } diff --git a/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/config_1_4_0.go b/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/config_1_4_0.go index 666ad79e59f..087c21e9333 100644 --- a/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/config_1_4_0.go +++ b/core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/config_1_4_0.go @@ -68,6 +68,7 @@ func (c *CCIPContracts) createExecOffchainConfig(t *testing.T, inflightCacheExpi 0.07, *config.MustNewDuration(inflightCacheExpiry), *config.MustNewDuration(rootSnoozeTime), + uint32(0), ).Encode() require.NoError(t, err) return config diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go index 429751e169e..06334560e05 100644 --- a/integration-tests/ccip-tests/actions/ccip_helpers.go +++ b/integration-tests/ccip-tests/actions/ccip_helpers.go @@ -185,6 +185,7 @@ type CCIPCommon struct { gasUpdateWatcherMu *sync.Mutex gasUpdateWatcher map[uint64]*big.Int // key - destchain id; value - timestamp of update GasUpdateEvents []contracts.GasUpdateEvent + AllowOutOfOrder bool } // FreeUpUnusedSpace sets nil to various elements of ccipModule which are only used @@ -273,6 +274,9 @@ func (ccipModule *CCIPCommon) CurseARM() (*types.Transaction, error) { func (ccipModule *CCIPCommon) LoadContractAddresses(conf *laneconfig.LaneConfig, noOfTokens *int) { if conf != nil { + if conf.AllowOutOfOrder { + ccipModule.AllowOutOfOrder = true + } if common.IsHexAddress(conf.FeeToken) { ccipModule.FeeToken = &contracts.LinkToken{ EthAddress: common.HexToAddress(conf.FeeToken), @@ -1626,7 +1630,7 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe if sendRequestedEvents, exists := value.([]*evm_2_evm_onramp.EVM2EVMOnRampCCIPSendRequested); exists { for _, sendRequestedEvent := range sendRequestedEvents { raw := sendRequestedEvent.Raw - hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(raw.BlockNumber))) + hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(raw.BlockNumber)) if err == nil { if hdr.Timestamp.After(lastSeenTimestamp) { foundAt = pointer.ToTime(hdr.Timestamp) @@ -1709,6 +1713,7 @@ func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested( // CCIPMsg constructs the message for a CCIP request func (sourceCCIP *SourceCCIPModule) CCIPMsg( receiver common.Address, + allowOutOfOrder bool, gasLimit *big.Int, ) (router.ClientEVM2AnyMessage, error) { length := sourceCCIP.MsgDataLength @@ -1750,9 +1755,17 @@ func (sourceCCIP *SourceCCIPModule) CCIPMsg( return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed encoding the receiver address: %w", err) } - extraArgsV1, err := testhelpers.GetEVMExtraArgsV1(gasLimit, false) + var extraArgs []byte + matchErr := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{ + contracts.OnRampContract: contracts.V1_5_0, + }) + if matchErr != nil { + extraArgs, err = testhelpers.GetEVMExtraArgsV1(gasLimit, false) + } else { + extraArgs, err = testhelpers.GetEVMExtraArgsV2(gasLimit, allowOutOfOrder) + } if err != nil { - return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed encoding the options field: %w", err) + return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed getting extra args: %w", err) } // form the message for transfer return router.ClientEVM2AnyMessage{ @@ -1760,7 +1773,7 @@ func (sourceCCIP *SourceCCIPModule) CCIPMsg( Data: []byte(data), TokenAmounts: tokenAndAmounts, FeeToken: common.HexToAddress(sourceCCIP.Common.FeeToken.Address()), - ExtraArgs: extraArgsV1, + ExtraArgs: extraArgs, }, nil } @@ -1775,7 +1788,7 @@ func (sourceCCIP *SourceCCIPModule) SendRequest( return common.Hash{}, d, nil, fmt.Errorf("failed getting the chain selector: %w", err) } // form the message for transfer - msg, err := sourceCCIP.CCIPMsg(receiver, gasLimit) + msg, err := sourceCCIP.CCIPMsg(receiver, sourceCCIP.Common.AllowOutOfOrder, gasLimit) if err != nil { return common.Hash{}, d, nil, fmt.Errorf("failed forming the ccip msg: %w", err) } @@ -2218,7 +2231,7 @@ func (destCCIP *DestCCIPModule) AssertNoReportAcceptedEventReceived(lggr *zerolo e, exists := value.(*evm_2_evm_offramp.EVM2EVMOffRampExecutionStateChanged) if exists { vLogs := e.Raw - hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, big.NewInt(int64(vLogs.BlockNumber))) + hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, new(big.Int).SetUint64(vLogs.BlockNumber)) if err != nil { return true } @@ -2259,7 +2272,7 @@ func (destCCIP *DestCCIPModule) AssertNoExecutionStateChangedEventReceived( e, exists := value.(*contracts.EVM2EVMOffRampExecutionStateChanged) if exists { vLogs := e.LogInfo - hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, big.NewInt(int64(vLogs.BlockNumber))) + hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, new(big.Int).SetUint64(vLogs.BlockNumber)) if err != nil { return true } @@ -2288,7 +2301,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( reqStat *testreporters.RequestStat, execState testhelpers.MessageExecutionState, ) (uint8, error) { - lggr.Info().Int64("seqNum", int64(seqNum)).Str("Timeout", timeout.String()).Msg("Waiting for ExecutionStateChanged event") + lggr.Info().Uint64("seqNum", seqNum).Str("Timeout", timeout.String()).Msg("Waiting for ExecutionStateChanged event") timer := time.NewTimer(timeout) defer timer.Stop() ticker := time.NewTicker(time.Second) @@ -2306,7 +2319,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( destCCIP.ExecStateChangedWatcher.Delete(seqNum) vLogs := e.LogInfo receivedAt := time.Now().UTC() - hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(vLogs.BlockNumber))) + hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(vLogs.BlockNumber)) if err == nil { receivedAt = hdr.Timestamp } @@ -2319,7 +2332,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( gasUsed = receipt.GasUsed } if testhelpers.MessageExecutionState(e.State) == execState { - lggr.Info().Int64("seqNum", int64(seqNum)).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received") + lggr.Info().Uint64("seqNum", seqNum).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received") reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, receivedAt.Sub(timeNow), testreporters.Success, &testreporters.TransactionStats{ @@ -2363,7 +2376,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted( prevEventAt time.Time, reqStat *testreporters.RequestStat, ) (*contracts.CommitStoreReportAccepted, time.Time, error) { - lggr.Info().Int64("seqNum", int64(seqNum)).Str("Timeout", timeout.String()).Msg("Waiting for ReportAccepted event") + lggr.Info().Uint64("seqNum", seqNum).Str("Timeout", timeout.String()).Msg("Waiting for ReportAccepted event") timer := time.NewTimer(timeout) defer timer.Stop() resetTimerCount := 0 @@ -2379,7 +2392,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted( // if the value is processed, delete it from the map destCCIP.ReportAcceptedWatcher.Delete(seqNum) receivedAt := time.Now().UTC() - hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(reportAccepted.LogInfo.BlockNumber))) + hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(reportAccepted.LogInfo.BlockNumber)) if err == nil { receivedAt = hdr.Timestamp } @@ -2488,7 +2501,7 @@ func (destCCIP *DestCCIPModule) AssertReportBlessed( // if the value is processed, delete it from the map destCCIP.ReportBlessedBySeqNum.Delete(seqNum) } - hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(vLogs.BlockNumber))) + hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(vLogs.BlockNumber)) if err == nil { receivedAt = hdr.Timestamp } @@ -2536,7 +2549,7 @@ func (destCCIP *DestCCIPModule) AssertSeqNumberExecuted( timeNow time.Time, reqStat *testreporters.RequestStat, ) error { - lggr.Info().Int64("seqNum", int64(seqNumberBefore)).Str("Timeout", timeout.String()).Msg("Waiting to be processed by commit store") + lggr.Info().Uint64("seqNum", seqNumberBefore).Str("Timeout", timeout.String()).Msg("Waiting to be processed by commit store") timer := time.NewTimer(timeout) defer timer.Stop() resetTimerCount := 0 @@ -2793,7 +2806,11 @@ func (lane *CCIPLane) AddToSentReqs(txHash common.Hash, reqStats []*testreporter func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) error { var ccipMultipleMsg []contracts.CCIPMsgData feeToken := common.HexToAddress(lane.Source.Common.FeeToken.Address()) - genericMsg, err := lane.Source.CCIPMsg(lane.Dest.ReceiverDapp.EthAddress, big.NewInt(DefaultDestinationGasLimit)) + genericMsg, err := lane.Source.CCIPMsg( + lane.Dest.ReceiverDapp.EthAddress, + lane.Source.Common.AllowOutOfOrder, + big.NewInt(DefaultDestinationGasLimit), + ) if err != nil { return fmt.Errorf("failed to form the ccip message: %w", err) } @@ -2885,10 +2902,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) func (lane *CCIPLane) SendRequests(noOfRequests int, gasLimit *big.Int) error { for i := 1; i <= noOfRequests; i++ { stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq+i), lane.SourceNetworkName, lane.DestNetworkName) - txHash, txConfirmationDur, fee, err := lane.Source.SendRequest( - lane.Dest.ReceiverDapp.EthAddress, - gasLimit, - ) + txHash, txConfirmationDur, fee, err := lane.Source.SendRequest(lane.Dest.ReceiverDapp.EthAddress, gasLimit) if err != nil { stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure, nil) return fmt.Errorf("could not send request: %w", err) @@ -3515,6 +3529,7 @@ func (lane *CCIPLane) DeployNewCCIPLane( srcConf = lane.SrcNetworkLaneCfg destConf = lane.DstNetworkLaneCfg commitAndExecOnSameDON = pointer.GetBool(testConf.CommitAndExecuteOnSameDON) + allowOutOfOrder = pointer.GetBool(testConf.AllowOutOfOrder) withPipeline = pointer.GetBool(testConf.TokenConfig.WithPipeline) configureCLNodes = !pointer.GetBool(testConf.ExistingDeployment) ) @@ -3529,6 +3544,13 @@ func (lane *CCIPLane) DeployNewCCIPLane( if err != nil { return fmt.Errorf("failed to create source module: %w", err) } + + // If AllowOutOfOrder is set globally in test config, the assumption is to set it for every lane. + // However, if this is set as false, and set as true for specific chain in lane_configs, then apply it + // only for a lane where source network is of that chain. + if allowOutOfOrder { + lane.Source.Common.AllowOutOfOrder = true + } lane.Dest, err = DefaultDestinationCCIPModule( lane.Logger, testConf, destChainClient, sourceChainClient.GetChainID().Uint64(), @@ -3781,12 +3803,18 @@ func SetOCR2Config( nodes = execNodes } if destCCIP.OffRamp != nil { + // Use out of order batching strategy if we expect to be sending out of order messages + batchingStrategyID := uint32(0) + if pointer.GetBool(testConf.AllowOutOfOrder) { + batchingStrategyID = uint32(1) + } execOffchainCfg, err := contracts.NewExecOffchainConfig( 1, BatchGasLimit, 0.7, *inflightExpiryExec, *commonconfig.MustNewDuration(RootSnoozeTime), + batchingStrategyID, ) if err != nil { return fmt.Errorf("failed to create exec offchain config: %w", err) diff --git a/integration-tests/ccip-tests/contracts/contract_deployer.go b/integration-tests/ccip-tests/contracts/contract_deployer.go index 211bbc1ebb4..579d16da4fc 100644 --- a/integration-tests/ccip-tests/contracts/contract_deployer.go +++ b/integration-tests/ccip-tests/contracts/contract_deployer.go @@ -77,7 +77,7 @@ func MatchContractVersionsOrAbove(requiredContractVersions map[Name]Version) err // if the version is less than 1.5.0, then token admin registry is not needed func NeedTokenAdminRegistry() bool { return MatchContractVersionsOrAbove(map[Name]Version{ - TokenPoolContract: V1_5_0_dev, + TokenPoolContract: V1_5_0, }) == nil } @@ -1044,6 +1044,7 @@ func (e *CCIPContractsDeployer) DeployOnRamp( MaxPerMsgGasLimit: 4_000_000, DefaultTokenFeeUSDCents: 50, DefaultTokenDestGasOverhead: 125_000, + EnforceOutOfOrder: false, }, evm_2_evm_onramp.RateLimiterConfig{ Capacity: opts.Capacity, @@ -1465,12 +1466,14 @@ func NewExecOnchainConfig( } } +// NewExecOffchainConfig creates a config for the OffChain portion of how CCIP operates func NewExecOffchainConfig( destOptimisticConfirmations uint32, batchGasLimit uint32, relativeBoostPerWaitHour float64, inflightCacheExpiry config.Duration, rootSnoozeTime config.Duration, + batchingStrategyID uint32, // See ccipexec package ) (ccipconfig.OffchainConfig, error) { switch VersionMap[OffRampContract] { case Latest: @@ -1480,6 +1483,7 @@ func NewExecOffchainConfig( relativeBoostPerWaitHour, inflightCacheExpiry, rootSnoozeTime, + batchingStrategyID, ), nil case V1_2_0: return testhelpers_1_4_0.NewExecOffchainConfig( @@ -1488,6 +1492,7 @@ func NewExecOffchainConfig( relativeBoostPerWaitHour, inflightCacheExpiry, rootSnoozeTime, + batchingStrategyID, ), nil default: return nil, fmt.Errorf("version not supported: %s", VersionMap[OffRampContract]) diff --git a/integration-tests/ccip-tests/contracts/contract_models.go b/integration-tests/ccip-tests/contracts/contract_models.go index e3f6e45fad2..83fe12a60a6 100644 --- a/integration-tests/ccip-tests/contracts/contract_models.go +++ b/integration-tests/ccip-tests/contracts/contract_models.go @@ -112,9 +112,9 @@ const ( var ( V1_2_0 = MustVersion("1.2.0") V1_4_0 = MustVersion("1.4.0") - V1_5_0_dev = MustVersion("1.5.0") - LatestPoolVersion = V1_5_0_dev - Latest = V1_5_0_dev + V1_5_0 = MustVersion("1.5.0") + LatestPoolVersion = V1_5_0 + Latest = V1_5_0 VersionMap = map[Name]Version{ PriceRegistryContract: V1_2_0, OffRampContract: Latest, @@ -1616,22 +1616,58 @@ func (w OnRampWrapper) ParseCCIPSendRequested(l types.Log) (uint64, error) { return 0, fmt.Errorf("no instance found to parse CCIPSendRequested") } -func (w OnRampWrapper) GetDynamicConfig(opts *bind.CallOpts) (uint32, error) { +// GetDynamicConfig retrieves the dynamic config for the onramp +func (w OnRampWrapper) GetDynamicConfig(opts *bind.CallOpts) (evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig, error) { if w.Latest != nil { cfg, err := w.Latest.GetDynamicConfig(opts) if err != nil { - return 0, err + return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{}, err } - return cfg.MaxDataBytes, nil + return cfg, nil } if w.V1_2_0 != nil { cfg, err := w.V1_2_0.GetDynamicConfig(opts) if err != nil { - return 0, err + return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{}, err } - return cfg.MaxDataBytes, nil + return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{ + Router: cfg.Router, + MaxNumberOfTokensPerMsg: cfg.MaxNumberOfTokensPerMsg, + DestGasOverhead: cfg.DestGasOverhead, + DestGasPerPayloadByte: cfg.DestGasPerPayloadByte, + DestDataAvailabilityOverheadGas: cfg.DestDataAvailabilityOverheadGas, + DestGasPerDataAvailabilityByte: cfg.DestGasPerDataAvailabilityByte, + DestDataAvailabilityMultiplierBps: cfg.DestDataAvailabilityMultiplierBps, + PriceRegistry: cfg.PriceRegistry, + MaxDataBytes: cfg.MaxDataBytes, + MaxPerMsgGasLimit: cfg.MaxPerMsgGasLimit, + }, nil + } + return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{}, fmt.Errorf("no instance found to get dynamic config") +} + +// SetDynamicConfig sets the dynamic config for the onramp +// Note that you cannot set only a single field, you must set all fields or they will be reset to zero values +// You can use GetDynamicConfig to get the current config and modify it as needed +func (w OnRampWrapper) SetDynamicConfig(opts *bind.TransactOpts, dynamicConfig evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig) (*types.Transaction, error) { + if w.Latest != nil { + return w.Latest.SetDynamicConfig(opts, dynamicConfig) + } + if w.V1_2_0 != nil { + return w.V1_2_0.SetDynamicConfig(opts, evm_2_evm_onramp_1_2_0.EVM2EVMOnRampDynamicConfig{ + Router: dynamicConfig.Router, + MaxNumberOfTokensPerMsg: dynamicConfig.MaxNumberOfTokensPerMsg, + DestGasOverhead: dynamicConfig.DestGasOverhead, + DestGasPerPayloadByte: dynamicConfig.DestGasPerPayloadByte, + DestDataAvailabilityOverheadGas: dynamicConfig.DestDataAvailabilityOverheadGas, + DestGasPerDataAvailabilityByte: dynamicConfig.DestGasPerDataAvailabilityByte, + DestDataAvailabilityMultiplierBps: dynamicConfig.DestDataAvailabilityMultiplierBps, + PriceRegistry: dynamicConfig.PriceRegistry, + MaxDataBytes: dynamicConfig.MaxDataBytes, + MaxPerMsgGasLimit: dynamicConfig.MaxPerMsgGasLimit, + }) } - return 0, fmt.Errorf("no instance found to get dynamic config") + return nil, fmt.Errorf("no instance found to set dynamic config") } func (w OnRampWrapper) ApplyPoolUpdates(opts *bind.TransactOpts, tokens []common.Address, pools []common.Address) (*types.Transaction, error) { diff --git a/integration-tests/ccip-tests/contracts/laneconfig/parse_contracts.go b/integration-tests/ccip-tests/contracts/laneconfig/parse_contracts.go index 332bd48ab31..c96cb3dfdc6 100644 --- a/integration-tests/ccip-tests/contracts/laneconfig/parse_contracts.go +++ b/integration-tests/ccip-tests/contracts/laneconfig/parse_contracts.go @@ -21,6 +21,7 @@ var ( type CommonContracts struct { IsNativeFeeToken bool `json:"is_native_fee_token,omitempty"` + AllowOutOfOrder bool `json:"allow_out_of_order,omitempty"` // expected to set this value as True for ZK chain source networks IsMockARM bool `json:"is_mock_arm,omitempty"` FeeToken string `json:"fee_token"` BridgeTokens []string `json:"bridge_tokens,omitempty"` diff --git a/integration-tests/ccip-tests/load/ccip_loadgen.go b/integration-tests/ccip-tests/load/ccip_loadgen.go index 9dc8ce16e5c..bc7627b4e08 100644 --- a/integration-tests/ccip-tests/load/ccip_loadgen.go +++ b/integration-tests/ccip-tests/load/ccip_loadgen.go @@ -22,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/contracts" "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testconfig" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" @@ -132,8 +133,9 @@ func (c *CCIPE2ELoad) BeforeAllCall() { c.EOAReceiver = c.msg.Receiver } if c.SendMaxDataIntermittentlyInMsgCount > 0 { - c.MaxDataBytes, err = sourceCCIP.OnRamp.Instance.GetDynamicConfig(nil) + cfg, err := sourceCCIP.OnRamp.Instance.GetDynamicConfig(nil) require.NoError(c.t, err, "failed to fetch dynamic config") + c.MaxDataBytes = cfg.MaxDataBytes } // if the msg is sent via multicall, transfer the token transfer amount to multicall contract if sourceCCIP.Common.MulticallEnabled && @@ -189,11 +191,27 @@ func (c *CCIPE2ELoad) CCIPMsg() (router.ClientEVM2AnyMessage, *testreporters.Req if !msgDetails.IsTokenTransfer() { msg.TokenAmounts = []router.ClientEVMTokenAmount{} } - extraArgsV1, err := testhelpers.GetEVMExtraArgsV1(big.NewInt(gasLimit), false) + + var ( + extraArgs []byte + err error + ) + matchErr := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{ + contracts.OnRampContract: contracts.V1_5_0, + }) + // TODO: The CCIP Offchain upgrade tests make the AllowOutOfOrder flag tricky in this case. + // It runs with out of date contract versions at first, then upgrades them. So transactions will assume that the new contracts are there + // before being deployed. So setting v2 args will break the test. This is a bit of a hack to get around that. + // The test will soon be deprecated, so a temporary solution is fine. + if matchErr != nil || !c.Lane.Source.Common.AllowOutOfOrder { + extraArgs, err = testhelpers.GetEVMExtraArgsV1(big.NewInt(gasLimit), false) + } else { + extraArgs, err = testhelpers.GetEVMExtraArgsV2(big.NewInt(gasLimit), c.Lane.Source.Common.AllowOutOfOrder) + } if err != nil { return router.ClientEVM2AnyMessage{}, stats, err } - msg.ExtraArgs = extraArgsV1 + msg.ExtraArgs = extraArgs // if gaslimit is 0, set the receiver to EOA if gasLimit == 0 { msg.Receiver = c.EOAReceiver diff --git a/integration-tests/ccip-tests/load/helper.go b/integration-tests/ccip-tests/load/helper.go index e4ee4573b2a..34860c8871c 100644 --- a/integration-tests/ccip-tests/load/helper.go +++ b/integration-tests/ccip-tests/load/helper.go @@ -199,10 +199,7 @@ func (l *LoadArgs) ValidateCurseFollowedByUncurse() { // try to send requests on lanes on which curse is applied on source RMN and the request should revert // data-only transfer is sufficient lane.Source.TransferAmount = []*big.Int{} - failedTx, _, _, err := lane.Source.SendRequest( - lane.Dest.ReceiverDapp.EthAddress, - big.NewInt(actions.DefaultDestinationGasLimit), // gas limit - ) + failedTx, _, _, err := lane.Source.SendRequest(lane.Dest.ReceiverDapp.EthAddress, big.NewInt(actions.DefaultDestinationGasLimit)) if lane.Source.Common.ChainClient.GetNetworkConfig().MinimumConfirmations > 0 { require.Error(l.t, err) } else { diff --git a/integration-tests/ccip-tests/smoke/ccip_test.go b/integration-tests/ccip-tests/smoke/ccip_test.go index 0ed7f60dd3f..08054459481 100644 --- a/integration-tests/ccip-tests/smoke/ccip_test.go +++ b/integration-tests/ccip-tests/smoke/ccip_test.go @@ -244,10 +244,7 @@ func TestSmokeCCIPRateLimit(t *testing.T) { tc.lane.Source.Common.ChainClient.GetDefaultWallet(), src.Common.Router.Address(), src.TransferAmount[0]), ) require.NoError(t, tc.lane.Source.Common.ChainClient.WaitForEvents()) - failedTx, _, _, err := tc.lane.Source.SendRequest( - tc.lane.Dest.ReceiverDapp.EthAddress, - big.NewInt(actions.DefaultDestinationGasLimit), // gas limit - ) + failedTx, _, _, err := tc.lane.Source.SendRequest(tc.lane.Dest.ReceiverDapp.EthAddress, big.NewInt(actions.DefaultDestinationGasLimit)) require.NoError(t, err) require.Error(t, tc.lane.Source.Common.ChainClient.WaitForEvents()) errReason, v, err := tc.lane.Source.Common.ChainClient.RevertReasonFromTx(failedTx, evm_2_evm_onramp.EVM2EVMOnRampABI) @@ -272,10 +269,7 @@ func TestSmokeCCIPRateLimit(t *testing.T) { // try to send again with amount more than the amount refilled by rate and // this should fail, as the refill rate is not enough to refill the capacity src.TransferAmount[0] = new(big.Int).Mul(AggregatedRateLimitRate, big.NewInt(10)) - failedTx, _, _, err = tc.lane.Source.SendRequest( - tc.lane.Dest.ReceiverDapp.EthAddress, - big.NewInt(actions.DefaultDestinationGasLimit), // gas limit - ) + failedTx, _, _, err = tc.lane.Source.SendRequest(tc.lane.Dest.ReceiverDapp.EthAddress, big.NewInt(actions.DefaultDestinationGasLimit)) tc.lane.Logger.Info().Str("tokensToSend", src.TransferAmount[0].String()).Msg("More than Aggregated Rate") require.NoError(t, err) require.Error(t, tc.lane.Source.Common.ChainClient.WaitForEvents()) @@ -339,10 +333,7 @@ func TestSmokeCCIPRateLimit(t *testing.T) { src.TransferAmount[0] = tokensToSend tc.lane.Logger.Info().Str("tokensToSend", tokensToSend.String()).Msg("More than Token Pool Capacity") - failedTx, _, _, err = tc.lane.Source.SendRequest( - tc.lane.Dest.ReceiverDapp.EthAddress, - big.NewInt(actions.DefaultDestinationGasLimit), // gas limit - ) + failedTx, _, _, err = tc.lane.Source.SendRequest(tc.lane.Dest.ReceiverDapp.EthAddress, big.NewInt(actions.DefaultDestinationGasLimit)) require.NoError(t, err) require.Error(t, tc.lane.Source.Common.ChainClient.WaitForEvents()) errReason, v, err = tc.lane.Source.Common.ChainClient.RevertReasonFromTx(failedTx, lock_release_token_pool.LockReleaseTokenPoolABI) @@ -374,10 +365,7 @@ func TestSmokeCCIPRateLimit(t *testing.T) { src.Common.ChainClient.GetDefaultWallet(), src.Common.Router.Address(), src.TransferAmount[0]), ) require.NoError(t, tc.lane.Source.Common.ChainClient.WaitForEvents()) - failedTx, _, _, err = tc.lane.Source.SendRequest( - tc.lane.Dest.ReceiverDapp.EthAddress, - big.NewInt(actions.DefaultDestinationGasLimit), - ) + failedTx, _, _, err = tc.lane.Source.SendRequest(tc.lane.Dest.ReceiverDapp.EthAddress, big.NewInt(actions.DefaultDestinationGasLimit)) require.NoError(t, err) require.Error(t, tc.lane.Source.Common.ChainClient.WaitForEvents()) errReason, v, err = tc.lane.Source.Common.ChainClient.RevertReasonFromTx(failedTx, lock_release_token_pool.LockReleaseTokenPoolABI) @@ -406,8 +394,8 @@ func TestSmokeCCIPOnRampLimits(t *testing.T) { "This test modifies contract state. Before running it, ensure you are willing and able to do so.", ) err := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{ - contracts.OffRampContract: contracts.V1_5_0_dev, - contracts.OnRampContract: contracts.V1_5_0_dev, + contracts.OffRampContract: contracts.V1_5_0, + contracts.OnRampContract: contracts.V1_5_0, }) require.NoError(t, err, "Required contract versions not met") @@ -627,8 +615,8 @@ func TestSmokeCCIPTokenPoolRateLimits(t *testing.T) { "This test modifies contract state. Before running it, ensure you are willing and able to do so.", ) err := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{ - contracts.OffRampContract: contracts.V1_5_0_dev, - contracts.OnRampContract: contracts.V1_5_0_dev, + contracts.OffRampContract: contracts.V1_5_0, + contracts.OnRampContract: contracts.V1_5_0, }) require.NoError(t, err, "Required contract versions not met") @@ -1006,7 +994,7 @@ func testOffRampRateLimits(t *testing.T, rateLimiterConfig contracts.RateLimiter "This test modifies contract state. Before running it, ensure you are willing and able to do so.", ) err := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{ - contracts.OffRampContract: contracts.V1_5_0_dev, + contracts.OffRampContract: contracts.V1_5_0, }) require.NoError(t, err, "Required contract versions not met") require.False(t, pointer.GetBool(TestCfg.TestGroupInput.ExistingDeployment), "This test modifies contract state and cannot be run on existing deployments") diff --git a/integration-tests/ccip-tests/testconfig/README.md b/integration-tests/ccip-tests/testconfig/README.md index d464ae247ff..ff57ecaa220 100644 --- a/integration-tests/ccip-tests/testconfig/README.md +++ b/integration-tests/ccip-tests/testconfig/README.md @@ -480,6 +480,10 @@ Specifies whether to set up bi-directional lanes between networks. Specifies whether commit and execution jobs are to be run on the same Chainlink node. +### CCIP.Groups.[testgroup].AllowOutOfOrder + +Specifies whether out of order execution is allowed globally for all the chains. + ### CCIP.Groups.[testgroup].NoOfCommitNodes Specifies the number of nodes on which commit jobs are to be run. This needs to be lesser than the total number of nodes mentioned in [CCIP.Env.NewCLCluster.NoOfNodes](#ccipenvnewclclusternoofnodes) or [CCIP.Env.ExistingCLCluster.NoOfNodes](#ccipenvexistingclclusternoofnodes). diff --git a/integration-tests/ccip-tests/testconfig/ccip.go b/integration-tests/ccip-tests/testconfig/ccip.go index 19ebd304ceb..76e00ecf496 100644 --- a/integration-tests/ccip-tests/testconfig/ccip.go +++ b/integration-tests/ccip-tests/testconfig/ccip.go @@ -266,6 +266,7 @@ type CCIPTestGroupConfig struct { KeepEnvAlive *bool `toml:",omitempty"` BiDirectionalLane *bool `toml:",omitempty"` CommitAndExecuteOnSameDON *bool `toml:",omitempty"` + AllowOutOfOrder *bool `toml:",omitempty"` // To set out of order execution globally NoOfCommitNodes int `toml:",omitempty"` MsgDetails *MsgDetails `toml:",omitempty"` TokenConfig *TokenConfig `toml:",omitempty"` diff --git a/integration-tests/ccip-tests/testconfig/tomls/ccip-default.toml b/integration-tests/ccip-tests/testconfig/tomls/ccip-default.toml index 0157ac24fb4..b03f03a6dab 100644 --- a/integration-tests/ccip-tests/testconfig/tomls/ccip-default.toml +++ b/integration-tests/ccip-tests/testconfig/tomls/ccip-default.toml @@ -27,7 +27,7 @@ ethereum_version = "eth1" # geth, besu, erigon or nethermind execution_layer = "geth" # eth2-only, if set to true environment startup will wait until at least 1 epoch has been finalised -wait_for_finalization=false +wait_for_finalization = false [CCIP.Env.PrivateEthereumNetworks.SIMULATED_1.EthereumChainConfig] # eth2-only, the lower the value the faster the block production (3 is minimum) @@ -239,6 +239,8 @@ EIP1559FeeCapBufferBlocks = 0 KeepEnvAlive = false # if true, the test will not tear down the test environment after the test is finished CommitAndExecuteOnSameDON = true # if true, and the test is building the env from scratch, same chainlink nodes will be used for Commit and Execution jobs. +AllowOutOfOrder = false # if true, all the lanes will allow out of order execution and it +# overrides settings from lane_config. To allow out of order execution per lane, then send "allow_out_of_order":true, similar to is_native_fee_token variable. # Otherwise Commit and execution jobs will be set up in different nodes based on the number of nodes specified in NoOfCommitNodes and CCIP.Env.NewCLCluster.NoOfNodes BiDirectionalLane = true # True uses both the lanes. If bidirectional is false only one way lane is set up. NoOfCommitNodes = 5 # no of chainlink nodes with Commit job @@ -271,7 +273,7 @@ TimeoutForPriceUpdate = '15m' # Duration to wait for the price update to time-ou # Now testing only with dynamic price getter (no pipeline). # Could be removed once the pipeline is completely removed. WithPipeline = false -NoOfTokensPerChain = 2 # number of bridge tokens to be deployed per network; if MsgType = 'Token'/'DataWithToken' +NoOfTokensPerChain = 2 # number of bridge tokens to be deployed per network; if MsgType = 'Token'/'DataWithToken' CCIPOwnerTokens = false # if true, the test will use deploy the tokens by the CCIPOwner, otherwise the tokens will be deployed by a non-owner account, only applicable for 1.5 pools and onwards #NoOfTokensWithDynamicPrice = 15 # number of tokens with dynamic price to be deployed @@ -293,6 +295,7 @@ CCIPOwnerTokens = false # if true, the test will use deploy the tokens by the CC KeepEnvAlive = false # same as above CommitAndExecuteOnSameDON = true # same as above +AllowOutOfOrder = false # same as above BiDirectionalLane = true # same as above NoOfCommitNodes = 5 # same as above PhaseTimeout = '10m' # same as above @@ -351,7 +354,7 @@ TimeoutForPriceUpdate = '15m' # Duration to wait for the price update to time-ou # Now testing only with dynamic price getter (no pipeline). # Could be removed once the pipeline is completely removed. WithPipeline = false -NoOfTokensPerChain = 2 # number of bridge tokens to be deployed per network; if MsgType = 'Token'/'DataWithToken' +NoOfTokensPerChain = 2 # number of bridge tokens to be deployed per network; if MsgType = 'Token'/'DataWithToken' CCIPOwnerTokens = false # if true, the test will use deploy the tokens by the CCIPOwner, otherwise the tokens and pools will be deployed by a non-owner account, # only applicable for 1.5 pools and onwards, if you are running with pre-1.5 pools, then set this to true to deploy token pools by CCIPOwner, otherwise # the test will fail @@ -403,6 +406,7 @@ CCIPOwnerTokens = false # if true, the test will use deploy the tokens by the CC #NetworkPairs = ['SEPOLIA,OPTIMISM_GOERLI','SEPOLIA,POLYGON_MUMBAI','AVALANCHE_FUJI,SEPOLIA','SEPOLIA,BASE_GOERLI','SEPOLIA,BSC_TESTNET','SEPOLIA,WEMIX_TESTNET','AVALANCHE_FUJI,OPTIMISM_GOERLI','AVALANCHE_FUJI,POLYGON_MUMBAI','AVALANCHE_FUJI,BSC_TESTNET','AVALANCHE_FUJI,BASE_GOERLI','OPTIMISM_GOERLI,BASE_GOERLI','OPTIMISM_GOERLI,POLYGON_MUMBAI','BSC_TESTNET,POLYGON_MUMBAI','BSC_TESTNET,BASE_GOERLI','WEMIX_TESTNET,KROMA_SEPOLIA'] KeepEnvAlive = false CommitAndExecuteOnSameDON = false +AllowOutOfOrder = false BiDirectionalLane = true NoOfCommitNodes = 5 PhaseTimeout = '50m'