Skip to content

Commit

Permalink
Enables OOO Execution for E2E Tests (#14392)
Browse files Browse the repository at this point in the history
* Enables OOO Execution for E2E Tests

* Out of order false by default

* Port over CCIP fix

* Changeset

* Fix changeset

* Fix lint
  • Loading branch information
kalverra authored Sep 18, 2024
1 parent a7348f0 commit 3f83f9e
Show file tree
Hide file tree
Showing 15 changed files with 153 additions and 60 deletions.
5 changes: 5 additions & 0 deletions .changeset/rare-crabs-tell.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink": patch
---

#added Adds the ability to use out of order execution transactions in CCIP E2E tests
2 changes: 2 additions & 0 deletions core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,15 @@ func NewExecOffchainConfig(
RelativeBoostPerWaitHour float64,
InflightCacheExpiry config.Duration,
RootSnoozeTime config.Duration,
BatchingStrategyID uint32,
) ExecOffchainConfig {
return ExecOffchainConfig{v1_2_0.JSONExecOffchainConfig{
DestOptimisticConfirmations: DestOptimisticConfirmations,
BatchGasLimit: BatchGasLimit,
RelativeBoostPerWaitHour: RelativeBoostPerWaitHour,
InflightCacheExpiry: InflightCacheExpiry,
RootSnoozeTime: RootSnoozeTime,
BatchingStrategyID: BatchingStrategyID,
}}
}

Expand Down
1 change: 1 addition & 0 deletions core/services/ocr2/plugins/ccip/testhelpers/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func (c *CCIPContracts) createExecOffchainConfig(t *testing.T, inflightCacheExpi
0.07,
*config.MustNewDuration(inflightCacheExpiry),
*config.MustNewDuration(rootSnoozeTime),
uint32(0),
).Encode()
require.NoError(t, err)
return config
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,13 +157,15 @@ func NewExecOffchainConfig(
RelativeBoostPerWaitHour float64,
InflightCacheExpiry config.Duration,
RootSnoozeTime config.Duration,
BatchingStrategyID uint32,
) ExecOffchainConfig {
return ExecOffchainConfig{v1_2_0.JSONExecOffchainConfig{
DestOptimisticConfirmations: DestOptimisticConfirmations,
BatchGasLimit: BatchGasLimit,
RelativeBoostPerWaitHour: RelativeBoostPerWaitHour,
InflightCacheExpiry: InflightCacheExpiry,
RootSnoozeTime: RootSnoozeTime,
BatchingStrategyID: BatchingStrategyID,
}}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func (c *CCIPContracts) createExecOffchainConfig(t *testing.T, inflightCacheExpi
0.07,
*config.MustNewDuration(inflightCacheExpiry),
*config.MustNewDuration(rootSnoozeTime),
uint32(0),
).Encode()
require.NoError(t, err)
return config
Expand Down
66 changes: 47 additions & 19 deletions integration-tests/ccip-tests/actions/ccip_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ type CCIPCommon struct {
gasUpdateWatcherMu *sync.Mutex
gasUpdateWatcher map[uint64]*big.Int // key - destchain id; value - timestamp of update
GasUpdateEvents []contracts.GasUpdateEvent
AllowOutOfOrder bool
}

// FreeUpUnusedSpace sets nil to various elements of ccipModule which are only used
Expand Down Expand Up @@ -273,6 +274,9 @@ func (ccipModule *CCIPCommon) CurseARM() (*types.Transaction, error) {

func (ccipModule *CCIPCommon) LoadContractAddresses(conf *laneconfig.LaneConfig, noOfTokens *int) {
if conf != nil {
if conf.AllowOutOfOrder {
ccipModule.AllowOutOfOrder = true
}
if common.IsHexAddress(conf.FeeToken) {
ccipModule.FeeToken = &contracts.LinkToken{
EthAddress: common.HexToAddress(conf.FeeToken),
Expand Down Expand Up @@ -1626,7 +1630,7 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe
if sendRequestedEvents, exists := value.([]*evm_2_evm_onramp.EVM2EVMOnRampCCIPSendRequested); exists {
for _, sendRequestedEvent := range sendRequestedEvents {
raw := sendRequestedEvent.Raw
hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(raw.BlockNumber)))
hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(raw.BlockNumber))
if err == nil {
if hdr.Timestamp.After(lastSeenTimestamp) {
foundAt = pointer.ToTime(hdr.Timestamp)
Expand Down Expand Up @@ -1709,6 +1713,7 @@ func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested(
// CCIPMsg constructs the message for a CCIP request
func (sourceCCIP *SourceCCIPModule) CCIPMsg(
receiver common.Address,
allowOutOfOrder bool,
gasLimit *big.Int,
) (router.ClientEVM2AnyMessage, error) {
length := sourceCCIP.MsgDataLength
Expand Down Expand Up @@ -1750,17 +1755,25 @@ func (sourceCCIP *SourceCCIPModule) CCIPMsg(
return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed encoding the receiver address: %w", err)
}

extraArgsV1, err := testhelpers.GetEVMExtraArgsV1(gasLimit, false)
var extraArgs []byte
matchErr := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{
contracts.OnRampContract: contracts.V1_5_0,
})
if matchErr != nil {
extraArgs, err = testhelpers.GetEVMExtraArgsV1(gasLimit, false)
} else {
extraArgs, err = testhelpers.GetEVMExtraArgsV2(gasLimit, allowOutOfOrder)
}
if err != nil {
return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed encoding the options field: %w", err)
return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed getting extra args: %w", err)
}
// form the message for transfer
return router.ClientEVM2AnyMessage{
Receiver: receiverAddr,
Data: []byte(data),
TokenAmounts: tokenAndAmounts,
FeeToken: common.HexToAddress(sourceCCIP.Common.FeeToken.Address()),
ExtraArgs: extraArgsV1,
ExtraArgs: extraArgs,
}, nil
}

Expand All @@ -1775,7 +1788,7 @@ func (sourceCCIP *SourceCCIPModule) SendRequest(
return common.Hash{}, d, nil, fmt.Errorf("failed getting the chain selector: %w", err)
}
// form the message for transfer
msg, err := sourceCCIP.CCIPMsg(receiver, gasLimit)
msg, err := sourceCCIP.CCIPMsg(receiver, sourceCCIP.Common.AllowOutOfOrder, gasLimit)
if err != nil {
return common.Hash{}, d, nil, fmt.Errorf("failed forming the ccip msg: %w", err)
}
Expand Down Expand Up @@ -2218,7 +2231,7 @@ func (destCCIP *DestCCIPModule) AssertNoReportAcceptedEventReceived(lggr *zerolo
e, exists := value.(*evm_2_evm_offramp.EVM2EVMOffRampExecutionStateChanged)
if exists {
vLogs := e.Raw
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, big.NewInt(int64(vLogs.BlockNumber)))
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, new(big.Int).SetUint64(vLogs.BlockNumber))
if err != nil {
return true
}
Expand Down Expand Up @@ -2259,7 +2272,7 @@ func (destCCIP *DestCCIPModule) AssertNoExecutionStateChangedEventReceived(
e, exists := value.(*contracts.EVM2EVMOffRampExecutionStateChanged)
if exists {
vLogs := e.LogInfo
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, big.NewInt(int64(vLogs.BlockNumber)))
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, new(big.Int).SetUint64(vLogs.BlockNumber))
if err != nil {
return true
}
Expand Down Expand Up @@ -2288,7 +2301,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged(
reqStat *testreporters.RequestStat,
execState testhelpers.MessageExecutionState,
) (uint8, error) {
lggr.Info().Int64("seqNum", int64(seqNum)).Str("Timeout", timeout.String()).Msg("Waiting for ExecutionStateChanged event")
lggr.Info().Uint64("seqNum", seqNum).Str("Timeout", timeout.String()).Msg("Waiting for ExecutionStateChanged event")
timer := time.NewTimer(timeout)
defer timer.Stop()
ticker := time.NewTicker(time.Second)
Expand All @@ -2306,7 +2319,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged(
destCCIP.ExecStateChangedWatcher.Delete(seqNum)
vLogs := e.LogInfo
receivedAt := time.Now().UTC()
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(vLogs.BlockNumber)))
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(vLogs.BlockNumber))
if err == nil {
receivedAt = hdr.Timestamp
}
Expand All @@ -2319,7 +2332,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged(
gasUsed = receipt.GasUsed
}
if testhelpers.MessageExecutionState(e.State) == execState {
lggr.Info().Int64("seqNum", int64(seqNum)).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received")
lggr.Info().Uint64("seqNum", seqNum).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received")
reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, receivedAt.Sub(timeNow),
testreporters.Success,
&testreporters.TransactionStats{
Expand Down Expand Up @@ -2363,7 +2376,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted(
prevEventAt time.Time,
reqStat *testreporters.RequestStat,
) (*contracts.CommitStoreReportAccepted, time.Time, error) {
lggr.Info().Int64("seqNum", int64(seqNum)).Str("Timeout", timeout.String()).Msg("Waiting for ReportAccepted event")
lggr.Info().Uint64("seqNum", seqNum).Str("Timeout", timeout.String()).Msg("Waiting for ReportAccepted event")
timer := time.NewTimer(timeout)
defer timer.Stop()
resetTimerCount := 0
Expand All @@ -2379,7 +2392,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted(
// if the value is processed, delete it from the map
destCCIP.ReportAcceptedWatcher.Delete(seqNum)
receivedAt := time.Now().UTC()
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(reportAccepted.LogInfo.BlockNumber)))
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(reportAccepted.LogInfo.BlockNumber))
if err == nil {
receivedAt = hdr.Timestamp
}
Expand Down Expand Up @@ -2488,7 +2501,7 @@ func (destCCIP *DestCCIPModule) AssertReportBlessed(
// if the value is processed, delete it from the map
destCCIP.ReportBlessedBySeqNum.Delete(seqNum)
}
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(vLogs.BlockNumber)))
hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(vLogs.BlockNumber))
if err == nil {
receivedAt = hdr.Timestamp
}
Expand Down Expand Up @@ -2536,7 +2549,7 @@ func (destCCIP *DestCCIPModule) AssertSeqNumberExecuted(
timeNow time.Time,
reqStat *testreporters.RequestStat,
) error {
lggr.Info().Int64("seqNum", int64(seqNumberBefore)).Str("Timeout", timeout.String()).Msg("Waiting to be processed by commit store")
lggr.Info().Uint64("seqNum", seqNumberBefore).Str("Timeout", timeout.String()).Msg("Waiting to be processed by commit store")
timer := time.NewTimer(timeout)
defer timer.Stop()
resetTimerCount := 0
Expand Down Expand Up @@ -2793,7 +2806,11 @@ func (lane *CCIPLane) AddToSentReqs(txHash common.Hash, reqStats []*testreporter
func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) error {
var ccipMultipleMsg []contracts.CCIPMsgData
feeToken := common.HexToAddress(lane.Source.Common.FeeToken.Address())
genericMsg, err := lane.Source.CCIPMsg(lane.Dest.ReceiverDapp.EthAddress, big.NewInt(DefaultDestinationGasLimit))
genericMsg, err := lane.Source.CCIPMsg(
lane.Dest.ReceiverDapp.EthAddress,
lane.Source.Common.AllowOutOfOrder,
big.NewInt(DefaultDestinationGasLimit),
)
if err != nil {
return fmt.Errorf("failed to form the ccip message: %w", err)
}
Expand Down Expand Up @@ -2885,10 +2902,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address)
func (lane *CCIPLane) SendRequests(noOfRequests int, gasLimit *big.Int) error {
for i := 1; i <= noOfRequests; i++ {
stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq+i), lane.SourceNetworkName, lane.DestNetworkName)
txHash, txConfirmationDur, fee, err := lane.Source.SendRequest(
lane.Dest.ReceiverDapp.EthAddress,
gasLimit,
)
txHash, txConfirmationDur, fee, err := lane.Source.SendRequest(lane.Dest.ReceiverDapp.EthAddress, gasLimit)
if err != nil {
stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure, nil)
return fmt.Errorf("could not send request: %w", err)
Expand Down Expand Up @@ -3515,6 +3529,7 @@ func (lane *CCIPLane) DeployNewCCIPLane(
srcConf = lane.SrcNetworkLaneCfg
destConf = lane.DstNetworkLaneCfg
commitAndExecOnSameDON = pointer.GetBool(testConf.CommitAndExecuteOnSameDON)
allowOutOfOrder = pointer.GetBool(testConf.AllowOutOfOrder)
withPipeline = pointer.GetBool(testConf.TokenConfig.WithPipeline)
configureCLNodes = !pointer.GetBool(testConf.ExistingDeployment)
)
Expand All @@ -3529,6 +3544,13 @@ func (lane *CCIPLane) DeployNewCCIPLane(
if err != nil {
return fmt.Errorf("failed to create source module: %w", err)
}

// If AllowOutOfOrder is set globally in test config, the assumption is to set it for every lane.
// However, if this is set as false, and set as true for specific chain in lane_configs, then apply it
// only for a lane where source network is of that chain.
if allowOutOfOrder {
lane.Source.Common.AllowOutOfOrder = true
}
lane.Dest, err = DefaultDestinationCCIPModule(
lane.Logger, testConf,
destChainClient, sourceChainClient.GetChainID().Uint64(),
Expand Down Expand Up @@ -3781,12 +3803,18 @@ func SetOCR2Config(
nodes = execNodes
}
if destCCIP.OffRamp != nil {
// Use out of order batching strategy if we expect to be sending out of order messages
batchingStrategyID := uint32(0)
if pointer.GetBool(testConf.AllowOutOfOrder) {
batchingStrategyID = uint32(1)
}
execOffchainCfg, err := contracts.NewExecOffchainConfig(
1,
BatchGasLimit,
0.7,
*inflightExpiryExec,
*commonconfig.MustNewDuration(RootSnoozeTime),
batchingStrategyID,
)
if err != nil {
return fmt.Errorf("failed to create exec offchain config: %w", err)
Expand Down
7 changes: 6 additions & 1 deletion integration-tests/ccip-tests/contracts/contract_deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func MatchContractVersionsOrAbove(requiredContractVersions map[Name]Version) err
// if the version is less than 1.5.0, then token admin registry is not needed
func NeedTokenAdminRegistry() bool {
return MatchContractVersionsOrAbove(map[Name]Version{
TokenPoolContract: V1_5_0_dev,
TokenPoolContract: V1_5_0,
}) == nil
}

Expand Down Expand Up @@ -1044,6 +1044,7 @@ func (e *CCIPContractsDeployer) DeployOnRamp(
MaxPerMsgGasLimit: 4_000_000,
DefaultTokenFeeUSDCents: 50,
DefaultTokenDestGasOverhead: 125_000,
EnforceOutOfOrder: false,
},
evm_2_evm_onramp.RateLimiterConfig{
Capacity: opts.Capacity,
Expand Down Expand Up @@ -1465,12 +1466,14 @@ func NewExecOnchainConfig(
}
}

// NewExecOffchainConfig creates a config for the OffChain portion of how CCIP operates
func NewExecOffchainConfig(
destOptimisticConfirmations uint32,
batchGasLimit uint32,
relativeBoostPerWaitHour float64,
inflightCacheExpiry config.Duration,
rootSnoozeTime config.Duration,
batchingStrategyID uint32, // See ccipexec package
) (ccipconfig.OffchainConfig, error) {
switch VersionMap[OffRampContract] {
case Latest:
Expand All @@ -1480,6 +1483,7 @@ func NewExecOffchainConfig(
relativeBoostPerWaitHour,
inflightCacheExpiry,
rootSnoozeTime,
batchingStrategyID,
), nil
case V1_2_0:
return testhelpers_1_4_0.NewExecOffchainConfig(
Expand All @@ -1488,6 +1492,7 @@ func NewExecOffchainConfig(
relativeBoostPerWaitHour,
inflightCacheExpiry,
rootSnoozeTime,
batchingStrategyID,
), nil
default:
return nil, fmt.Errorf("version not supported: %s", VersionMap[OffRampContract])
Expand Down
54 changes: 45 additions & 9 deletions integration-tests/ccip-tests/contracts/contract_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@ const (
var (
V1_2_0 = MustVersion("1.2.0")
V1_4_0 = MustVersion("1.4.0")
V1_5_0_dev = MustVersion("1.5.0")
LatestPoolVersion = V1_5_0_dev
Latest = V1_5_0_dev
V1_5_0 = MustVersion("1.5.0")
LatestPoolVersion = V1_5_0
Latest = V1_5_0
VersionMap = map[Name]Version{
PriceRegistryContract: V1_2_0,
OffRampContract: Latest,
Expand Down Expand Up @@ -1616,22 +1616,58 @@ func (w OnRampWrapper) ParseCCIPSendRequested(l types.Log) (uint64, error) {
return 0, fmt.Errorf("no instance found to parse CCIPSendRequested")
}

func (w OnRampWrapper) GetDynamicConfig(opts *bind.CallOpts) (uint32, error) {
// GetDynamicConfig retrieves the dynamic config for the onramp
func (w OnRampWrapper) GetDynamicConfig(opts *bind.CallOpts) (evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig, error) {
if w.Latest != nil {
cfg, err := w.Latest.GetDynamicConfig(opts)
if err != nil {
return 0, err
return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{}, err
}
return cfg.MaxDataBytes, nil
return cfg, nil
}
if w.V1_2_0 != nil {
cfg, err := w.V1_2_0.GetDynamicConfig(opts)
if err != nil {
return 0, err
return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{}, err
}
return cfg.MaxDataBytes, nil
return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{
Router: cfg.Router,
MaxNumberOfTokensPerMsg: cfg.MaxNumberOfTokensPerMsg,
DestGasOverhead: cfg.DestGasOverhead,
DestGasPerPayloadByte: cfg.DestGasPerPayloadByte,
DestDataAvailabilityOverheadGas: cfg.DestDataAvailabilityOverheadGas,
DestGasPerDataAvailabilityByte: cfg.DestGasPerDataAvailabilityByte,
DestDataAvailabilityMultiplierBps: cfg.DestDataAvailabilityMultiplierBps,
PriceRegistry: cfg.PriceRegistry,
MaxDataBytes: cfg.MaxDataBytes,
MaxPerMsgGasLimit: cfg.MaxPerMsgGasLimit,
}, nil
}
return evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig{}, fmt.Errorf("no instance found to get dynamic config")
}

// SetDynamicConfig sets the dynamic config for the onramp
// Note that you cannot set only a single field, you must set all fields or they will be reset to zero values
// You can use GetDynamicConfig to get the current config and modify it as needed
func (w OnRampWrapper) SetDynamicConfig(opts *bind.TransactOpts, dynamicConfig evm_2_evm_onramp.EVM2EVMOnRampDynamicConfig) (*types.Transaction, error) {
if w.Latest != nil {
return w.Latest.SetDynamicConfig(opts, dynamicConfig)
}
if w.V1_2_0 != nil {
return w.V1_2_0.SetDynamicConfig(opts, evm_2_evm_onramp_1_2_0.EVM2EVMOnRampDynamicConfig{
Router: dynamicConfig.Router,
MaxNumberOfTokensPerMsg: dynamicConfig.MaxNumberOfTokensPerMsg,
DestGasOverhead: dynamicConfig.DestGasOverhead,
DestGasPerPayloadByte: dynamicConfig.DestGasPerPayloadByte,
DestDataAvailabilityOverheadGas: dynamicConfig.DestDataAvailabilityOverheadGas,
DestGasPerDataAvailabilityByte: dynamicConfig.DestGasPerDataAvailabilityByte,
DestDataAvailabilityMultiplierBps: dynamicConfig.DestDataAvailabilityMultiplierBps,
PriceRegistry: dynamicConfig.PriceRegistry,
MaxDataBytes: dynamicConfig.MaxDataBytes,
MaxPerMsgGasLimit: dynamicConfig.MaxPerMsgGasLimit,
})
}
return 0, fmt.Errorf("no instance found to get dynamic config")
return nil, fmt.Errorf("no instance found to set dynamic config")
}

func (w OnRampWrapper) ApplyPoolUpdates(opts *bind.TransactOpts, tokens []common.Address, pools []common.Address) (*types.Transaction, error) {
Expand Down
Loading

0 comments on commit 3f83f9e

Please sign in to comment.