From 436b68410b4e93030e99eaf73c7af9ad1c3b702a Mon Sep 17 00:00:00 2001 From: Dimitris Grigoriou Date: Fri, 17 Jan 2025 20:42:40 +0200 Subject: [PATCH] Txmv2 (#15467) * TXMv2 alpha version * TXM fixes * Fix * Update orchestrator * Add builder * Add txmv2 tests * Update retry logic * Add backoff mechanism * Add multi address support * Add backoff mechanism for broadcasting and backfilling * Minor fix * Add check to dummy keystore * Fix inmemory store logging per address * Remove unnecessary const * Fix txm to work with enabled addresses from keystore * AttemptBuilder fixes * Add AttemptBuilder close * Minor updates * Make purgable attempts empty * Fix Idempotency in Store Manager * Update trigger * Fix lint * Fix more lint * Fix lint * Fix lint * More lint fixes * Fix lint * Fix lint final * Fix races * Update logs * Fix lint * Start documentation * Update DummyKeystore Add method * Txmv2 stuck tx detection (#15436) * Stuck tx detector alpha * Update stuck tx detection * Add stuck_tx_detection and dual broadcast client * Add support for TXMv2 * Fix orchestrator's monitoring call * Fix AttemptBuilder * Enable DualBroadcast client * Switch DualBroadcast params to pointers * Add context to client * Fix lint * Fix DualBroadcast client * More lint fixes * Fix lint * Make nonce nullable * Update configs * Add prom metrics * Add transaction confirmation metric * Improve logs * Address feedback * Update tests * Fix orchestrator log * Fix Nonce log * Improvements * Update tests * Add fixes * Improvements * Improve logs * Move initialization of nonce * Add Beholder metrics * Improve InMemoryStorage * Support different priceMax per key * Upgrades * Fix config tests * Fix docs * Fix config test lint * Update configs * Reuse transaction states * Fix docs * Deprecate DualBroadcastDetection * Add health report * Fix configs * Bump mockery * Update testfiles * Update docs * Address feedback * Add backfill tests * Update docs --- .mockery.yaml | 6 + .../evm/config/chain_scoped_transactions.go | 25 + core/chains/evm/config/config.go | 8 + core/chains/evm/config/toml/config.go | 61 +- .../evm/config/toml/defaults/fallback.toml | 3 + core/chains/evm/keystore/eth.go | 1 + core/chains/evm/keystore/mocks/eth.go | 60 ++ core/chains/evm/txm/attempt_builder.go | 161 +++++ core/chains/evm/txm/attempt_builder_test.go | 97 +++ .../evm/txm/clientwrappers/chain_client.go | 31 + .../clientwrappers/dual_broadcast_client.go | 128 ++++ .../evm/txm/clientwrappers/geth_client.go | 51 ++ .../evm/txm/docs/TRANSACTION_MANAGER_V2.md | 14 + core/chains/evm/txm/dummy_keystore.go | 64 ++ core/chains/evm/txm/metrics.go | 93 +++ core/chains/evm/txm/mocks/attempt_builder.go | 161 +++++ core/chains/evm/txm/mocks/client.go | 204 ++++++ core/chains/evm/txm/mocks/keystore.go | 98 +++ core/chains/evm/txm/mocks/tx_store.go | 647 ++++++++++++++++++ core/chains/evm/txm/orchestrator.go | 363 ++++++++++ core/chains/evm/txm/storage/inmemory_store.go | 358 ++++++++++ .../evm/txm/storage/inmemory_store_manager.go | 136 ++++ .../storage/inmemory_store_manager_test.go | 36 + .../evm/txm/storage/inmemory_store_test.go | 559 +++++++++++++++ core/chains/evm/txm/stuck_tx_detector.go | 123 ++++ core/chains/evm/txm/stuck_tx_detector_test.go | 80 +++ core/chains/evm/txm/txm.go | 447 ++++++++++++ core/chains/evm/txm/txm_test.go | 327 +++++++++ core/chains/evm/txm/types/transaction.go | 192 ++++++ core/chains/evm/txmgr/builder.go | 53 ++ core/chains/legacyevm/evm_txm.go | 43 +- core/config/docs/chains-evm.toml | 10 + core/config/docs/docs_test.go | 5 + core/services/chainlink/config_test.go | 15 + .../chainlink/testdata/config-full.toml | 3 + .../chainlink/testdata/config-invalid.toml | 6 + .../config-multi-chain-effective.toml | 9 + core/web/resolver/testdata/config-full.toml | 3 + .../config-multi-chain-effective.toml | 9 + docs/CONFIG.md | 283 ++++++++ .../node/validate/defaults-override.txtar | 3 + .../disk-based-logging-disabled.txtar | 3 + .../validate/disk-based-logging-no-dir.txtar | 3 + .../node/validate/disk-based-logging.txtar | 3 + .../node/validate/fallback-override.txtar | 6 + testdata/scripts/node/validate/invalid.txtar | 3 + testdata/scripts/node/validate/valid.txtar | 3 + 47 files changed, 4982 insertions(+), 15 deletions(-) create mode 100644 core/chains/evm/txm/attempt_builder.go create mode 100644 core/chains/evm/txm/attempt_builder_test.go create mode 100644 core/chains/evm/txm/clientwrappers/chain_client.go create mode 100644 core/chains/evm/txm/clientwrappers/dual_broadcast_client.go create mode 100644 core/chains/evm/txm/clientwrappers/geth_client.go create mode 100644 core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md create mode 100644 core/chains/evm/txm/dummy_keystore.go create mode 100644 core/chains/evm/txm/metrics.go create mode 100644 core/chains/evm/txm/mocks/attempt_builder.go create mode 100644 core/chains/evm/txm/mocks/client.go create mode 100644 core/chains/evm/txm/mocks/keystore.go create mode 100644 core/chains/evm/txm/mocks/tx_store.go create mode 100644 core/chains/evm/txm/orchestrator.go create mode 100644 core/chains/evm/txm/storage/inmemory_store.go create mode 100644 core/chains/evm/txm/storage/inmemory_store_manager.go create mode 100644 core/chains/evm/txm/storage/inmemory_store_manager_test.go create mode 100644 core/chains/evm/txm/storage/inmemory_store_test.go create mode 100644 core/chains/evm/txm/stuck_tx_detector.go create mode 100644 core/chains/evm/txm/stuck_tx_detector_test.go create mode 100644 core/chains/evm/txm/txm.go create mode 100644 core/chains/evm/txm/txm_test.go create mode 100644 core/chains/evm/txm/types/transaction.go diff --git a/.mockery.yaml b/.mockery.yaml index b7dbb8a1e85..88dcf307d20 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -81,6 +81,12 @@ packages: BalanceMonitor: config: dir: "{{ .InterfaceDir }}/../mocks" + github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm: + interfaces: + Client: + TxStore: + AttemptBuilder: + Keystore: github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr: interfaces: ChainConfig: diff --git a/core/chains/evm/config/chain_scoped_transactions.go b/core/chains/evm/config/chain_scoped_transactions.go index 8cddce20e65..3da171c98f0 100644 --- a/core/chains/evm/config/chain_scoped_transactions.go +++ b/core/chains/evm/config/chain_scoped_transactions.go @@ -39,6 +39,31 @@ func (t *transactionsConfig) MaxQueued() uint64 { return uint64(*t.c.MaxQueued) } +func (t *transactionsConfig) TransactionManagerV2() TransactionManagerV2 { + return &transactionManagerV2Config{c: t.c.TransactionManagerV2} +} + +type transactionManagerV2Config struct { + c toml.TransactionManagerV2Config +} + +func (t *transactionManagerV2Config) Enabled() bool { + return *t.c.Enabled +} + +func (t *transactionManagerV2Config) BlockTime() *time.Duration { + d := t.c.BlockTime.Duration() + return &d +} + +func (t *transactionManagerV2Config) CustomURL() *url.URL { + return t.c.CustomURL.URL() +} + +func (t *transactionManagerV2Config) DualBroadcast() *bool { + return t.c.DualBroadcast +} + func (t *transactionsConfig) AutoPurge() AutoPurgeConfig { return &autoPurgeConfig{c: t.c.AutoPurge} } diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index fbaf1ff6dda..c76cb6953e5 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -111,6 +111,7 @@ type Transactions interface { MaxInFlight() uint32 MaxQueued() uint64 AutoPurge() AutoPurgeConfig + TransactionManagerV2() TransactionManagerV2 } type AutoPurgeConfig interface { @@ -120,6 +121,13 @@ type AutoPurgeConfig interface { DetectionApiUrl() *url.URL } +type TransactionManagerV2 interface { + Enabled() bool + BlockTime() *time.Duration + CustomURL() *url.URL + DualBroadcast() *bool +} + type GasEstimator interface { BlockHistory() BlockHistory FeeHistory() FeeHistory diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 807e1141791..7a3dddcb1de 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -6,6 +6,7 @@ import ( "net/url" "slices" "strconv" + "time" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/pelletier/go-toml/v2" @@ -471,6 +472,27 @@ func (c *Chain) ValidateConfig() (err error) { return } +func (c *Transactions) ValidateConfig() (err error) { + if c.TransactionManagerV2.Enabled != nil && *c.TransactionManagerV2.Enabled && + c.TransactionManagerV2.DualBroadcast != nil && *c.TransactionManagerV2.DualBroadcast { + if c.TransactionManagerV2.CustomURL == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "TransactionManagerV2.CustomURL", Msg: "must be set if DualBroadcast is enabled"}) + } + if c.AutoPurge.Enabled != nil && !*c.AutoPurge.Enabled { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "AutoPurge.Enabled", Value: false, Msg: "cannot be false if DualBroadcast is enabled"}) + } + if c.AutoPurge.DetectionApiUrl == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "AutoPurge.DetectionApiUrl", Msg: "must be set if DualBroadcast is enabled"}) + } + if c.AutoPurge.Threshold == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "AutoPurge.Threshold", Msg: "needs to be set if auto-purge feature is enabled"}) + } else if *c.AutoPurge.Threshold == 0 { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "AutoPurge.Threshold", Value: 0, Msg: "cannot be 0 if auto-purge feature is enabled"}) + } + } + return +} + type Transactions struct { Enabled *bool ForwardersEnabled *bool @@ -480,7 +502,8 @@ type Transactions struct { ReaperThreshold *commonconfig.Duration ResendAfterThreshold *commonconfig.Duration - AutoPurge AutoPurgeConfig `toml:",omitempty"` + AutoPurge AutoPurgeConfig `toml:",omitempty"` + TransactionManagerV2 TransactionManagerV2Config `toml:",omitempty"` } func (t *Transactions) setFrom(f *Transactions) { @@ -506,6 +529,7 @@ func (t *Transactions) setFrom(f *Transactions) { t.ResendAfterThreshold = v } t.AutoPurge.setFrom(&f.AutoPurge) + t.TransactionManagerV2.setFrom(&f.TransactionManagerV2) } type AutoPurgeConfig struct { @@ -530,6 +554,41 @@ func (a *AutoPurgeConfig) setFrom(f *AutoPurgeConfig) { } } +type TransactionManagerV2Config struct { + Enabled *bool `toml:",omitempty"` + BlockTime *commonconfig.Duration `toml:",omitempty"` + CustomURL *commonconfig.URL `toml:",omitempty"` + DualBroadcast *bool `toml:",omitempty"` +} + +func (t *TransactionManagerV2Config) setFrom(f *TransactionManagerV2Config) { + if v := f.Enabled; v != nil { + t.Enabled = f.Enabled + } + if v := f.BlockTime; v != nil { + t.BlockTime = f.BlockTime + } + if v := f.CustomURL; v != nil { + t.CustomURL = f.CustomURL + } + if v := f.DualBroadcast; v != nil { + t.DualBroadcast = f.DualBroadcast + } +} + +func (t *TransactionManagerV2Config) ValidateConfig() (err error) { + if t.Enabled != nil && *t.Enabled { + if t.BlockTime == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "BlockTime", Msg: "must be set if TransactionManagerV2 feature is enabled"}) + return + } + if t.BlockTime.Duration() < 2*time.Second { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "BlockTime", Msg: "must be equal to or greater than 2 seconds"}) + } + } + return +} + type OCR2 struct { Automation Automation `toml:",omitempty"` } diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index d2a6f0e4a2d..4b6caa20787 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -30,6 +30,9 @@ ResendAfterThreshold = '1m' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true diff --git a/core/chains/evm/keystore/eth.go b/core/chains/evm/keystore/eth.go index ff71e0a4f18..9c0986d9c3d 100644 --- a/core/chains/evm/keystore/eth.go +++ b/core/chains/evm/keystore/eth.go @@ -13,5 +13,6 @@ type Eth interface { CheckEnabled(ctx context.Context, address common.Address, chainID *big.Int) error EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) SubscribeToKeyChanges(ctx context.Context) (ch chan struct{}, unsub func()) } diff --git a/core/chains/evm/keystore/mocks/eth.go b/core/chains/evm/keystore/mocks/eth.go index aefe6ff7548..96125e66936 100644 --- a/core/chains/evm/keystore/mocks/eth.go +++ b/core/chains/evm/keystore/mocks/eth.go @@ -133,6 +133,66 @@ func (_c *Eth_EnabledAddressesForChain_Call) RunAndReturn(run func(context.Conte return _c } +// SignMessage provides a mock function with given fields: ctx, address, message +func (_m *Eth) SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) { + ret := _m.Called(ctx, address, message) + + if len(ret) == 0 { + panic("no return value specified for SignMessage") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) ([]byte, error)); ok { + return rf(ctx, address, message) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) []byte); ok { + r0 = rf(ctx, address, message) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, []byte) error); ok { + r1 = rf(ctx, address, message) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Eth_SignMessage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SignMessage' +type Eth_SignMessage_Call struct { + *mock.Call +} + +// SignMessage is a helper method to define mock.On call +// - ctx context.Context +// - address common.Address +// - message []byte +func (_e *Eth_Expecter) SignMessage(ctx interface{}, address interface{}, message interface{}) *Eth_SignMessage_Call { + return &Eth_SignMessage_Call{Call: _e.mock.On("SignMessage", ctx, address, message)} +} + +func (_c *Eth_SignMessage_Call) Run(run func(ctx context.Context, address common.Address, message []byte)) *Eth_SignMessage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].([]byte)) + }) + return _c +} + +func (_c *Eth_SignMessage_Call) Return(_a0 []byte, _a1 error) *Eth_SignMessage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Eth_SignMessage_Call) RunAndReturn(run func(context.Context, common.Address, []byte) ([]byte, error)) *Eth_SignMessage_Call { + _c.Call.Return(run) + return _c +} + // SignTx provides a mock function with given fields: ctx, fromAddress, tx, chainID func (_m *Eth) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { ret := _m.Called(ctx, fromAddress, tx, chainID) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go new file mode 100644 index 00000000000..16ed0f1a86a --- /dev/null +++ b/core/chains/evm/txm/attempt_builder.go @@ -0,0 +1,161 @@ +package txm + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + evmtypes "github.com/ethereum/go-ethereum/core/types" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type AttemptBuilderKeystore interface { + SignTx(ctx context.Context, fromAddress common.Address, tx *evmtypes.Transaction, chainID *big.Int) (*evmtypes.Transaction, error) +} + +type attemptBuilder struct { + gas.EvmFeeEstimator + chainID *big.Int + priceMaxKey func(common.Address) *assets.Wei + keystore AttemptBuilderKeystore +} + +func NewAttemptBuilder(chainID *big.Int, priceMaxKey func(common.Address) *assets.Wei, estimator gas.EvmFeeEstimator, keystore AttemptBuilderKeystore) *attemptBuilder { + return &attemptBuilder{ + chainID: chainID, + priceMaxKey: priceMaxKey, + EvmFeeEstimator: estimator, + keystore: keystore, + } +} + +func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { + fee, estimatedGasLimit, err := a.EvmFeeEstimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMaxKey(tx.FromAddress), &tx.FromAddress, &tx.ToAddress) + if err != nil { + return nil, err + } + txType := evmtypes.LegacyTxType + if dynamic { + txType = evmtypes.DynamicFeeTxType + } + return a.newCustomAttempt(ctx, tx, fee, estimatedGasLimit, byte(txType), lggr) +} + +func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { + bumpedFee, bumpedFeeLimit, err := a.EvmFeeEstimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMaxKey(tx.FromAddress), nil) + if err != nil { + return nil, err + } + return a.newCustomAttempt(ctx, tx, bumpedFee, bumpedFeeLimit, previousAttempt.Type, lggr) +} + +func (a *attemptBuilder) newCustomAttempt( + ctx context.Context, + tx *types.Transaction, + fee gas.EvmFee, + estimatedGasLimit uint64, + txType byte, + lggr logger.Logger, +) (attempt *types.Attempt, err error) { + switch txType { + case 0x0: + if fee.GasPrice == nil { + err = fmt.Errorf("tried to create attempt of type %v for txID: %v but estimator did not return legacy fee", txType, tx.ID) + logger.Sugared(lggr).AssumptionViolation(err.Error()) + return + } + return a.newLegacyAttempt(ctx, tx, fee.GasPrice, estimatedGasLimit) + case 0x2: + if !fee.ValidDynamic() { + err = fmt.Errorf("tried to create attempt of type %v for txID: %v but estimator did not return dynamic fee", txType, tx.ID) + logger.Sugared(lggr).AssumptionViolation(err.Error()) + return + } + return a.newDynamicFeeAttempt(ctx, tx, fee.DynamicFee, estimatedGasLimit) + default: + return nil, fmt.Errorf("cannot build attempt, unrecognized transaction type: %v", txType) + } +} + +func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transaction, gasPrice *assets.Wei, estimatedGasLimit uint64) (*types.Attempt, error) { + var data []byte + var toAddress common.Address + value := big.NewInt(0) + if !tx.IsPurgeable { + data = tx.Data + toAddress = tx.ToAddress + value = tx.Value + } + if tx.Nonce == nil { + return nil, fmt.Errorf("failed to create attempt for txID: %v: nonce empty", tx.ID) + } + legacyTx := evmtypes.LegacyTx{ + Nonce: *tx.Nonce, + To: &toAddress, + Value: value, + Gas: estimatedGasLimit, + GasPrice: gasPrice.ToInt(), + Data: data, + } + + signedTx, err := a.keystore.SignTx(ctx, tx.FromAddress, evmtypes.NewTx(&legacyTx), a.chainID) + if err != nil { + return nil, fmt.Errorf("failed to sign attempt for txID: %v, err: %w", tx.ID, err) + } + + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{GasPrice: gasPrice}, + Hash: signedTx.Hash(), + GasLimit: estimatedGasLimit, + Type: evmtypes.LegacyTxType, + SignedTransaction: signedTx, + } + + return attempt, nil +} + +func (a *attemptBuilder) newDynamicFeeAttempt(ctx context.Context, tx *types.Transaction, dynamicFee gas.DynamicFee, estimatedGasLimit uint64) (*types.Attempt, error) { + var data []byte + var toAddress common.Address + value := big.NewInt(0) + if !tx.IsPurgeable { + data = tx.Data + toAddress = tx.ToAddress + value = tx.Value + } + if tx.Nonce == nil { + return nil, fmt.Errorf("failed to create attempt for txID: %v: nonce empty", tx.ID) + } + dynamicTx := evmtypes.DynamicFeeTx{ + Nonce: *tx.Nonce, + To: &toAddress, + Value: value, + Gas: estimatedGasLimit, + GasFeeCap: dynamicFee.GasFeeCap.ToInt(), + GasTipCap: dynamicFee.GasTipCap.ToInt(), + Data: data, + } + + signedTx, err := a.keystore.SignTx(ctx, tx.FromAddress, evmtypes.NewTx(&dynamicTx), a.chainID) + if err != nil { + return nil, fmt.Errorf("failed to sign attempt for txID: %v, err: %w", tx.ID, err) + } + + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{DynamicFee: gas.DynamicFee{GasFeeCap: dynamicFee.GasFeeCap, GasTipCap: dynamicFee.GasTipCap}}, + Hash: signedTx.Hash(), + GasLimit: estimatedGasLimit, + Type: evmtypes.DynamicFeeTxType, + SignedTransaction: signedTx, + } + + return attempt, nil +} diff --git a/core/chains/evm/txm/attempt_builder_test.go b/core/chains/evm/txm/attempt_builder_test.go new file mode 100644 index 00000000000..65330cd39d7 --- /dev/null +++ b/core/chains/evm/txm/attempt_builder_test.go @@ -0,0 +1,97 @@ +package txm + +import ( + "math/big" + "testing" + + evmtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore/mocks" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +func TestAttemptBuilder_newLegacyAttempt(t *testing.T) { + ks := mocks.NewEth(t) + ab := NewAttemptBuilder(testutils.FixtureChainID, nil, nil, ks) + address := testutils.NewAddress() + toAddress := testutils.NewAddress() + lggr := logger.Test(t) + var gasLimit uint64 = 100 + + t.Run("fails if GasPrice is nil", func(t *testing.T) { + tx := &types.Transaction{ID: 10, FromAddress: address} + _, err := ab.newCustomAttempt(tests.Context(t), tx, gas.EvmFee{DynamicFee: gas.DynamicFee{GasTipCap: assets.NewWeiI(1), GasFeeCap: assets.NewWeiI(2)}}, gasLimit, evmtypes.LegacyTxType, lggr) + require.Error(t, err) + assert.Contains(t, err.Error(), "estimator did not return legacy fee") + }) + + t.Run("fails if tx doesn't have a nonce", func(t *testing.T) { + tx := &types.Transaction{ID: 10, FromAddress: address} + _, err := ab.newCustomAttempt(tests.Context(t), tx, gas.EvmFee{GasPrice: assets.NewWeiI(25)}, gasLimit, evmtypes.LegacyTxType, lggr) + require.Error(t, err) + assert.Contains(t, err.Error(), "nonce empty") + }) + + t.Run("creates attempt with fields", func(t *testing.T) { + var nonce uint64 = 77 + tx := &types.Transaction{ID: 10, FromAddress: address, Nonce: &nonce} + legacyTx := evmtypes.NewTx(&evmtypes.LegacyTx{Nonce: nonce, To: &toAddress, Gas: gasLimit, GasPrice: big.NewInt(25)}) + ks.On("SignTx", mock.Anything, mock.Anything, mock.Anything, testutils.FixtureChainID).Return(legacyTx, nil).Once() + a, err := ab.newCustomAttempt(tests.Context(t), tx, gas.EvmFee{GasPrice: assets.NewWeiI(25)}, gasLimit, evmtypes.LegacyTxType, lggr) + require.NoError(t, err) + assert.Equal(t, tx.ID, a.TxID) + assert.Equal(t, evmtypes.LegacyTxType, int(a.Type)) + assert.NotNil(t, a.Fee.GasPrice) + assert.Equal(t, "25 wei", a.Fee.GasPrice.String()) + assert.Nil(t, a.Fee.GasTipCap) + assert.Nil(t, a.Fee.GasFeeCap) + assert.Equal(t, gasLimit, a.GasLimit) + }) +} + +func TestAttemptBuilder_newDynamicFeeAttempt(t *testing.T) { + ks := mocks.NewEth(t) + ab := NewAttemptBuilder(testutils.FixtureChainID, nil, nil, ks) + address := testutils.NewAddress() + toAddress := testutils.NewAddress() + lggr := logger.Test(t) + var gasLimit uint64 = 100 + + t.Run("fails if DynamicFee is invalid", func(t *testing.T) { + tx := &types.Transaction{ID: 10, FromAddress: address} + _, err := ab.newCustomAttempt(tests.Context(t), tx, gas.EvmFee{GasPrice: assets.NewWeiI(1)}, gasLimit, evmtypes.DynamicFeeTxType, lggr) + require.Error(t, err) + assert.Contains(t, err.Error(), "estimator did not return dynamic fee") + }) + + t.Run("fails if tx doesn't have a nonce", func(t *testing.T) { + tx := &types.Transaction{ID: 10, FromAddress: address} + _, err := ab.newCustomAttempt(tests.Context(t), tx, gas.EvmFee{DynamicFee: gas.DynamicFee{GasTipCap: assets.NewWeiI(1), GasFeeCap: assets.NewWeiI(2)}}, gasLimit, evmtypes.DynamicFeeTxType, lggr) + require.Error(t, err) + assert.Contains(t, err.Error(), "nonce empty") + }) + + t.Run("creates attempt with fields", func(t *testing.T) { + var nonce uint64 = 77 + tx := &types.Transaction{ID: 10, FromAddress: address, Nonce: &nonce} + legacyTx := evmtypes.NewTx(&evmtypes.LegacyTx{Nonce: nonce, To: &toAddress, Gas: gasLimit, GasPrice: big.NewInt(25)}) + ks.On("SignTx", mock.Anything, mock.Anything, mock.Anything, testutils.FixtureChainID).Return(legacyTx, nil).Once() + a, err := ab.newCustomAttempt(tests.Context(t), tx, gas.EvmFee{DynamicFee: gas.DynamicFee{GasTipCap: assets.NewWeiI(1), GasFeeCap: assets.NewWeiI(2)}}, gasLimit, evmtypes.DynamicFeeTxType, lggr) + require.NoError(t, err) + assert.Equal(t, tx.ID, a.TxID) + assert.Equal(t, evmtypes.DynamicFeeTxType, int(a.Type)) + assert.Equal(t, "1 wei", a.Fee.DynamicFee.GasTipCap.String()) + assert.Equal(t, "2 wei", a.Fee.DynamicFee.GasFeeCap.String()) + assert.Nil(t, a.Fee.GasPrice) + assert.Equal(t, gasLimit, a.GasLimit) + }) +} diff --git a/core/chains/evm/txm/clientwrappers/chain_client.go b/core/chains/evm/txm/clientwrappers/chain_client.go new file mode 100644 index 00000000000..7638cc53443 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/chain_client.go @@ -0,0 +1,31 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type ChainClient struct { + c client.Client +} + +func NewChainClient(client client.Client) *ChainClient { + return &ChainClient{c: client} +} + +func (c *ChainClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return c.c.NonceAt(ctx, address, blockNumber) +} + +func (c *ChainClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + return c.c.PendingNonceAt(ctx, address) +} + +func (c *ChainClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return c.c.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go new file mode 100644 index 00000000000..481c26cbc2b --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -0,0 +1,128 @@ +package clientwrappers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "net/url" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type DualBroadcastClientKeystore interface { + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) +} + +type DualBroadcastClient struct { + c client.Client + keystore DualBroadcastClientKeystore + customURL *url.URL +} + +func NewDualBroadcastClient(c client.Client, keystore DualBroadcastClientKeystore, customURL *url.URL) *DualBroadcastClient { + return &DualBroadcastClient{ + c: c, + keystore: keystore, + customURL: customURL, + } +} + +func (d *DualBroadcastClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return d.c.NonceAt(ctx, address, blockNumber) +} + +func (d *DualBroadcastClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":["%s","pending"], "id":1}`, address.String())) + response, err := d.signAndPostMessage(ctx, address, body, "") + if err != nil { + return 0, err + } + + nonce, err := hexutil.DecodeUint64(response) + if err != nil { + return 0, fmt.Errorf("failed to decode response %v into uint64: %w", response, err) + } + return nonce, nil +} + +func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + meta, err := tx.GetMeta() + if err != nil { + return err + } + + if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { + data, err := attempt.SignedTransaction.MarshalBinary() + if err != nil { + return err + } + params := "" + if meta.DualBroadcastParams != nil { + params = *meta.DualBroadcastParams + } + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"], "id":1}`, hexutil.Encode(data))) + _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, params) + return err + } + + return d.c.SendTransaction(ctx, attempt.SignedTransaction) +} + +func (d *DualBroadcastClient) signAndPostMessage(ctx context.Context, address common.Address, body []byte, urlParams string) (result string, err error) { + bodyReader := bytes.NewReader(body) + postReq, err := http.NewRequestWithContext(ctx, http.MethodPost, d.customURL.String()+"?"+urlParams, bodyReader) + if err != nil { + return + } + + hashedBody := crypto.Keccak256Hash(body).Hex() + signedMessage, err := d.keystore.SignMessage(ctx, address, []byte(hashedBody)) + if err != nil { + return + } + + postReq.Header.Add("X-Flashbots-signature", address.String()+":"+hexutil.Encode(signedMessage)) + postReq.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(postReq) + if err != nil { + return result, fmt.Errorf("request %v failed: %w", postReq, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("request %v failed with status: %d", postReq, resp.StatusCode) + } + + keyJSON, err := io.ReadAll(resp.Body) + if err != nil { + return + } + var response postResponse + err = json.Unmarshal(keyJSON, &response) + if err != nil { + return result, fmt.Errorf("failed to unmarshal response into struct: %w: %s", err, string(keyJSON)) + } + if response.Error.Message != "" { + return result, errors.New(response.Error.Message) + } + return response.Result, nil +} + +type postResponse struct { + Result string `json:"result,omitempty"` + Error postError +} + +type postError struct { + Message string `json:"message,omitempty"` +} diff --git a/core/chains/evm/txm/clientwrappers/geth_client.go b/core/chains/evm/txm/clientwrappers/geth_client.go new file mode 100644 index 00000000000..d97e5cfae35 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/geth_client.go @@ -0,0 +1,51 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" +) + +type GethClient struct { + *ethclient.Client +} + +func NewGethClient(client *ethclient.Client) *GethClient { + return &GethClient{ + Client: client, + } +} + +func (g *GethClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { + return g.Client.Client().BatchCallContext(ctx, b) +} + +func (g *GethClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + return g.Client.Client().CallContext(ctx, result, method, args...) +} + +func (g *GethClient) CallContract(ctx context.Context, message ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + var hex hexutil.Bytes + err := g.CallContext(ctx, &hex, "eth_call", client.ToBackwardCompatibleCallArg(message), client.ToBackwardCompatibleBlockNumArg(blockNumber)) + return hex, err +} + +func (g *GethClient) HeadByNumber(ctx context.Context, number *big.Int) (*evmtypes.Head, error) { + hexNumber := client.ToBlockNumArg(number) + args := []interface{}{hexNumber, false} + head := new(evmtypes.Head) + err := g.CallContext(ctx, head, "eth_getBlockByNumber", args...) + return head, err +} + +func (g *GethClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return g.Client.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md new file mode 100644 index 00000000000..d408cc7d733 --- /dev/null +++ b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md @@ -0,0 +1,14 @@ + +# Transaction Manager V2 + +## Configs +- `EIP1559`: enables EIP-1559 mode. This means the transaction manager will create and broadcast Dynamic attempts. Set this to false to broadcast Legacy transactions. +- `BlockTime`: controls the interval of the backfill loop. This dictates how frequently the transaction manager will check for confirmed transactions, rebroadcast stuck ones, and fill any nonce gaps. Transactions are getting confirmed only during new blocks so it's best if you set this to a value close to the block time. At least one RPC call is made during each BlockTime interval so the recommended minimum is 2s. A small jitter is applied so the timeout won't be exactly the same each time. +- `RetryBlockThreshold`: is the number of blocks to wait for a transaction stuck in the mempool before automatically rebroadcasting it with a new attempt. +- `EmptyTxLimitDefault`: sets default gas limit for empty transactions. Empty transactions are created in case there is a nonce gap or another stuck transaction in the mempool to fill a given nonce. These are empty transactions and they don't have any data or value. + +## Metrics +- `txm_num_broadcasted_transactions`: total number of successful broadcasted transactions. +- `txm_num_confirmed_transactions`: total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs. +- `txm_num_nonce_gaps`: total number of nonce gaps created that the transaction manager had to fill. +- `txm_time_until_tx_confirmed`: The amount of time elapsed from a transaction being broadcast to being included in a block. \ No newline at end of file diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go new file mode 100644 index 00000000000..01816dfcbbd --- /dev/null +++ b/core/chains/evm/txm/dummy_keystore.go @@ -0,0 +1,64 @@ +package txm + +import ( + "context" + "crypto/ecdsa" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +type DummyKeystore struct { + privateKeyMap map[common.Address]*ecdsa.PrivateKey +} + +func NewKeystore() *DummyKeystore { + return &DummyKeystore{privateKeyMap: make(map[common.Address]*ecdsa.PrivateKey)} +} + +func (k *DummyKeystore) Add(privateKeyString string) error { + privateKey, err := crypto.HexToECDSA(privateKeyString) + if err != nil { + return err + } + + publicKey := privateKey.Public() + publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) + if !ok { + return fmt.Errorf("error casting public key: %v to ECDSA", publicKey) + } + + address := crypto.PubkeyToAddress(*publicKeyECDSA) + k.privateKeyMap[address] = privateKey + return nil +} + +func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + if key, exists := k.privateKeyMap[fromAddress]; exists { + return types.SignTx(tx, types.LatestSignerForChainID(chainID), key) + } + return nil, fmt.Errorf("private key for address: %v not found", fromAddress) +} + +func (k *DummyKeystore) SignMessage(ctx context.Context, address common.Address, data []byte) ([]byte, error) { + key, exists := k.privateKeyMap[address] + if !exists { + return nil, fmt.Errorf("private key for address: %v not found", address) + } + signature, err := crypto.Sign(accounts.TextHash(data), key) + if err != nil { + return nil, fmt.Errorf("failed to sign message for address: %v", address) + } + return signature, nil +} + +func (k *DummyKeystore) EnabledAddressesForChain(_ context.Context, _ *big.Int) (addresses []common.Address, err error) { + for address := range k.privateKeyMap { + addresses = append(addresses, address) + } + return +} diff --git a/core/chains/evm/txm/metrics.go b/core/chains/evm/txm/metrics.go new file mode 100644 index 00000000000..5ccc711ef09 --- /dev/null +++ b/core/chains/evm/txm/metrics.go @@ -0,0 +1,93 @@ +package txm + +import ( + "context" + "fmt" + "math/big" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/metrics" +) + +var ( + promNumBroadcastedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_broadcasted_transactions", + Help: "Total number of successful broadcasted transactions.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs or when filling the nonce for untracked transactions.", + }, []string{"chainID"}) + promNumNonceGaps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_nonce_gaps", + Help: "Total number of nonce gaps created that the transaction manager had to fill.", + }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "txm_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + }, []string{"chainID"}) +) + +type txmMetrics struct { + metrics.Labeler + chainID *big.Int + numBroadcastedTxs metric.Int64Counter + numConfirmedTxs metric.Int64Counter + numNonceGaps metric.Int64Counter + timeUntilTxConfirmed metric.Float64Histogram +} + +func NewTxmMetrics(chainID *big.Int) (*txmMetrics, error) { + numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("txm_num_broadcasted_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register broadcasted txs number: %w", err) + } + + numConfirmedTxs, err := beholder.GetMeter().Int64Counter("txm_num_confirmed_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register confirmed txs number: %w", err) + } + + numNonceGaps, err := beholder.GetMeter().Int64Counter("txm_num_nonce_gaps") + if err != nil { + return nil, fmt.Errorf("failed to register nonce gaps number: %w", err) + } + + timeUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("txm_time_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register time until tx confirmed: %w", err) + } + + return &txmMetrics{ + chainID: chainID, + Labeler: metrics.NewLabeler().With("chainID", chainID.String()), + numBroadcastedTxs: numBroadcastedTxs, + numConfirmedTxs: numConfirmedTxs, + numNonceGaps: numNonceGaps, + timeUntilTxConfirmed: timeUntilTxConfirmed, + }, nil +} + +func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { + promNumBroadcastedTxs.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.numBroadcastedTxs.Add(ctx, 1) +} + +func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { + promNumConfirmedTxs.WithLabelValues(m.chainID.String()).Add(float64(confirmedTransactions)) + m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions)) +} + +func (m *txmMetrics) IncrementNumNonceGaps(ctx context.Context) { + promNumNonceGaps.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.numNonceGaps.Add(ctx, 1) +} + +func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { + promTimeUntilTxConfirmed.WithLabelValues(m.chainID.String()).Observe(duration) + m.timeUntilTxConfirmed.Record(ctx, duration) +} diff --git a/core/chains/evm/txm/mocks/attempt_builder.go b/core/chains/evm/txm/mocks/attempt_builder.go new file mode 100644 index 00000000000..91961e5d420 --- /dev/null +++ b/core/chains/evm/txm/mocks/attempt_builder.go @@ -0,0 +1,161 @@ +// Code generated by mockery v2.50.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + logger "github.com/smartcontractkit/chainlink-common/pkg/logger" + mock "github.com/stretchr/testify/mock" + + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +// AttemptBuilder is an autogenerated mock type for the AttemptBuilder type +type AttemptBuilder struct { + mock.Mock +} + +type AttemptBuilder_Expecter struct { + mock *mock.Mock +} + +func (_m *AttemptBuilder) EXPECT() *AttemptBuilder_Expecter { + return &AttemptBuilder_Expecter{mock: &_m.Mock} +} + +// NewAttempt provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *AttemptBuilder) NewAttempt(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 bool) (*types.Attempt, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for NewAttempt") + } + + var r0 *types.Attempt + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, bool) (*types.Attempt, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, bool) *types.Attempt); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Attempt) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, logger.Logger, *types.Transaction, bool) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AttemptBuilder_NewAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewAttempt' +type AttemptBuilder_NewAttempt_Call struct { + *mock.Call +} + +// NewAttempt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 logger.Logger +// - _a2 *types.Transaction +// - _a3 bool +func (_e *AttemptBuilder_Expecter) NewAttempt(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *AttemptBuilder_NewAttempt_Call { + return &AttemptBuilder_NewAttempt_Call{Call: _e.mock.On("NewAttempt", _a0, _a1, _a2, _a3)} +} + +func (_c *AttemptBuilder_NewAttempt_Call) Run(run func(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 bool)) *AttemptBuilder_NewAttempt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(logger.Logger), args[2].(*types.Transaction), args[3].(bool)) + }) + return _c +} + +func (_c *AttemptBuilder_NewAttempt_Call) Return(_a0 *types.Attempt, _a1 error) *AttemptBuilder_NewAttempt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AttemptBuilder_NewAttempt_Call) RunAndReturn(run func(context.Context, logger.Logger, *types.Transaction, bool) (*types.Attempt, error)) *AttemptBuilder_NewAttempt_Call { + _c.Call.Return(run) + return _c +} + +// NewBumpAttempt provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *AttemptBuilder) NewBumpAttempt(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 types.Attempt) (*types.Attempt, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for NewBumpAttempt") + } + + var r0 *types.Attempt + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, types.Attempt) (*types.Attempt, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, types.Attempt) *types.Attempt); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Attempt) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, logger.Logger, *types.Transaction, types.Attempt) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AttemptBuilder_NewBumpAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewBumpAttempt' +type AttemptBuilder_NewBumpAttempt_Call struct { + *mock.Call +} + +// NewBumpAttempt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 logger.Logger +// - _a2 *types.Transaction +// - _a3 types.Attempt +func (_e *AttemptBuilder_Expecter) NewBumpAttempt(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *AttemptBuilder_NewBumpAttempt_Call { + return &AttemptBuilder_NewBumpAttempt_Call{Call: _e.mock.On("NewBumpAttempt", _a0, _a1, _a2, _a3)} +} + +func (_c *AttemptBuilder_NewBumpAttempt_Call) Run(run func(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 types.Attempt)) *AttemptBuilder_NewBumpAttempt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(logger.Logger), args[2].(*types.Transaction), args[3].(types.Attempt)) + }) + return _c +} + +func (_c *AttemptBuilder_NewBumpAttempt_Call) Return(_a0 *types.Attempt, _a1 error) *AttemptBuilder_NewBumpAttempt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AttemptBuilder_NewBumpAttempt_Call) RunAndReturn(run func(context.Context, logger.Logger, *types.Transaction, types.Attempt) (*types.Attempt, error)) *AttemptBuilder_NewBumpAttempt_Call { + _c.Call.Return(run) + return _c +} + +// NewAttemptBuilder creates a new instance of AttemptBuilder. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAttemptBuilder(t interface { + mock.TestingT + Cleanup(func()) +}) *AttemptBuilder { + mock := &AttemptBuilder{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go new file mode 100644 index 00000000000..cac2e55491a --- /dev/null +++ b/core/chains/evm/txm/mocks/client.go @@ -0,0 +1,204 @@ +// Code generated by mockery v2.50.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + big "math/big" + + common "github.com/ethereum/go-ethereum/common" + + mock "github.com/stretchr/testify/mock" + + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +// Client is an autogenerated mock type for the Client type +type Client struct { + mock.Mock +} + +type Client_Expecter struct { + mock *mock.Mock +} + +func (_m *Client) EXPECT() *Client_Expecter { + return &Client_Expecter{mock: &_m.Mock} +} + +// NonceAt provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Client) NonceAt(_a0 context.Context, _a1 common.Address, _a2 *big.Int) (uint64, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for NonceAt") + } + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) (uint64, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) uint64); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Client_NonceAt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NonceAt' +type Client_NonceAt_Call struct { + *mock.Call +} + +// NonceAt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +// - _a2 *big.Int +func (_e *Client_Expecter) NonceAt(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Client_NonceAt_Call { + return &Client_NonceAt_Call{Call: _e.mock.On("NonceAt", _a0, _a1, _a2)} +} + +func (_c *Client_NonceAt_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 *big.Int)) *Client_NonceAt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].(*big.Int)) + }) + return _c +} + +func (_c *Client_NonceAt_Call) Return(_a0 uint64, _a1 error) *Client_NonceAt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Client_NonceAt_Call) RunAndReturn(run func(context.Context, common.Address, *big.Int) (uint64, error)) *Client_NonceAt_Call { + _c.Call.Return(run) + return _c +} + +// PendingNonceAt provides a mock function with given fields: _a0, _a1 +func (_m *Client) PendingNonceAt(_a0 context.Context, _a1 common.Address) (uint64, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for PendingNonceAt") + } + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address) (uint64, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address) uint64); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Client_PendingNonceAt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PendingNonceAt' +type Client_PendingNonceAt_Call struct { + *mock.Call +} + +// PendingNonceAt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +func (_e *Client_Expecter) PendingNonceAt(_a0 interface{}, _a1 interface{}) *Client_PendingNonceAt_Call { + return &Client_PendingNonceAt_Call{Call: _e.mock.On("PendingNonceAt", _a0, _a1)} +} + +func (_c *Client_PendingNonceAt_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *Client_PendingNonceAt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address)) + }) + return _c +} + +func (_c *Client_PendingNonceAt_Call) Return(_a0 uint64, _a1 error) *Client_PendingNonceAt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Client_PendingNonceAt_Call) RunAndReturn(run func(context.Context, common.Address) (uint64, error)) *Client_PendingNonceAt_Call { + _c.Call.Return(run) + return _c +} + +// SendTransaction provides a mock function with given fields: ctx, tx, attempt +func (_m *Client) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + ret := _m.Called(ctx, tx, attempt) + + if len(ret) == 0 { + panic("no return value specified for SendTransaction") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction, *types.Attempt) error); ok { + r0 = rf(ctx, tx, attempt) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Client_SendTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SendTransaction' +type Client_SendTransaction_Call struct { + *mock.Call +} + +// SendTransaction is a helper method to define mock.On call +// - ctx context.Context +// - tx *types.Transaction +// - attempt *types.Attempt +func (_e *Client_Expecter) SendTransaction(ctx interface{}, tx interface{}, attempt interface{}) *Client_SendTransaction_Call { + return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", ctx, tx, attempt)} +} + +func (_c *Client_SendTransaction_Call) Run(run func(ctx context.Context, tx *types.Transaction, attempt *types.Attempt)) *Client_SendTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.Transaction), args[2].(*types.Attempt)) + }) + return _c +} + +func (_c *Client_SendTransaction_Call) Return(_a0 error) *Client_SendTransaction_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction, *types.Attempt) error) *Client_SendTransaction_Call { + _c.Call.Return(run) + return _c +} + +// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewClient(t interface { + mock.TestingT + Cleanup(func()) +}) *Client { + mock := &Client{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/mocks/keystore.go b/core/chains/evm/txm/mocks/keystore.go new file mode 100644 index 00000000000..3d11a6fa549 --- /dev/null +++ b/core/chains/evm/txm/mocks/keystore.go @@ -0,0 +1,98 @@ +// Code generated by mockery v2.50.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + big "math/big" + + common "github.com/ethereum/go-ethereum/common" + + mock "github.com/stretchr/testify/mock" +) + +// Keystore is an autogenerated mock type for the Keystore type +type Keystore struct { + mock.Mock +} + +type Keystore_Expecter struct { + mock *mock.Mock +} + +func (_m *Keystore) EXPECT() *Keystore_Expecter { + return &Keystore_Expecter{mock: &_m.Mock} +} + +// EnabledAddressesForChain provides a mock function with given fields: ctx, chainID +func (_m *Keystore) EnabledAddressesForChain(ctx context.Context, chainID *big.Int) ([]common.Address, error) { + ret := _m.Called(ctx, chainID) + + if len(ret) == 0 { + panic("no return value specified for EnabledAddressesForChain") + } + + var r0 []common.Address + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *big.Int) ([]common.Address, error)); ok { + return rf(ctx, chainID) + } + if rf, ok := ret.Get(0).(func(context.Context, *big.Int) []common.Address); ok { + r0 = rf(ctx, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]common.Address) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *big.Int) error); ok { + r1 = rf(ctx, chainID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Keystore_EnabledAddressesForChain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnabledAddressesForChain' +type Keystore_EnabledAddressesForChain_Call struct { + *mock.Call +} + +// EnabledAddressesForChain is a helper method to define mock.On call +// - ctx context.Context +// - chainID *big.Int +func (_e *Keystore_Expecter) EnabledAddressesForChain(ctx interface{}, chainID interface{}) *Keystore_EnabledAddressesForChain_Call { + return &Keystore_EnabledAddressesForChain_Call{Call: _e.mock.On("EnabledAddressesForChain", ctx, chainID)} +} + +func (_c *Keystore_EnabledAddressesForChain_Call) Run(run func(ctx context.Context, chainID *big.Int)) *Keystore_EnabledAddressesForChain_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*big.Int)) + }) + return _c +} + +func (_c *Keystore_EnabledAddressesForChain_Call) Return(addresses []common.Address, err error) *Keystore_EnabledAddressesForChain_Call { + _c.Call.Return(addresses, err) + return _c +} + +func (_c *Keystore_EnabledAddressesForChain_Call) RunAndReturn(run func(context.Context, *big.Int) ([]common.Address, error)) *Keystore_EnabledAddressesForChain_Call { + _c.Call.Return(run) + return _c +} + +// NewKeystore creates a new instance of Keystore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewKeystore(t interface { + mock.TestingT + Cleanup(func()) +}) *Keystore { + mock := &Keystore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/mocks/tx_store.go b/core/chains/evm/txm/mocks/tx_store.go new file mode 100644 index 00000000000..318b36942b8 --- /dev/null +++ b/core/chains/evm/txm/mocks/tx_store.go @@ -0,0 +1,647 @@ +// Code generated by mockery v2.50.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + common "github.com/ethereum/go-ethereum/common" + + mock "github.com/stretchr/testify/mock" + + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +// TxStore is an autogenerated mock type for the TxStore type +type TxStore struct { + mock.Mock +} + +type TxStore_Expecter struct { + mock *mock.Mock +} + +func (_m *TxStore) EXPECT() *TxStore_Expecter { + return &TxStore_Expecter{mock: &_m.Mock} +} + +// AbandonPendingTransactions provides a mock function with given fields: _a0, _a1 +func (_m *TxStore) AbandonPendingTransactions(_a0 context.Context, _a1 common.Address) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for AbandonPendingTransactions") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_AbandonPendingTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbandonPendingTransactions' +type TxStore_AbandonPendingTransactions_Call struct { + *mock.Call +} + +// AbandonPendingTransactions is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +func (_e *TxStore_Expecter) AbandonPendingTransactions(_a0 interface{}, _a1 interface{}) *TxStore_AbandonPendingTransactions_Call { + return &TxStore_AbandonPendingTransactions_Call{Call: _e.mock.On("AbandonPendingTransactions", _a0, _a1)} +} + +func (_c *TxStore_AbandonPendingTransactions_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *TxStore_AbandonPendingTransactions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address)) + }) + return _c +} + +func (_c *TxStore_AbandonPendingTransactions_Call) Return(_a0 error) *TxStore_AbandonPendingTransactions_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_AbandonPendingTransactions_Call) RunAndReturn(run func(context.Context, common.Address) error) *TxStore_AbandonPendingTransactions_Call { + _c.Call.Return(run) + return _c +} + +// AppendAttemptToTransaction provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *TxStore) AppendAttemptToTransaction(_a0 context.Context, _a1 uint64, _a2 common.Address, _a3 *types.Attempt) error { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for AppendAttemptToTransaction") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address, *types.Attempt) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_AppendAttemptToTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AppendAttemptToTransaction' +type TxStore_AppendAttemptToTransaction_Call struct { + *mock.Call +} + +// AppendAttemptToTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 common.Address +// - _a3 *types.Attempt +func (_e *TxStore_Expecter) AppendAttemptToTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *TxStore_AppendAttemptToTransaction_Call { + return &TxStore_AppendAttemptToTransaction_Call{Call: _e.mock.On("AppendAttemptToTransaction", _a0, _a1, _a2, _a3)} +} + +func (_c *TxStore_AppendAttemptToTransaction_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address, _a3 *types.Attempt)) *TxStore_AppendAttemptToTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address), args[3].(*types.Attempt)) + }) + return _c +} + +func (_c *TxStore_AppendAttemptToTransaction_Call) Return(_a0 error) *TxStore_AppendAttemptToTransaction_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_AppendAttemptToTransaction_Call) RunAndReturn(run func(context.Context, uint64, common.Address, *types.Attempt) error) *TxStore_AppendAttemptToTransaction_Call { + _c.Call.Return(run) + return _c +} + +// CreateEmptyUnconfirmedTransaction provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *TxStore) CreateEmptyUnconfirmedTransaction(_a0 context.Context, _a1 common.Address, _a2 uint64, _a3 uint64) (*types.Transaction, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for CreateEmptyUnconfirmedTransaction") + } + + var r0 *types.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64, uint64) (*types.Transaction, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64, uint64) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, uint64, uint64) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// TxStore_CreateEmptyUnconfirmedTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateEmptyUnconfirmedTransaction' +type TxStore_CreateEmptyUnconfirmedTransaction_Call struct { + *mock.Call +} + +// CreateEmptyUnconfirmedTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +// - _a2 uint64 +// - _a3 uint64 +func (_e *TxStore_Expecter) CreateEmptyUnconfirmedTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *TxStore_CreateEmptyUnconfirmedTransaction_Call { + return &TxStore_CreateEmptyUnconfirmedTransaction_Call{Call: _e.mock.On("CreateEmptyUnconfirmedTransaction", _a0, _a1, _a2, _a3)} +} + +func (_c *TxStore_CreateEmptyUnconfirmedTransaction_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 uint64, _a3 uint64)) *TxStore_CreateEmptyUnconfirmedTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].(uint64), args[3].(uint64)) + }) + return _c +} + +func (_c *TxStore_CreateEmptyUnconfirmedTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *TxStore_CreateEmptyUnconfirmedTransaction_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *TxStore_CreateEmptyUnconfirmedTransaction_Call) RunAndReturn(run func(context.Context, common.Address, uint64, uint64) (*types.Transaction, error)) *TxStore_CreateEmptyUnconfirmedTransaction_Call { + _c.Call.Return(run) + return _c +} + +// CreateTransaction provides a mock function with given fields: _a0, _a1 +func (_m *TxStore) CreateTransaction(_a0 context.Context, _a1 *types.TxRequest) (*types.Transaction, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for CreateTransaction") + } + + var r0 *types.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *types.TxRequest) (*types.Transaction, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *types.TxRequest) *types.Transaction); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *types.TxRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// TxStore_CreateTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTransaction' +type TxStore_CreateTransaction_Call struct { + *mock.Call +} + +// CreateTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *types.TxRequest +func (_e *TxStore_Expecter) CreateTransaction(_a0 interface{}, _a1 interface{}) *TxStore_CreateTransaction_Call { + return &TxStore_CreateTransaction_Call{Call: _e.mock.On("CreateTransaction", _a0, _a1)} +} + +func (_c *TxStore_CreateTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.TxRequest)) *TxStore_CreateTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.TxRequest)) + }) + return _c +} + +func (_c *TxStore_CreateTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *TxStore_CreateTransaction_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *TxStore_CreateTransaction_Call) RunAndReturn(run func(context.Context, *types.TxRequest) (*types.Transaction, error)) *TxStore_CreateTransaction_Call { + _c.Call.Return(run) + return _c +} + +// DeleteAttemptForUnconfirmedTx provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *TxStore) DeleteAttemptForUnconfirmedTx(_a0 context.Context, _a1 uint64, _a2 *types.Attempt, _a3 common.Address) error { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for DeleteAttemptForUnconfirmedTx") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, *types.Attempt, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_DeleteAttemptForUnconfirmedTx_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteAttemptForUnconfirmedTx' +type TxStore_DeleteAttemptForUnconfirmedTx_Call struct { + *mock.Call +} + +// DeleteAttemptForUnconfirmedTx is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 *types.Attempt +// - _a3 common.Address +func (_e *TxStore_Expecter) DeleteAttemptForUnconfirmedTx(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *TxStore_DeleteAttemptForUnconfirmedTx_Call { + return &TxStore_DeleteAttemptForUnconfirmedTx_Call{Call: _e.mock.On("DeleteAttemptForUnconfirmedTx", _a0, _a1, _a2, _a3)} +} + +func (_c *TxStore_DeleteAttemptForUnconfirmedTx_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 *types.Attempt, _a3 common.Address)) *TxStore_DeleteAttemptForUnconfirmedTx_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(*types.Attempt), args[3].(common.Address)) + }) + return _c +} + +func (_c *TxStore_DeleteAttemptForUnconfirmedTx_Call) Return(_a0 error) *TxStore_DeleteAttemptForUnconfirmedTx_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_DeleteAttemptForUnconfirmedTx_Call) RunAndReturn(run func(context.Context, uint64, *types.Attempt, common.Address) error) *TxStore_DeleteAttemptForUnconfirmedTx_Call { + _c.Call.Return(run) + return _c +} + +// FetchUnconfirmedTransactionAtNonceWithCount provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) FetchUnconfirmedTransactionAtNonceWithCount(_a0 context.Context, _a1 uint64, _a2 common.Address) (*types.Transaction, int, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for FetchUnconfirmedTransactionAtNonceWithCount") + } + + var r0 *types.Transaction + var r1 int + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) (*types.Transaction, int, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64, common.Address) int); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(context.Context, uint64, common.Address) error); ok { + r2 = rf(_a0, _a1, _a2) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchUnconfirmedTransactionAtNonceWithCount' +type TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call struct { + *mock.Call +} + +// FetchUnconfirmedTransactionAtNonceWithCount is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 common.Address +func (_e *TxStore_Expecter) FetchUnconfirmedTransactionAtNonceWithCount(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { + return &TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call{Call: _e.mock.On("FetchUnconfirmedTransactionAtNonceWithCount", _a0, _a1, _a2)} +} + +func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) + }) + return _c +} + +func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) Return(_a0 *types.Transaction, _a1 int, _a2 error) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) RunAndReturn(run func(context.Context, uint64, common.Address) (*types.Transaction, int, error)) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { + _c.Call.Return(run) + return _c +} + +// MarkConfirmedAndReorgedTransactions provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) MarkConfirmedAndReorgedTransactions(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]*types.Transaction, []uint64, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for MarkConfirmedAndReorgedTransactions") + } + + var r0 []*types.Transaction + var r1 []uint64 + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) []*types.Transaction); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64, common.Address) []uint64); ok { + r1 = rf(_a0, _a1, _a2) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]uint64) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, uint64, common.Address) error); ok { + r2 = rf(_a0, _a1, _a2) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// TxStore_MarkConfirmedAndReorgedTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkConfirmedAndReorgedTransactions' +type TxStore_MarkConfirmedAndReorgedTransactions_Call struct { + *mock.Call +} + +// MarkConfirmedAndReorgedTransactions is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 common.Address +func (_e *TxStore_Expecter) MarkConfirmedAndReorgedTransactions(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkConfirmedAndReorgedTransactions_Call { + return &TxStore_MarkConfirmedAndReorgedTransactions_Call{Call: _e.mock.On("MarkConfirmedAndReorgedTransactions", _a0, _a1, _a2)} +} + +func (_c *TxStore_MarkConfirmedAndReorgedTransactions_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_MarkConfirmedAndReorgedTransactions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) + }) + return _c +} + +func (_c *TxStore_MarkConfirmedAndReorgedTransactions_Call) Return(_a0 []*types.Transaction, _a1 []uint64, _a2 error) *TxStore_MarkConfirmedAndReorgedTransactions_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *TxStore_MarkConfirmedAndReorgedTransactions_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error)) *TxStore_MarkConfirmedAndReorgedTransactions_Call { + _c.Call.Return(run) + return _c +} + +// MarkTxFatal provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) MarkTxFatal(_a0 context.Context, _a1 *types.Transaction, _a2 common.Address) error { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for MarkTxFatal") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_MarkTxFatal_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTxFatal' +type TxStore_MarkTxFatal_Call struct { + *mock.Call +} + +// MarkTxFatal is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *types.Transaction +// - _a2 common.Address +func (_e *TxStore_Expecter) MarkTxFatal(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkTxFatal_Call { + return &TxStore_MarkTxFatal_Call{Call: _e.mock.On("MarkTxFatal", _a0, _a1, _a2)} +} + +func (_c *TxStore_MarkTxFatal_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction, _a2 common.Address)) *TxStore_MarkTxFatal_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.Transaction), args[2].(common.Address)) + }) + return _c +} + +func (_c *TxStore_MarkTxFatal_Call) Return(_a0 error) *TxStore_MarkTxFatal_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_MarkTxFatal_Call) RunAndReturn(run func(context.Context, *types.Transaction, common.Address) error) *TxStore_MarkTxFatal_Call { + _c.Call.Return(run) + return _c +} + +// MarkUnconfirmedTransactionPurgeable provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) MarkUnconfirmedTransactionPurgeable(_a0 context.Context, _a1 uint64, _a2 common.Address) error { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for MarkUnconfirmedTransactionPurgeable") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_MarkUnconfirmedTransactionPurgeable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkUnconfirmedTransactionPurgeable' +type TxStore_MarkUnconfirmedTransactionPurgeable_Call struct { + *mock.Call +} + +// MarkUnconfirmedTransactionPurgeable is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 common.Address +func (_e *TxStore_Expecter) MarkUnconfirmedTransactionPurgeable(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { + return &TxStore_MarkUnconfirmedTransactionPurgeable_Call{Call: _e.mock.On("MarkUnconfirmedTransactionPurgeable", _a0, _a1, _a2)} +} + +func (_c *TxStore_MarkUnconfirmedTransactionPurgeable_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) + }) + return _c +} + +func (_c *TxStore_MarkUnconfirmedTransactionPurgeable_Call) Return(_a0 error) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_MarkUnconfirmedTransactionPurgeable_Call) RunAndReturn(run func(context.Context, uint64, common.Address) error) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { + _c.Call.Return(run) + return _c +} + +// UpdateTransactionBroadcast provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *TxStore) UpdateTransactionBroadcast(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash, _a4 common.Address) error { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) + + if len(ret) == 0 { + panic("no return value specified for UpdateTransactionBroadcast") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, uint64, common.Hash, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_UpdateTransactionBroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTransactionBroadcast' +type TxStore_UpdateTransactionBroadcast_Call struct { + *mock.Call +} + +// UpdateTransactionBroadcast is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 uint64 +// - _a3 common.Hash +// - _a4 common.Address +func (_e *TxStore_Expecter) UpdateTransactionBroadcast(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}, _a4 interface{}) *TxStore_UpdateTransactionBroadcast_Call { + return &TxStore_UpdateTransactionBroadcast_Call{Call: _e.mock.On("UpdateTransactionBroadcast", _a0, _a1, _a2, _a3, _a4)} +} + +func (_c *TxStore_UpdateTransactionBroadcast_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash, _a4 common.Address)) *TxStore_UpdateTransactionBroadcast_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(uint64), args[3].(common.Hash), args[4].(common.Address)) + }) + return _c +} + +func (_c *TxStore_UpdateTransactionBroadcast_Call) Return(_a0 error) *TxStore_UpdateTransactionBroadcast_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_UpdateTransactionBroadcast_Call) RunAndReturn(run func(context.Context, uint64, uint64, common.Hash, common.Address) error) *TxStore_UpdateTransactionBroadcast_Call { + _c.Call.Return(run) + return _c +} + +// UpdateUnstartedTransactionWithNonce provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) UpdateUnstartedTransactionWithNonce(_a0 context.Context, _a1 common.Address, _a2 uint64) (*types.Transaction, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for UpdateUnstartedTransactionWithNonce") + } + + var r0 *types.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64) (*types.Transaction, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, uint64) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// TxStore_UpdateUnstartedTransactionWithNonce_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateUnstartedTransactionWithNonce' +type TxStore_UpdateUnstartedTransactionWithNonce_Call struct { + *mock.Call +} + +// UpdateUnstartedTransactionWithNonce is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +// - _a2 uint64 +func (_e *TxStore_Expecter) UpdateUnstartedTransactionWithNonce(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_UpdateUnstartedTransactionWithNonce_Call { + return &TxStore_UpdateUnstartedTransactionWithNonce_Call{Call: _e.mock.On("UpdateUnstartedTransactionWithNonce", _a0, _a1, _a2)} +} + +func (_c *TxStore_UpdateUnstartedTransactionWithNonce_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 uint64)) *TxStore_UpdateUnstartedTransactionWithNonce_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].(uint64)) + }) + return _c +} + +func (_c *TxStore_UpdateUnstartedTransactionWithNonce_Call) Return(_a0 *types.Transaction, _a1 error) *TxStore_UpdateUnstartedTransactionWithNonce_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *TxStore_UpdateUnstartedTransactionWithNonce_Call) RunAndReturn(run func(context.Context, common.Address, uint64) (*types.Transaction, error)) *TxStore_UpdateUnstartedTransactionWithNonce_Call { + _c.Call.Return(run) + return _c +} + +// NewTxStore creates a new instance of TxStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTxStore(t interface { + mock.TestingT + Cleanup(func()) +}) *TxStore { + mock := &TxStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go new file mode 100644 index 00000000000..ae981e153b0 --- /dev/null +++ b/core/chains/evm/txm/orchestrator.go @@ -0,0 +1,363 @@ +package txm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" + nullv4 "gopkg.in/guregu/null.v4" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-common/pkg/utils" + + "github.com/smartcontractkit/chainlink/v2/common/txmgr" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/common/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/forwarders" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + txmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" +) + +type OrchestratorTxStore interface { + Add(addresses ...common.Address) error + FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*txmtypes.Transaction, int, error) + FindTxWithIdempotencyKey(context.Context, string) (*txmtypes.Transaction, error) +} + +type OrchestratorKeystore interface { + CheckEnabled(ctx context.Context, address common.Address, chainID *big.Int) error + EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) +} + +type OrchestratorAttemptBuilder[ + BLOCK_HASH types.Hashable, + HEAD types.Head[BLOCK_HASH], +] interface { + services.Service + OnNewLongestChain(ctx context.Context, head HEAD) +} + +// Generics are necessary to keep TXMv2 backwards compatible +type Orchestrator[ + BLOCK_HASH types.Hashable, + HEAD types.Head[BLOCK_HASH], +] struct { + services.StateMachine + lggr logger.SugaredLogger + chainID *big.Int + txm *Txm + txStore OrchestratorTxStore + fwdMgr *forwarders.FwdMgr + keystore OrchestratorKeystore + attemptBuilder OrchestratorAttemptBuilder[BLOCK_HASH, HEAD] + resumeCallback txmgr.ResumeCallback +} + +func NewTxmOrchestrator[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]]( + lggr logger.Logger, + chainID *big.Int, + txm *Txm, + txStore OrchestratorTxStore, + fwdMgr *forwarders.FwdMgr, + keystore OrchestratorKeystore, + attemptBuilder OrchestratorAttemptBuilder[BLOCK_HASH, HEAD], +) *Orchestrator[BLOCK_HASH, HEAD] { + return &Orchestrator[BLOCK_HASH, HEAD]{ + lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), + chainID: chainID, + txm: txm, + txStore: txStore, + keystore: keystore, + attemptBuilder: attemptBuilder, + fwdMgr: fwdMgr, + } +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) Start(ctx context.Context) error { + return o.StartOnce("Orchestrator", func() error { + var ms services.MultiStart + if err := ms.Start(ctx, o.attemptBuilder); err != nil { + return fmt.Errorf("Orchestrator: AttemptBuilder failed to start: %w", err) + } + addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) + if err != nil { + return err + } + for _, address := range addresses { + err := o.txStore.Add(address) + if err != nil { + return err + } + } + if err := ms.Start(ctx, o.txm); err != nil { + return fmt.Errorf("Orchestrator: Txm failed to start: %w", err) + } + if o.fwdMgr != nil { + if err := ms.Start(ctx, o.fwdMgr); err != nil { + return fmt.Errorf("Orchestrator: ForwarderManager failed to start: %w", err) + } + } + return nil + }) +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) Close() (merr error) { + return o.StopOnce("Orchestrator", func() error { + if o.fwdMgr != nil { + if err := o.fwdMgr.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop ForwarderManager: %w", err)) + } + } + if err := o.txm.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop Txm: %w", err)) + } + if err := o.attemptBuilder.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop AttemptBuilder: %w", err)) + } + return merr + }) +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) Trigger(addr common.Address) { + o.txm.Trigger(addr) +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) Name() string { + return o.lggr.Name() +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) HealthReport() map[string]error { + return map[string]error{o.Name(): o.Healthy()} +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) RegisterResumeCallback(fn txmgr.ResumeCallback) { + o.resumeCallback = fn +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) Reset(addr common.Address, abandon bool) error { + ok := o.IfStarted(func() { + if err := o.txm.Abandon(addr); err != nil { + o.lggr.Error(err) + } + }) + if !ok { + return errors.New("Orchestrator not started yet") + } + return nil +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) OnNewLongestChain(ctx context.Context, head HEAD) { + ok := o.IfStarted(func() { + o.attemptBuilder.OnNewLongestChain(ctx, head) + }) + if !ok { + o.lggr.Debugw("Not started; ignoring head", "head", head, "state", o.State()) + } +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + var wrappedTx *txmtypes.Transaction + if request.IdempotencyKey != nil { + wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(ctx, *request.IdempotencyKey) + if err != nil { + return + } + } + + if wrappedTx != nil { + o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", *wrappedTx.IdempotencyKey) + } else { + if kErr := o.keystore.CheckEnabled(ctx, request.FromAddress, o.chainID); kErr != nil { + return tx, fmt.Errorf("cannot send transaction from %s on chain ID %s: %w", request.FromAddress, o.chainID.String(), kErr) + } + + var pipelineTaskRunID uuid.NullUUID + if request.PipelineTaskRunID != nil { + pipelineTaskRunID.UUID = *request.PipelineTaskRunID + pipelineTaskRunID.Valid = true + } + + if o.fwdMgr != nil && (!utils.IsZero(request.ForwarderAddress)) { + fwdPayload, fwdErr := o.fwdMgr.ConvertPayload(request.ToAddress, request.EncodedPayload) + if fwdErr == nil { + // Handling meta not set at caller. + if request.Meta != nil { + request.Meta.FwdrDestAddress = &request.ToAddress + } else { + request.Meta = &txmgrtypes.TxMeta[common.Address, common.Hash]{ + FwdrDestAddress: &request.ToAddress, + } + } + request.ToAddress = request.ForwarderAddress + request.EncodedPayload = fwdPayload + } else { + o.lggr.Errorf("Failed to use forwarder set upstream: %v", fwdErr.Error()) + } + } + + var meta *sqlutil.JSON + if request.Meta != nil { + raw, mErr := json.Marshal(request.Meta) + if mErr != nil { + return tx, mErr + } + m := sqlutil.JSON(raw) + meta = &m + } + + wrappedTxRequest := &txmtypes.TxRequest{ + IdempotencyKey: request.IdempotencyKey, + ChainID: o.chainID, + FromAddress: request.FromAddress, + ToAddress: request.ToAddress, + Value: &request.Value, + Data: request.EncodedPayload, + SpecifiedGasLimit: request.FeeLimit, + Meta: meta, + ForwarderAddress: request.ForwarderAddress, + + PipelineTaskRunID: pipelineTaskRunID, + MinConfirmations: request.MinConfirmations, + SignalCallback: request.SignalCallback, + } + + wrappedTx, err = o.txm.CreateTransaction(ctx, wrappedTxRequest) + if err != nil { + return + } + o.txm.Trigger(request.FromAddress) + } + + if wrappedTx.ID > math.MaxInt64 { + return tx, fmt.Errorf("overflow for int64: %d", wrappedTx.ID) + } + + tx = txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]{ + ID: int64(wrappedTx.ID), + IdempotencyKey: wrappedTx.IdempotencyKey, + FromAddress: wrappedTx.FromAddress, + ToAddress: wrappedTx.ToAddress, + EncodedPayload: wrappedTx.Data, + Value: *wrappedTx.Value, + FeeLimit: wrappedTx.SpecifiedGasLimit, + CreatedAt: wrappedTx.CreatedAt, + Meta: wrappedTx.Meta, + Subject: wrappedTx.Subject, + ChainID: wrappedTx.ChainID, + + PipelineTaskRunID: wrappedTx.PipelineTaskRunID, + MinConfirmations: wrappedTx.MinConfirmations, + SignalCallback: wrappedTx.SignalCallback, + CallbackCompleted: wrappedTx.CallbackCompleted, + } + return +} + +// CountTransactionsByState was required for backwards compatibility and it's used only for unconfirmed transactions. +func (o *Orchestrator[BLOCK_HASH, HEAD]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (uint32, error) { + addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) + if err != nil { + return 0, err + } + total := 0 + for _, address := range addresses { + _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, address) + if err != nil { + return 0, err + } + total += count + } + + //nolint:gosec // disable G115 + return uint32(total), nil +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (time nullv4.Int, err error) { + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} + +//nolint:revive // keep API backwards compatible +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) { + if o.fwdMgr != nil { + forwarder, err = o.fwdMgr.ForwarderFor(ctx, eoa) + } + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) { + if o.fwdMgr != nil { + forwarder, err = o.fwdMgr.ForwarderForOCR2Feeds(ctx, eoa, ocr2AggregatorID) + } + return +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { + // Loads attempts and receipts in the transaction + tx, err := o.txStore.FindTxWithIdempotencyKey(ctx, transactionID) + if err != nil || tx == nil { + return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s: %w", transactionID, err) + } + + switch tx.State { + case txmgr.TxUnconfirmed: + return commontypes.Pending, nil + case txmgr.TxConfirmed: + // Return unconfirmed for confirmed transactions because they are not yet finalized + return commontypes.Unconfirmed, nil + case txmgr.TxFinalized: + return commontypes.Finalized, nil + case txmgr.TxFatalError: + return commontypes.Fatal, nil + default: + return commontypes.Unknown, nil + } +} + +func (o *Orchestrator[BLOCK_HASH, HEAD]) SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value big.Int, gasLimit uint64) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + txRequest := txmgrtypes.TxRequest[common.Address, common.Hash]{ + FromAddress: from, + ToAddress: to, + EncodedPayload: []byte{}, + Value: value, + FeeLimit: gasLimit, + //Strategy: NewSendEveryStrategy(), + } + + tx, err = o.CreateTransaction(ctx, txRequest) + if err != nil { + return + } + + // Trigger the Txm to check for new transaction + o.txm.Trigger(from) + return tx, err +} diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go new file mode 100644 index 00000000000..57217913d76 --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -0,0 +1,358 @@ +package storage + +import ( + "errors" + "fmt" + "math/big" + "sort" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + txmgr "github.com/smartcontractkit/chainlink/v2/common/txmgr" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +const ( + // maxQueuedTransactions is the max limit of UnstartedTransactions and ConfirmedTransactions structures. + maxQueuedTransactions = 250 + // pruneSubset controls the subset of confirmed transactions to prune when the structure reaches its max limit. + // i.e. if the value is 3 and the limit is 90, 30 transactions will be pruned. + pruneSubset = 3 +) + +type InMemoryStore struct { + sync.RWMutex + lggr logger.Logger + txIDCount uint64 + address common.Address + chainID *big.Int + + UnstartedTransactions []*types.Transaction + UnconfirmedTransactions map[uint64]*types.Transaction + ConfirmedTransactions map[uint64]*types.Transaction + FatalTransactions []*types.Transaction + + Transactions map[uint64]*types.Transaction +} + +func NewInMemoryStore(lggr logger.Logger, address common.Address, chainID *big.Int) *InMemoryStore { + return &InMemoryStore{ + lggr: logger.Named(lggr, "InMemoryStore"), + address: address, + chainID: chainID, + UnstartedTransactions: make([]*types.Transaction, 0, maxQueuedTransactions), + UnconfirmedTransactions: make(map[uint64]*types.Transaction), + ConfirmedTransactions: make(map[uint64]*types.Transaction, maxQueuedTransactions), + Transactions: make(map[uint64]*types.Transaction), + } +} + +func (m *InMemoryStore) AbandonPendingTransactions() { + // TODO: append existing fatal transactions and cap the size + m.Lock() + defer m.Unlock() + + for _, tx := range m.UnstartedTransactions { + tx.State = txmgr.TxFatalError + } + for _, tx := range m.FatalTransactions { + delete(m.Transactions, tx.ID) + } + m.FatalTransactions = m.UnstartedTransactions + m.UnstartedTransactions = []*types.Transaction{} + + for _, tx := range m.UnconfirmedTransactions { + tx.State = txmgr.TxFatalError + m.FatalTransactions = append(m.FatalTransactions, tx) + } + m.UnconfirmedTransactions = make(map[uint64]*types.Transaction) +} + +func (m *InMemoryStore) AppendAttemptToTransaction(txNonce uint64, attempt *types.Attempt) error { + m.Lock() + defer m.Unlock() + + tx, exists := m.UnconfirmedTransactions[txNonce] + if !exists { + return fmt.Errorf("unconfirmed tx was not found for nonce: %d - txID: %v", txNonce, attempt.TxID) + } + + if tx.ID != attempt.TxID { + return fmt.Errorf("unconfirmed tx with nonce exists but attempt points to a different txID. Found Tx: %v - txID: %v", m.UnconfirmedTransactions[txNonce], attempt.TxID) + } + + attempt.CreatedAt = time.Now() + attempt.ID = uint64(len(tx.Attempts)) // Attempts are not collectively tracked by the in-memory store so attemptIDs are not unique between transactions and can be reused. + tx.AttemptCount++ + m.UnconfirmedTransactions[txNonce].Attempts = append(m.UnconfirmedTransactions[txNonce].Attempts, attempt.DeepCopy()) + + return nil +} + +func (m *InMemoryStore) CountUnstartedTransactions() int { + m.RLock() + defer m.RUnlock() + + return len(m.UnstartedTransactions) +} + +func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + emptyTx := &types.Transaction{ + ID: m.txIDCount, + ChainID: m.chainID, + Nonce: &nonce, + FromAddress: m.address, + ToAddress: common.Address{}, + Value: big.NewInt(0), + SpecifiedGasLimit: gasLimit, + CreatedAt: time.Now(), + State: txmgr.TxUnconfirmed, + } + + if _, exists := m.UnconfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) + } + + if _, exists := m.ConfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("a confirmed tx with the same nonce already exists: %v", m.ConfirmedTransactions[nonce]) + } + + m.txIDCount++ + m.UnconfirmedTransactions[nonce] = emptyTx + m.Transactions[emptyTx.ID] = emptyTx + + return emptyTx.DeepCopy(), nil +} + +func (m *InMemoryStore) CreateTransaction(txRequest *types.TxRequest) *types.Transaction { + m.Lock() + defer m.Unlock() + + tx := &types.Transaction{ + ID: m.txIDCount, + IdempotencyKey: txRequest.IdempotencyKey, + ChainID: m.chainID, + FromAddress: m.address, + ToAddress: txRequest.ToAddress, + Value: txRequest.Value, + Data: txRequest.Data, + SpecifiedGasLimit: txRequest.SpecifiedGasLimit, + CreatedAt: time.Now(), + State: txmgr.TxUnstarted, + Meta: txRequest.Meta, + MinConfirmations: txRequest.MinConfirmations, + PipelineTaskRunID: txRequest.PipelineTaskRunID, + SignalCallback: txRequest.SignalCallback, + } + + uLen := len(m.UnstartedTransactions) + if uLen >= maxQueuedTransactions { + m.lggr.Warnw(fmt.Sprintf("Unstarted transactions queue for address: %v reached max limit of: %d. Dropping oldest transactions", m.address, maxQueuedTransactions), + "txs", m.UnstartedTransactions[0:uLen-maxQueuedTransactions+1]) // need to make room for the new tx + for _, tx := range m.UnstartedTransactions[0 : uLen-maxQueuedTransactions+1] { + delete(m.Transactions, tx.ID) + } + m.UnstartedTransactions = m.UnstartedTransactions[uLen-maxQueuedTransactions+1:] + } + + m.txIDCount++ + txCopy := tx.DeepCopy() + m.Transactions[txCopy.ID] = txCopy + m.UnstartedTransactions = append(m.UnstartedTransactions, txCopy) + return tx +} + +func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(latestNonce uint64) (txCopy *types.Transaction, unconfirmedCount int) { + m.RLock() + defer m.RUnlock() + + tx := m.UnconfirmedTransactions[latestNonce] + if tx != nil { + txCopy = tx.DeepCopy() + } + unconfirmedCount = len(m.UnconfirmedTransactions) + return +} + +func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) ([]*types.Transaction, []uint64, error) { + m.Lock() + defer m.Unlock() + + var confirmedTransactions []*types.Transaction + for _, tx := range m.UnconfirmedTransactions { + if tx.Nonce == nil { + return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + existingTx, exists := m.ConfirmedTransactions[*tx.Nonce] + if exists { + m.lggr.Errorw("Another confirmed transaction with the same nonce exists. Transaction will be overwritten.", + "existingTx", existingTx, "newTx", tx) + } + if *tx.Nonce < latestNonce { + tx.State = txmgr.TxConfirmed + confirmedTransactions = append(confirmedTransactions, tx.DeepCopy()) + m.ConfirmedTransactions[*tx.Nonce] = tx + delete(m.UnconfirmedTransactions, *tx.Nonce) + } + } + + var unconfirmedTransactionIDs []uint64 + for _, tx := range m.ConfirmedTransactions { + if tx.Nonce == nil { + return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + existingTx, exists := m.UnconfirmedTransactions[*tx.Nonce] + if exists { + m.lggr.Errorw("Another unconfirmed transaction with the same nonce exists. Transaction will overwritten.", + "existingTx", existingTx, "newTx", tx) + } + if *tx.Nonce >= latestNonce { + tx.State = txmgr.TxUnconfirmed + tx.LastBroadcastAt = nil // Mark reorged transaction as if it wasn't broadcasted before + unconfirmedTransactionIDs = append(unconfirmedTransactionIDs, tx.ID) + m.UnconfirmedTransactions[*tx.Nonce] = tx + delete(m.ConfirmedTransactions, *tx.Nonce) + } + } + + if len(m.ConfirmedTransactions) > maxQueuedTransactions { + prunedTxIDs := m.pruneConfirmedTransactions() + m.lggr.Debugf("Confirmed transactions map for address: %v reached max limit of: %d. Pruned 1/%d of the oldest confirmed transactions. TxIDs: %v", + m.address, maxQueuedTransactions, pruneSubset, prunedTxIDs) + } + sort.Slice(confirmedTransactions, func(i, j int) bool { return confirmedTransactions[i].ID < confirmedTransactions[j].ID }) + sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) + return confirmedTransactions, unconfirmedTransactionIDs, nil +} + +func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(nonce uint64) error { + m.Lock() + defer m.Unlock() + + tx, exists := m.UnconfirmedTransactions[nonce] + if !exists { + return fmt.Errorf("unconfirmed tx with nonce: %d was not found", nonce) + } + + tx.IsPurgeable = true + + return nil +} + +func (m *InMemoryStore) UpdateTransactionBroadcast(txID uint64, txNonce uint64, attemptHash common.Hash) error { + m.Lock() + defer m.Unlock() + + unconfirmedTx, exists := m.UnconfirmedTransactions[txNonce] + if !exists { + return fmt.Errorf("unconfirmed tx was not found for nonce: %d - txID: %v", txNonce, txID) + } + + // Set the same time for both the tx and its attempt + now := time.Now() + unconfirmedTx.LastBroadcastAt = &now + if unconfirmedTx.InitialBroadcastAt == nil { + unconfirmedTx.InitialBroadcastAt = &now + } + a, err := unconfirmedTx.FindAttemptByHash(attemptHash) + if err != nil { + return fmt.Errorf("UpdateTransactionBroadcast failed to find attempt. %w", err) + } + a.BroadcastAt = &now + + return nil +} + +func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + if len(m.UnstartedTransactions) == 0 { + m.lggr.Debugf("Unstarted transactions queue is empty for address: %v", m.address) + return nil, nil + } + + if tx, exists := m.UnconfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", tx) + } + + tx := m.UnstartedTransactions[0] + tx.Nonce = &nonce + tx.State = txmgr.TxUnconfirmed + + m.UnstartedTransactions = m.UnstartedTransactions[1:] + m.UnconfirmedTransactions[nonce] = tx + + return tx.DeepCopy(), nil +} + +// Shouldn't call lock because it's being called by a method that already has the lock +func (m *InMemoryStore) pruneConfirmedTransactions() []uint64 { + noncesToPrune := make([]uint64, 0, len(m.ConfirmedTransactions)) + for nonce := range m.ConfirmedTransactions { + noncesToPrune = append(noncesToPrune, nonce) + } + if len(noncesToPrune) == 0 { + return nil + } + sort.Slice(noncesToPrune, func(i, j int) bool { return noncesToPrune[i] < noncesToPrune[j] }) + minNonce := noncesToPrune[len(noncesToPrune)/pruneSubset] + + var txIDsToPrune []uint64 + for nonce, tx := range m.ConfirmedTransactions { + if nonce < minNonce { + txIDsToPrune = append(txIDsToPrune, tx.ID) + delete(m.Transactions, tx.ID) + delete(m.ConfirmedTransactions, nonce) + } + } + + sort.Slice(txIDsToPrune, func(i, j int) bool { return txIDsToPrune[i] < txIDsToPrune[j] }) + return txIDsToPrune +} + +// Error Handler +func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(transactionNonce uint64, attempt *types.Attempt) error { + m.Lock() + defer m.Unlock() + + tx, exists := m.UnconfirmedTransactions[transactionNonce] + if !exists { + return fmt.Errorf("unconfirmed tx was not found for nonce: %d - txID: %v", transactionNonce, attempt.TxID) + } + + for i, a := range tx.Attempts { + if a.Hash == attempt.Hash { + tx.Attempts = append(tx.Attempts[:i], tx.Attempts[i+1:]...) + return nil + } + } + + return fmt.Errorf("attempt with hash: %v for txID: %v was not found", attempt.Hash, attempt.TxID) +} + +func (m *InMemoryStore) MarkTxFatal(*types.Transaction) error { + return errors.New("not implemented") +} + +// Orchestrator +func (m *InMemoryStore) FindTxWithIdempotencyKey(idempotencyKey string) *types.Transaction { + m.RLock() + defer m.RUnlock() + + for _, tx := range m.Transactions { + if tx.IdempotencyKey != nil && *tx.IdempotencyKey == idempotencyKey { + return tx.DeepCopy() + } + } + + return nil +} diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go new file mode 100644 index 00000000000..a7538823eea --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -0,0 +1,136 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +const StoreNotFoundForAddress string = "InMemoryStore for address: %v not found" + +type InMemoryStoreManager struct { + lggr logger.Logger + chainID *big.Int + InMemoryStoreMap map[common.Address]*InMemoryStore +} + +func NewInMemoryStoreManager(lggr logger.Logger, chainID *big.Int) *InMemoryStoreManager { + inMemoryStoreMap := make(map[common.Address]*InMemoryStore) + return &InMemoryStoreManager{ + lggr: lggr, + chainID: chainID, + InMemoryStoreMap: inMemoryStoreMap} +} + +func (m *InMemoryStoreManager) AbandonPendingTransactions(_ context.Context, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + store.AbandonPendingTransactions() + return nil + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) Add(addresses ...common.Address) (err error) { + for _, address := range addresses { + if _, exists := m.InMemoryStoreMap[address]; exists { + err = errors.Join(err, fmt.Errorf("address %v already exists in store manager", address)) + } + m.InMemoryStoreMap[address] = NewInMemoryStore(m.lggr, address, m.chainID) + } + return +} + +func (m *InMemoryStoreManager) AppendAttemptToTransaction(_ context.Context, txNonce uint64, fromAddress common.Address, attempt *types.Attempt) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.AppendAttemptToTransaction(txNonce, attempt) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) CountUnstartedTransactions(fromAddress common.Address) (int, error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.CountUnstartedTransactions(), nil + } + return 0, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) CreateEmptyUnconfirmedTransaction(_ context.Context, fromAddress common.Address, nonce uint64, gasLimit uint64) (*types.Transaction, error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.CreateEmptyUnconfirmedTransaction(nonce, gasLimit) + } + return nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) CreateTransaction(_ context.Context, txRequest *types.TxRequest) (*types.Transaction, error) { + if store, exists := m.InMemoryStoreMap[txRequest.FromAddress]; exists { + return store.CreateTransaction(txRequest), nil + } + return nil, fmt.Errorf(StoreNotFoundForAddress, txRequest.FromAddress) +} + +func (m *InMemoryStoreManager) FetchUnconfirmedTransactionAtNonceWithCount(_ context.Context, nonce uint64, fromAddress common.Address) (tx *types.Transaction, count int, err error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + tx, count = store.FetchUnconfirmedTransactionAtNonceWithCount(nonce) + return + } + return nil, 0, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) MarkConfirmedAndReorgedTransactions(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxs []*types.Transaction, unconfirmedTxIDs []uint64, err error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + confirmedTxs, unconfirmedTxIDs, err = store.MarkConfirmedAndReorgedTransactions(nonce) + return + } + return nil, nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) MarkUnconfirmedTransactionPurgeable(_ context.Context, nonce uint64, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.MarkUnconfirmedTransactionPurgeable(nonce) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) UpdateTransactionBroadcast(_ context.Context, txID uint64, nonce uint64, attemptHash common.Hash, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.UpdateTransactionBroadcast(txID, nonce, attemptHash) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) UpdateUnstartedTransactionWithNonce(_ context.Context, fromAddress common.Address, nonce uint64) (*types.Transaction, error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.UpdateUnstartedTransactionWithNonce(nonce) + } + return nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) DeleteAttemptForUnconfirmedTx(_ context.Context, nonce uint64, attempt *types.Attempt, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.DeleteAttemptForUnconfirmedTx(nonce, attempt) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) MarkTxFatal(_ context.Context, tx *types.Transaction, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.MarkTxFatal(tx) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) FindTxWithIdempotencyKey(_ context.Context, idempotencyKey string) (*types.Transaction, error) { + for _, store := range m.InMemoryStoreMap { + tx := store.FindTxWithIdempotencyKey(idempotencyKey) + if tx != nil { + return tx, nil + } + } + return nil, nil +} diff --git a/core/chains/evm/txm/storage/inmemory_store_manager_test.go b/core/chains/evm/txm/storage/inmemory_store_manager_test.go new file mode 100644 index 00000000000..aff589fb9e1 --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store_manager_test.go @@ -0,0 +1,36 @@ +package storage + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" +) + +func TestAdd(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStoreManager(logger.Test(t), testutils.FixtureChainID) + // Adds a new address + err := m.Add(fromAddress) + require.NoError(t, err) + assert.Len(t, m.InMemoryStoreMap, 1) + + // Fails if address exists + err = m.Add(fromAddress) + require.Error(t, err) + + // Adds multiple addresses + fromAddress1 := testutils.NewAddress() + fromAddress2 := testutils.NewAddress() + addresses := []common.Address{fromAddress1, fromAddress2} + err = m.Add(addresses...) + require.NoError(t, err) + assert.Len(t, m.InMemoryStoreMap, 3) +} diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go new file mode 100644 index 00000000000..226cf284bba --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -0,0 +1,559 @@ +package storage + +import ( + "fmt" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink/v2/common/txmgr" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +func TestAbandonPendingTransactions(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + t.Run("abandons unstarted and unconfirmed transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + // Unstarted + tx1 := insertUnstartedTransaction(m) + tx2 := insertUnstartedTransaction(m) + + // Unconfirmed + tx3, err := insertUnconfirmedTransaction(m, 3) + require.NoError(t, err) + tx4, err := insertUnconfirmedTransaction(m, 4) + require.NoError(t, err) + + m.AbandonPendingTransactions() + + assert.Equal(t, txmgr.TxFatalError, tx1.State) + assert.Equal(t, txmgr.TxFatalError, tx2.State) + assert.Equal(t, txmgr.TxFatalError, tx3.State) + assert.Equal(t, txmgr.TxFatalError, tx4.State) + }) + + t.Run("skips all types apart from unstarted and unconfirmed transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + // Fatal + tx1 := insertFataTransaction(m) + tx2 := insertFataTransaction(m) + + // Confirmed + tx3, err := insertConfirmedTransaction(m, 3) + require.NoError(t, err) + tx4, err := insertConfirmedTransaction(m, 4) + require.NoError(t, err) + + m.AbandonPendingTransactions() + + assert.Equal(t, txmgr.TxFatalError, tx1.State) + assert.Equal(t, txmgr.TxFatalError, tx2.State) + assert.Equal(t, txmgr.TxConfirmed, tx3.State) + assert.Equal(t, txmgr.TxConfirmed, tx4.State) + assert.Len(t, m.Transactions, 2) // tx1, tx2 were dropped + }) +} + +func TestAppendAttemptToTransaction(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + + _, err := insertUnconfirmedTransaction(m, 10) // txID = 1, nonce = 10 + require.NoError(t, err) + _, err = insertConfirmedTransaction(m, 2) // txID = 2, nonce = 2 + require.NoError(t, err) + + t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { + var nonce uint64 = 1 + newAttempt := &types.Attempt{} + err := m.AppendAttemptToTransaction(nonce, newAttempt) + require.Error(t, err) + require.ErrorContains(t, err, "unconfirmed tx was not found") + }) + + t.Run("fails if unconfirmed transaction was found but doesn't match the txID", func(t *testing.T) { + var nonce uint64 = 10 + newAttempt := &types.Attempt{ + TxID: 2, + } + err := m.AppendAttemptToTransaction(nonce, newAttempt) + require.Error(t, err) + require.ErrorContains(t, err, "attempt points to a different txID") + }) + + t.Run("appends attempt to transaction", func(t *testing.T) { + var nonce uint64 = 10 + newAttempt := &types.Attempt{ + TxID: 1, + } + require.NoError(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + tx, _ := m.FetchUnconfirmedTransactionAtNonceWithCount(10) + assert.Len(t, tx.Attempts, 1) + assert.Equal(t, uint16(1), tx.AttemptCount) + assert.False(t, tx.Attempts[0].CreatedAt.IsZero()) + }) +} + +func TestCountUnstartedTransactions(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + + assert.Equal(t, 0, m.CountUnstartedTransactions()) + + insertUnstartedTransaction(m) + assert.Equal(t, 1, m.CountUnstartedTransactions()) + + _, err := insertConfirmedTransaction(m, 10) + require.NoError(t, err) + assert.Equal(t, 1, m.CountUnstartedTransactions()) +} + +func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + _, err := insertUnconfirmedTransaction(m, 1) + require.NoError(t, err) + _, err = insertConfirmedTransaction(m, 0) + require.NoError(t, err) + + t.Run("fails if unconfirmed transaction with the same nonce exists", func(t *testing.T) { + _, err := m.CreateEmptyUnconfirmedTransaction(1, 0) + require.Error(t, err) + }) + + t.Run("fails if confirmed transaction with the same nonce exists", func(t *testing.T) { + _, err := m.CreateEmptyUnconfirmedTransaction(0, 0) + require.Error(t, err) + }) + + t.Run("creates a new empty unconfirmed transaction", func(t *testing.T) { + tx, err := m.CreateEmptyUnconfirmedTransaction(2, 0) + require.NoError(t, err) + assert.Equal(t, txmgr.TxUnconfirmed, tx.State) + }) +} + +func TestCreateTransaction(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + + t.Run("creates new transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + now := time.Now() + txR1 := &types.TxRequest{} + txR2 := &types.TxRequest{} + tx1 := m.CreateTransaction(txR1) + assert.Equal(t, uint64(0), tx1.ID) + assert.LessOrEqual(t, now, tx1.CreatedAt) + + tx2 := m.CreateTransaction(txR2) + assert.Equal(t, uint64(1), tx2.ID) + assert.LessOrEqual(t, now, tx2.CreatedAt) + + assert.Equal(t, 2, m.CountUnstartedTransactions()) + }) + + t.Run("prunes oldest unstarted transactions if limit is reached", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + overshot := 5 + for i := 0; i < maxQueuedTransactions+overshot; i++ { + r := &types.TxRequest{} + tx := m.CreateTransaction(r) + //nolint:gosec // this won't overflow + assert.Equal(t, uint64(i), tx.ID) + } + // total shouldn't exceed maxQueuedTransactions + assert.Equal(t, maxQueuedTransactions, m.CountUnstartedTransactions()) + // earliest tx ID should be the same amount of the number of transactions that we dropped + tx, err := m.UpdateUnstartedTransactionWithNonce(0) + require.NoError(t, err) + //nolint:gosec // this won't overflow + assert.Equal(t, uint64(overshot), tx.ID) + }) +} + +func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + + tx, count := m.FetchUnconfirmedTransactionAtNonceWithCount(0) + assert.Nil(t, tx) + assert.Equal(t, 0, count) + + var nonce uint64 + _, err := insertUnconfirmedTransaction(m, nonce) + require.NoError(t, err) + tx, count = m.FetchUnconfirmedTransactionAtNonceWithCount(0) + assert.Equal(t, *tx.Nonce, nonce) + assert.Equal(t, 1, count) +} + +func TestMarkConfirmedAndReorgedTransactions(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + + t.Run("returns 0 if there are no transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + un, cn, err := m.MarkConfirmedAndReorgedTransactions(100) + require.NoError(t, err) + assert.Empty(t, un) + assert.Empty(t, cn) + }) + + t.Run("confirms transaction with nonce lower than the latest", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + ctx1, err := insertUnconfirmedTransaction(m, 0) + require.NoError(t, err) + + ctx2, err := insertUnconfirmedTransaction(m, 1) + require.NoError(t, err) + + ctxs, utxs, err := m.MarkConfirmedAndReorgedTransactions(1) + require.NoError(t, err) + assert.Equal(t, txmgr.TxConfirmed, ctx1.State) + assert.Equal(t, txmgr.TxUnconfirmed, ctx2.State) + assert.Equal(t, ctxs[0].ID, ctx1.ID) // Ensure order + assert.Empty(t, utxs) + }) + + t.Run("state remains the same if nonce didn't change", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + ctx1, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + + ctx2, err := insertUnconfirmedTransaction(m, 1) + require.NoError(t, err) + + ctxs, utxs, err := m.MarkConfirmedAndReorgedTransactions(1) + require.NoError(t, err) + assert.Equal(t, txmgr.TxConfirmed, ctx1.State) + assert.Equal(t, txmgr.TxUnconfirmed, ctx2.State) + assert.Empty(t, ctxs) + assert.Empty(t, utxs) + }) + + t.Run("unconfirms transaction with nonce equal to or higher than the latest", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + ctx1, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + + ctx2, err := insertConfirmedTransaction(m, 1) + require.NoError(t, err) + + ctxs, utxs, err := m.MarkConfirmedAndReorgedTransactions(1) + require.NoError(t, err) + assert.Equal(t, txmgr.TxConfirmed, ctx1.State) + assert.Equal(t, txmgr.TxUnconfirmed, ctx2.State) + assert.Equal(t, utxs[0], ctx2.ID) + assert.Empty(t, ctxs) + }) + + t.Run("logs an error during confirmation if a transaction with the same nonce already exists", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + m := NewInMemoryStore(lggr, fromAddress, testutils.FixtureChainID) + _, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + _, err = insertUnconfirmedTransaction(m, 0) + require.NoError(t, err) + + _, _, err = m.MarkConfirmedAndReorgedTransactions(1) + require.NoError(t, err) + tests.AssertLogEventually(t, observedLogs, "Another confirmed transaction with the same nonce exists") + }) + + t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + overshot := 5 + for i := 0; i < maxQueuedTransactions+overshot; i++ { + //nolint:gosec // this won't overflow + _, err := insertConfirmedTransaction(m, uint64(i)) + require.NoError(t, err) + } + assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions+overshot) + //nolint:gosec // this won't overflow + _, _, err := m.MarkConfirmedAndReorgedTransactions(uint64(maxQueuedTransactions + overshot)) + require.NoError(t, err) + assert.Len(t, m.ConfirmedTransactions, 170) + }) +} + +func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + + // fails if tx was not found + err := m.MarkUnconfirmedTransactionPurgeable(0) + require.Error(t, err) + + tx, err := insertUnconfirmedTransaction(m, 0) + require.NoError(t, err) + err = m.MarkUnconfirmedTransactionPurgeable(0) + require.NoError(t, err) + assert.True(t, tx.IsPurgeable) +} + +func TestUpdateTransactionBroadcast(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + hash := testutils.NewHash() + t.Run("fails if unconfirmed transaction was not found", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + var nonce uint64 + require.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + }) + + t.Run("fails if attempt was not found for a given transaction", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + var nonce uint64 + tx, err := insertUnconfirmedTransaction(m, nonce) + require.NoError(t, err) + require.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + + // Attempt with different hash + attempt := &types.Attempt{TxID: tx.ID, Hash: testutils.NewHash()} + tx.Attempts = append(tx.Attempts, attempt) + require.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + }) + + t.Run("updates transaction's and attempt's broadcast times", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + var nonce uint64 + tx, err := insertUnconfirmedTransaction(m, nonce) + require.NoError(t, err) + attempt := &types.Attempt{TxID: tx.ID, Hash: hash} + tx.Attempts = append(tx.Attempts, attempt) + require.NoError(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + assert.False(t, tx.LastBroadcastAt.IsZero()) + assert.False(t, attempt.BroadcastAt.IsZero()) + assert.False(t, tx.InitialBroadcastAt.IsZero()) + }) +} + +func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + t.Run("returns nil if there are no unstarted transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + tx, err := m.UpdateUnstartedTransactionWithNonce(0) + require.NoError(t, err) + assert.Nil(t, tx) + }) + + t.Run("fails if there is already another unconfirmed transaction with the same nonce", func(t *testing.T) { + var nonce uint64 + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + insertUnstartedTransaction(m) + _, err := insertUnconfirmedTransaction(m, nonce) + require.NoError(t, err) + + _, err = m.UpdateUnstartedTransactionWithNonce(nonce) + require.Error(t, err) + }) + + t.Run("updates unstarted transaction to unconfirmed and assigns a nonce", func(t *testing.T) { + var nonce uint64 + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + insertUnstartedTransaction(m) + + tx, err := m.UpdateUnstartedTransactionWithNonce(nonce) + require.NoError(t, err) + assert.Equal(t, nonce, *tx.Nonce) + assert.Equal(t, txmgr.TxUnconfirmed, tx.State) + assert.Empty(t, m.UnstartedTransactions) + }) +} + +func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + var nonce uint64 + tx := &types.Transaction{Nonce: &nonce} + attempt := &types.Attempt{TxID: 0} + err := m.DeleteAttemptForUnconfirmedTx(*tx.Nonce, attempt) + require.Error(t, err) + }) + + t.Run("fails if corresponding unconfirmed attempt for txID was not found", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + _, err := insertUnconfirmedTransaction(m, 0) + require.NoError(t, err) + + attempt := &types.Attempt{TxID: 2, Hash: testutils.NewHash()} + err = m.DeleteAttemptForUnconfirmedTx(0, attempt) + + require.Error(t, err) + }) + + t.Run("deletes attempt of unconfirmed transaction", func(t *testing.T) { + hash := testutils.NewHash() + var nonce uint64 + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + tx, err := insertUnconfirmedTransaction(m, nonce) + require.NoError(t, err) + + attempt := &types.Attempt{TxID: 0, Hash: hash} + tx.Attempts = append(tx.Attempts, attempt) + err = m.DeleteAttemptForUnconfirmedTx(nonce, attempt) + require.NoError(t, err) + + assert.Empty(t, tx.Attempts) + }) +} + +func TestFindTxWithIdempotencyKey(t *testing.T) { + t.Parallel() + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + tx, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + + ik := "IK" + tx.IdempotencyKey = &ik + itx := m.FindTxWithIdempotencyKey(ik) + assert.Equal(t, ik, *itx.IdempotencyKey) + + uik := "Unknown" + itx = m.FindTxWithIdempotencyKey(uik) + assert.Nil(t, itx) +} + +func TestPruneConfirmedTransactions(t *testing.T) { + t.Parallel() + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + total := 5 + for i := 0; i < total; i++ { + //nolint:gosec // this won't overflow + _, err := insertConfirmedTransaction(m, uint64(i)) + require.NoError(t, err) + } + prunedTxIDs := m.pruneConfirmedTransactions() + left := total - total/pruneSubset + assert.Len(t, m.ConfirmedTransactions, left) + assert.Len(t, prunedTxIDs, total/pruneSubset) +} + +func insertUnstartedTransaction(m *InMemoryStore) *types.Transaction { + m.Lock() + defer m.Unlock() + + var nonce uint64 + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: &nonce, + FromAddress: m.address, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: txmgr.TxUnstarted, + } + + m.UnstartedTransactions = append(m.UnstartedTransactions, tx) + m.Transactions[tx.ID] = tx + return tx +} + +func insertUnconfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: &nonce, + FromAddress: m.address, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: txmgr.TxUnconfirmed, + } + + if _, exists := m.UnconfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) + } + + m.UnconfirmedTransactions[nonce] = tx + m.Transactions[tx.ID] = tx + return tx, nil +} + +func insertConfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: &nonce, + FromAddress: m.address, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: txmgr.TxConfirmed, + } + + if _, exists := m.ConfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("a confirmed tx with the same nonce already exists: %v", m.ConfirmedTransactions[nonce]) + } + + m.ConfirmedTransactions[nonce] = tx + m.Transactions[tx.ID] = tx + return tx, nil +} + +func insertFataTransaction(m *InMemoryStore) *types.Transaction { + m.Lock() + defer m.Unlock() + + var nonce uint64 + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: &nonce, + FromAddress: m.address, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: txmgr.TxFatalError, + } + + m.FatalTransactions = append(m.FatalTransactions, tx) + m.Transactions[tx.ID] = tx + return tx +} diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go new file mode 100644 index 00000000000..2da4ad4cd67 --- /dev/null +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -0,0 +1,123 @@ +package txm + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type StuckTxDetectorConfig struct { + BlockTime time.Duration + StuckTxBlockThreshold uint32 + DetectionURL string + DualBroadcast bool +} + +type stuckTxDetector struct { + lggr logger.Logger + chainType chaintype.ChainType + config StuckTxDetectorConfig + lastPurgeMap map[common.Address]time.Time +} + +func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, config StuckTxDetectorConfig) *stuckTxDetector { + return &stuckTxDetector{ + lggr: lggr, + chainType: chaintype, + config: config, + lastPurgeMap: make(map[common.Address]time.Time), + } +} + +func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) { + //nolint:gocritic //placeholder for upcoming chaintypes + switch s.chainType { + default: + return s.timeBasedDetection(tx), nil + } +} + +// timeBasedDetection marks a transaction if all the following conditions are met: +// - LastBroadcastAt is not nil +// - Time since last broadcast is above the threshold +// - Time since last purge is above threshold +// +// NOTE: Potentially we can use a subset of threhsold for last purge check, because the transaction would have already been broadcasted to the mempool +// so it is more likely to be picked up compared to a transaction that hasn't been broadcasted before. This would avoid slowing down TXM for sebsequent transactions +// in case the current one is stuck. +func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { + threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) + if tx.LastBroadcastAt != nil && min(time.Since(*tx.LastBroadcastAt), time.Since(s.lastPurgeMap[tx.FromAddress])) > threshold { + s.lggr.Debugf("TxID: %v last broadcast was: %v and last purge: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", + tx.ID, tx.LastBroadcastAt, s.lastPurgeMap[tx.FromAddress], threshold) + s.lastPurgeMap[tx.FromAddress] = time.Now() + return true + } + return false +} + +type APIResponse struct { + Status string `json:"status,omitempty"` + Hash common.Hash `json:"hash,omitempty"` +} + +const ( + APIStatusPending = "PENDING" + APIStatusIncluded = "INCLUDED" + APIStatusFailed = "FAILED" + APIStatusCancelled = "CANCELLED" + APIStatusUnknown = "UNKNOWN" +) + +// Deprecated: DualBroadcastDetection doesn't provide any significant benefits in terms of speed and time +// based detection can replace it. +func (s *stuckTxDetector) DualBroadcastDetection(ctx context.Context, tx *types.Transaction) (bool, error) { + for _, attempt := range tx.Attempts { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.config.DetectionURL+attempt.Hash.String(), nil) + if err != nil { + return false, fmt.Errorf("failed to make request for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, fmt.Errorf("failed to get transaction status for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return false, fmt.Errorf("request %v failed with status: %d", req, resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return false, err + } + + var apiResponse APIResponse + err = json.Unmarshal(body, &apiResponse) + if err != nil { + return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w: %s", tx.ID, attempt.Hash, err, string(body)) + } + switch apiResponse.Status { + case APIStatusPending, APIStatusIncluded: + return false, nil + case APIStatusFailed, APIStatusCancelled: + s.lggr.Debugf("TxID: %v with attempHash: %v was marked as failed/cancelled by the RPC. Transaction is now considered stuck and will be purged.", + tx.ID, attempt.Hash) + return true, nil + case APIStatusUnknown: + continue + default: + continue + } + } + return false, nil +} diff --git a/core/chains/evm/txm/stuck_tx_detector_test.go b/core/chains/evm/txm/stuck_tx_detector_test.go new file mode 100644 index 00000000000..af5a765dcdb --- /dev/null +++ b/core/chains/evm/txm/stuck_tx_detector_test.go @@ -0,0 +1,80 @@ +package txm + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +func TestTimeBasedDetection(t *testing.T) { + t.Parallel() + + t.Run("returns false if transaction is not stuck", func(t *testing.T) { + config := StuckTxDetectorConfig{ + BlockTime: 10 * time.Second, + StuckTxBlockThreshold: 5, + } + fromAddress := testutils.NewAddress() + s := NewStuckTxDetector(logger.Test(t), "", config) + + // No previous broadcast + tx := &types.Transaction{ + ID: 1, + LastBroadcastAt: nil, + FromAddress: fromAddress, + } + assert.False(t, s.timeBasedDetection(tx)) + // Not enough time has passed since last broadcast + now := time.Now() + tx.LastBroadcastAt = &now + assert.False(t, s.timeBasedDetection(tx)) + // Not enough time has passed since last purge + tx.LastBroadcastAt = &time.Time{} + s.lastPurgeMap[fromAddress] = now + assert.False(t, s.timeBasedDetection(tx)) + }) + + t.Run("returns true if transaction is stuck", func(t *testing.T) { + config := StuckTxDetectorConfig{ + BlockTime: 10 * time.Second, + StuckTxBlockThreshold: 5, + } + fromAddress := testutils.NewAddress() + s := NewStuckTxDetector(logger.Test(t), "", config) + + tx := &types.Transaction{ + ID: 1, + LastBroadcastAt: &time.Time{}, + FromAddress: fromAddress, + } + assert.True(t, s.timeBasedDetection(tx)) + }) + + t.Run("marks first tx as stuck, updates purge time for address, and returns false for the second tx with the same broadcast time", func(t *testing.T) { + config := StuckTxDetectorConfig{ + BlockTime: 1 * time.Second, + StuckTxBlockThreshold: 10, + } + fromAddress := testutils.NewAddress() + s := NewStuckTxDetector(logger.Test(t), "", config) + + tx1 := &types.Transaction{ + ID: 1, + LastBroadcastAt: &time.Time{}, + FromAddress: fromAddress, + } + tx2 := &types.Transaction{ + ID: 2, + LastBroadcastAt: &time.Time{}, + FromAddress: fromAddress, + } + assert.True(t, s.timeBasedDetection(tx1)) + assert.False(t, s.timeBasedDetection(tx2)) + }) +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go new file mode 100644 index 00000000000..c8b4b6f1688 --- /dev/null +++ b/core/chains/evm/txm/txm.go @@ -0,0 +1,447 @@ +package txm + +import ( + "context" + "fmt" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/jpillora/backoff" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/utils" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +const ( + broadcastInterval time.Duration = 30 * time.Second + maxInFlightTransactions int = 16 + maxInFlightSubset int = 5 + maxAllowedAttempts uint16 = 10 + pendingNonceDefaultTimeout time.Duration = 30 * time.Second + pendingNonceRecheckInterval time.Duration = 1 * time.Second +) + +type Client interface { + PendingNonceAt(context.Context, common.Address) (uint64, error) + NonceAt(context.Context, common.Address, *big.Int) (uint64, error) + SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error +} + +type TxStore interface { + AbandonPendingTransactions(context.Context, common.Address) error + AppendAttemptToTransaction(context.Context, uint64, common.Address, *types.Attempt) error + CreateEmptyUnconfirmedTransaction(context.Context, common.Address, uint64, uint64) (*types.Transaction, error) + CreateTransaction(context.Context, *types.TxRequest) (*types.Transaction, error) + FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*types.Transaction, int, error) + MarkConfirmedAndReorgedTransactions(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error) + MarkUnconfirmedTransactionPurgeable(context.Context, uint64, common.Address) error + UpdateTransactionBroadcast(context.Context, uint64, uint64, common.Hash, common.Address) error + UpdateUnstartedTransactionWithNonce(context.Context, common.Address, uint64) (*types.Transaction, error) + + // ErrorHandler + DeleteAttemptForUnconfirmedTx(context.Context, uint64, *types.Attempt, common.Address) error + MarkTxFatal(context.Context, *types.Transaction, common.Address) error +} + +type AttemptBuilder interface { + NewAttempt(context.Context, logger.Logger, *types.Transaction, bool) (*types.Attempt, error) + NewBumpAttempt(context.Context, logger.Logger, *types.Transaction, types.Attempt) (*types.Attempt, error) +} + +type ErrorHandler interface { + HandleError(*types.Transaction, error, AttemptBuilder, Client, TxStore, func(common.Address, uint64), bool) (err error) +} + +type StuckTxDetector interface { + DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) +} + +type Keystore interface { + EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) +} + +type Config struct { + EIP1559 bool + BlockTime time.Duration + RetryBlockThreshold uint16 + EmptyTxLimitDefault uint64 +} + +type Txm struct { + services.StateMachine + lggr logger.SugaredLogger + chainID *big.Int + client Client + attemptBuilder AttemptBuilder + errorHandler ErrorHandler + stuckTxDetector StuckTxDetector + txStore TxStore + keystore Keystore + config Config + metrics *txmMetrics + + nonceMapMu sync.RWMutex + nonceMap map[common.Address]uint64 + + triggerCh map[common.Address]chan struct{} + stopCh services.StopChan + wg sync.WaitGroup +} + +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, stuckTxDetector StuckTxDetector, config Config, keystore Keystore) *Txm { + return &Txm{ + lggr: logger.Sugared(logger.Named(lggr, "Txm")), + keystore: keystore, + chainID: chainID, + client: client, + attemptBuilder: attemptBuilder, + txStore: txStore, + stuckTxDetector: stuckTxDetector, + config: config, + nonceMap: make(map[common.Address]uint64), + triggerCh: make(map[common.Address]chan struct{}), + } +} + +func (t *Txm) Start(ctx context.Context) error { + return t.StartOnce("Txm", func() error { + tm, err := NewTxmMetrics(t.chainID) + if err != nil { + return err + } + t.metrics = tm + t.stopCh = make(chan struct{}) + + addresses, err := t.keystore.EnabledAddressesForChain(ctx, t.chainID) + if err != nil { + return err + } + for _, address := range addresses { + t.startAddress(address) + } + return nil + }) +} + +func (t *Txm) startAddress(address common.Address) { + triggerCh := make(chan struct{}, 1) + t.triggerCh[address] = triggerCh + + t.wg.Add(2) + go t.broadcastLoop(address, triggerCh) + go t.backfillLoop(address) +} + +func (t *Txm) initializeNonce(ctx context.Context, address common.Address) { + ctxWithTimeout, cancel := context.WithTimeout(ctx, pendingNonceDefaultTimeout) + defer cancel() + for { + pendingNonce, err := t.client.PendingNonceAt(ctxWithTimeout, address) + if err != nil { + t.lggr.Errorw("Error when fetching initial nonce", "address", address, "err", err) + select { + case <-time.After(pendingNonceRecheckInterval): + case <-ctx.Done(): + t.lggr.Errorw("context error", "err", context.Cause(ctx)) + return + } + continue + } + t.setNonce(address, pendingNonce) + t.lggr.Debugf("Set initial nonce for address: %v to %d", address, pendingNonce) + return + } +} + +func (t *Txm) Close() error { + return t.StopOnce("Txm", func() error { + close(t.stopCh) + t.wg.Wait() + return nil + }) +} + +func (t *Txm) HealthReport() map[string]error { + return map[string]error{t.lggr.Name(): t.Healthy()} +} + +func (t *Txm) CreateTransaction(ctx context.Context, txRequest *types.TxRequest) (tx *types.Transaction, err error) { + tx, err = t.txStore.CreateTransaction(ctx, txRequest) + if err == nil { + t.lggr.Infow("Created transaction", "tx", tx) + } + return +} + +func (t *Txm) Trigger(address common.Address) { + if !t.IfStarted(func() { + triggerCh, exists := t.triggerCh[address] + if !exists { + return + } + triggerCh <- struct{}{} + }) { + t.lggr.Error("Txm unstarted") + } +} + +func (t *Txm) Abandon(address common.Address) error { + // TODO: restart txm + t.lggr.Infof("Dropping unstarted and unconfirmed transactions for address: %v", address) + return t.txStore.AbandonPendingTransactions(context.TODO(), address) +} + +func (t *Txm) getNonce(address common.Address) uint64 { + t.nonceMapMu.RLock() + defer t.nonceMapMu.RUnlock() + return t.nonceMap[address] +} + +func (t *Txm) setNonce(address common.Address, nonce uint64) { + t.nonceMapMu.Lock() + t.nonceMap[address] = nonce + defer t.nonceMapMu.Unlock() +} + +func newBackoff(minDuration time.Duration) backoff.Backoff { + return backoff.Backoff{ + Min: minDuration, + Max: 1 * time.Minute, + Jitter: true, + } +} + +func (t *Txm) broadcastLoop(address common.Address, triggerCh chan struct{}) { + defer t.wg.Done() + ctx, cancel := t.stopCh.NewCtx() + defer cancel() + broadcastWithBackoff := newBackoff(1 * time.Second) + var broadcastCh <-chan time.Time + + t.initializeNonce(ctx, address) + + for { + start := time.Now() + bo, err := t.broadcastTransaction(ctx, address) + if err != nil { + t.lggr.Errorw("Error during transaction broadcasting", "err", err) + } else { + t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) + } + if bo { + broadcastCh = time.After(broadcastWithBackoff.Duration()) + } else { + broadcastWithBackoff.Reset() + broadcastCh = time.After(utils.WithJitter(broadcastInterval)) + } + select { + case <-ctx.Done(): + return + case <-triggerCh: + continue + case <-broadcastCh: + continue + } + } +} + +func (t *Txm) backfillLoop(address common.Address) { + defer t.wg.Done() + ctx, cancel := t.stopCh.NewCtx() + defer cancel() + backfillWithBackoff := newBackoff(t.config.BlockTime) + backfillCh := time.After(utils.WithJitter(t.config.BlockTime)) + + for { + select { + case <-ctx.Done(): + return + case <-backfillCh: + start := time.Now() + bo, err := t.backfillTransactions(ctx, address) + if err != nil { + t.lggr.Errorw("Error during backfill", "err", err) + } else { + t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) + } + if bo { + backfillCh = time.After(backfillWithBackoff.Duration()) + } else { + backfillWithBackoff.Reset() + backfillCh = time.After(utils.WithJitter(t.config.BlockTime)) + } + } + } +} + +func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) (bool, error) { + for { + _, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, address) + if err != nil { + return false, err + } + + // Optimistically send up to maxInFlightSubset of the maxInFlightTransactions. After that threshold, broadcast more cautiously + // by checking the pending nonce so no more than maxInFlightSubset can get stuck simultaneously i.e. due + // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary + // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. + if unconfirmedCount >= maxInFlightSubset { + if unconfirmedCount > maxInFlightTransactions { + t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) + return true, nil + } + pendingNonce, e := t.client.PendingNonceAt(ctx, address) + if e != nil { + return false, e + } + nonce := t.getNonce(address) + if nonce > pendingNonce { + t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", + nonce, pendingNonce, unconfirmedCount) + return true, nil + } + } + + nonce := t.getNonce(address) + tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(ctx, address, nonce) + if err != nil { + return false, err + } + if tx == nil { + return false, nil + } + t.setNonce(address, nonce+1) + + if err := t.createAndSendAttempt(ctx, tx, address); err != nil { + return false, err + } + } +} + +func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, address common.Address) error { + attempt, err := t.attemptBuilder.NewAttempt(ctx, t.lggr, tx, t.config.EIP1559) + if err != nil { + return err + } + + if tx.Nonce == nil { + return fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + if err = t.txStore.AppendAttemptToTransaction(ctx, *tx.Nonce, address, attempt); err != nil { + return err + } + + return t.sendTransactionWithError(ctx, tx, attempt, address) +} + +func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transaction, attempt *types.Attempt, address common.Address) (err error) { + if tx.Nonce == nil { + return fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + start := time.Now() + txErr := t.client.SendTransaction(ctx, tx, attempt) + tx.AttemptCount++ + t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) + if txErr != nil && t.errorHandler != nil { + if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore, t.setNonce, false); err != nil { + return + } + } else if txErr != nil { + pendingNonce, err := t.client.PendingNonceAt(ctx, address) + if err != nil { + return err + } + if pendingNonce <= *tx.Nonce { + return fmt.Errorf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d. TxErr: %w", tx.ID, pendingNonce, *tx.Nonce, txErr) + } + } + + t.metrics.IncrementNumBroadcastedTxs(ctx) + return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, *tx.Nonce, attempt.Hash, address) +} + +func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) (bool, error) { + latestNonce, err := t.client.NonceAt(ctx, address, nil) + if err != nil { + return false, err + } + + confirmedTransactions, unconfirmedTransactionIDs, err := t.txStore.MarkConfirmedAndReorgedTransactions(ctx, latestNonce, address) + if err != nil { + return false, err + } + if len(confirmedTransactions) > 0 || len(unconfirmedTransactionIDs) > 0 { + t.metrics.IncrementNumConfirmedTxs(ctx, len(confirmedTransactions)) + confirmedTransactionIDs := t.extractMetrics(ctx, confirmedTransactions) + t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) + } + + tx, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, latestNonce, address) + if err != nil { + return false, err + } + if unconfirmedCount == 0 { + t.lggr.Debugf("All transactions confirmed for address: %v", address) + return false, err // TODO: add backoff to optimize requests + } + + if tx == nil || *tx.Nonce != latestNonce { + t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) + t.metrics.IncrementNumNonceGaps(ctx) + return false, t.createAndSendEmptyTx(ctx, latestNonce, address) + } else { //nolint:revive //easier to read + if !tx.IsPurgeable && t.stuckTxDetector != nil { + isStuck, err := t.stuckTxDetector.DetectStuckTransaction(ctx, tx) + if err != nil { + return false, err + } + if isStuck { + tx.IsPurgeable = true + err = t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, *tx.Nonce, address) + if err != nil { + return false, err + } + t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: %d", tx.ID) + return false, t.createAndSendAttempt(ctx, tx, address) + } + } + + if tx.AttemptCount >= maxAllowedAttempts { + return true, fmt.Errorf("reached max allowed attempts for txID: %d. TXM won't broadcast any more attempts."+ + "If this error persists, it means the transaction won't be confirmed and the TXM needs to be restarted."+ + "Look for any error messages from previous broadcasted attempts that may indicate why this happened, i.e. wallet is out of funds. Tx: %v", tx.ID, + tx.PrintWithAttempts()) + } + + if tx.LastBroadcastAt == nil || time.Since(*tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) { + // TODO: add optional graceful bumping strategy + t.lggr.Info("Rebroadcasting attempt for txID: ", tx.ID) + return false, t.createAndSendAttempt(ctx, tx, address) + } + } + return false, nil +} + +func (t *Txm) createAndSendEmptyTx(ctx context.Context, latestNonce uint64, address common.Address) error { + tx, err := t.txStore.CreateEmptyUnconfirmedTransaction(ctx, address, latestNonce, t.config.EmptyTxLimitDefault) + if err != nil { + return err + } + return t.createAndSendAttempt(ctx, tx, address) +} + +func (t *Txm) extractMetrics(ctx context.Context, txs []*types.Transaction) []uint64 { + confirmedTxIDs := make([]uint64, 0, len(txs)) + for _, tx := range txs { + confirmedTxIDs = append(confirmedTxIDs, tx.ID) + if tx.InitialBroadcastAt != nil { + t.metrics.RecordTimeUntilTxConfirmed(ctx, float64(time.Since(*tx.InitialBroadcastAt))) + } + } + return confirmedTxIDs +} diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go new file mode 100644 index 00000000000..af77f3ac084 --- /dev/null +++ b/core/chains/evm/txm/txm_test.go @@ -0,0 +1,327 @@ +package txm + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/mocks" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +func TestLifecycle(t *testing.T) { + t.Parallel() + + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + address1 := testutils.NewAddress() + address2 := testutils.NewAddress() + assert.NotEqual(t, address1, address2) + addresses := []common.Address{address1, address2} + keystore := mocks.NewKeystore(t) + + t.Run("retries if initial pending nonce call fails", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + config := Config{BlockTime: 1 * time.Minute} + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address1)) + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address1}, nil).Once() + txm := NewTxm(lggr, testutils.FixtureChainID, client, nil, txStore, nil, config, keystore) + client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() + client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(100), nil).Once() + require.NoError(t, txm.Start(tests.Context(t))) + tests.AssertLogEventually(t, observedLogs, "Error when fetching initial nonce") + tests.AssertLogEventually(t, observedLogs, fmt.Sprintf("Set initial nonce for address: %v to %d", address1, 100)) + }) + + t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { + config := Config{BlockTime: 200 * time.Millisecond} + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return(addresses, nil).Once() + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(addresses...)) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) + var nonce uint64 + // Start + client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() + client.On("PendingNonceAt", mock.Anything, address2).Return(nonce, nil).Once() + // backfill loop (may or may not be executed multiple times) + client.On("NonceAt", mock.Anything, address1, mock.Anything).Return(nonce, nil).Maybe() + client.On("NonceAt", mock.Anything, address2, mock.Anything).Return(nonce, nil).Maybe() + + servicetest.Run(t, txm) + tests.AssertLogEventually(t, observedLogs, "Backfill time elapsed") + }) +} + +func TestTrigger(t *testing.T) { + t.Parallel() + + address := testutils.NewAddress() + keystore := mocks.NewKeystore(t) + t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.ErrorLevel) + txm := NewTxm(lggr, nil, nil, nil, nil, nil, Config{}, keystore) + txm.Trigger(address) + tests.AssertLogEventually(t, observedLogs, "Txm unstarted") + }) + + t.Run("executes Trigger", func(t *testing.T) { + lggr := logger.Test(t) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address)) + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) + var nonce uint64 + // Start + client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Maybe() + servicetest.Run(t, txm) + txm.Trigger(address) + }) +} + +func TestBroadcastTransaction(t *testing.T) { + t.Parallel() + + ctx := tests.Context(t) + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + config := Config{} + address := testutils.NewAddress() + keystore := mocks.NewKeystore(t) + + t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { + mTxStore := mocks.NewTxStore(t) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) + bo, err := txm.broadcastTransaction(ctx, address) + require.Error(t, err) + assert.False(t, bo) + require.ErrorContains(t, err, "call failed") + }) + + t.Run("throws a warning and returns if unconfirmed transactions exceed maxInFlightTransactions", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + mTxStore := mocks.NewTxStore(t) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions+1, nil).Once() + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) + bo, err := txm.broadcastTransaction(ctx, address) + assert.True(t, bo) + require.NoError(t, err) + tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") + }) + + t.Run("checks pending nonce if unconfirmed transactions are equal or more than maxInFlightSubset", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + mTxStore := mocks.NewTxStore(t) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) + txm.setNonce(address, 1) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightSubset, nil).Twice() + + client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), nil).Once() // LocalNonce: 1, PendingNonce: 0 + bo, err := txm.broadcastTransaction(ctx, address) + assert.True(t, bo) + require.NoError(t, err) + + client.On("PendingNonceAt", mock.Anything, address).Return(uint64(1), nil).Once() // LocalNonce: 1, PendingNonce: 1 + mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() + bo, err = txm.broadcastTransaction(ctx, address) + assert.False(t, bo) + require.NoError(t, err) + tests.AssertLogCountEventually(t, observedLogs, "Reached transaction limit.", 1) + }) + + t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { + mTxStore := mocks.NewTxStore(t) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) + mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() + bo, err := txm.broadcastTransaction(ctx, address) + assert.False(t, bo) + require.Error(t, err) + require.ErrorContains(t, err, "call failed") + }) + + t.Run("returns if there are no unstarted transactions", func(t *testing.T) { + lggr := logger.Test(t) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address)) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) + bo, err := txm.broadcastTransaction(ctx, address) + require.NoError(t, err) + assert.False(t, bo) + assert.Equal(t, uint64(0), txm.getNonce(address)) + }) + + t.Run("picks a new tx and creates a new attempt then sends it and updates the broadcast time", func(t *testing.T) { + lggr := logger.Test(t) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address)) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) + txm.setNonce(address, 8) + metrics, err := NewTxmMetrics(testutils.FixtureChainID) + require.NoError(t, err) + txm.metrics = metrics + IDK := "IDK" + txRequest := &types.TxRequest{ + Data: []byte{100, 200}, + IdempotencyKey: &IDK, + ChainID: testutils.FixtureChainID, + FromAddress: address, + ToAddress: testutils.NewAddress(), + SpecifiedGasLimit: 22000, + } + tx, err := txm.CreateTransaction(tests.Context(t), txRequest) + require.NoError(t, err) + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{GasPrice: assets.NewWeiI(1)}, + GasLimit: 22000, + } + ab.On("NewAttempt", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(attempt, nil).Once() + client.On("SendTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + + bo, err := txm.broadcastTransaction(ctx, address) + require.NoError(t, err) + assert.False(t, bo) + assert.Equal(t, uint64(9), txm.getNonce(address)) + tx, err = txStore.FindTxWithIdempotencyKey(tests.Context(t), IDK) + require.NoError(t, err) + assert.Len(t, tx.Attempts, 1) + var zeroTime time.Time + assert.Greater(t, *tx.LastBroadcastAt, zeroTime) + assert.Greater(t, *tx.Attempts[0].BroadcastAt, zeroTime) + assert.Greater(t, *tx.InitialBroadcastAt, zeroTime) + }) +} + +func TestBackfillTransactions(t *testing.T) { + t.Parallel() + + ctx := tests.Context(t) + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + txStore := mocks.NewTxStore(t) + config := Config{} + address := testutils.NewAddress() + keystore := mocks.NewKeystore(t) + + t.Run("fails if latest nonce fetching fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() + bo, err := txm.backfillTransactions(ctx, address) + require.Error(t, err) + assert.False(t, bo) + require.ErrorContains(t, err, "latest nonce fail") + }) + + t.Run("fails if MarkConfirmedAndReorgedTransactions fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() + txStore.On("MarkConfirmedAndReorgedTransactions", mock.Anything, mock.Anything, address). + Return([]*types.Transaction{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() + bo, err := txm.backfillTransactions(ctx, address) + require.Error(t, err) + assert.False(t, bo) + require.ErrorContains(t, err, "marking transactions confirmed failed") + }) + + t.Run("fills nonce gap", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address)) + ab := mocks.NewAttemptBuilder(t) + c := Config{EIP1559: false, BlockTime: 10 * time.Minute, RetryBlockThreshold: 10, EmptyTxLimitDefault: 22000} + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, c, keystore) + emptyMetrics, err := NewTxmMetrics(testutils.FixtureChainID) + require.NoError(t, err) + txm.metrics = emptyMetrics + + // Add a new transaction that will be assigned with nonce = 1. Nonce = 0 is not being tracked by the txStore. This will trigger a nonce gap. + txRequest := &types.TxRequest{ + ChainID: testutils.FixtureChainID, + FromAddress: address, + ToAddress: testutils.NewAddress(), + } + _, err = txm.CreateTransaction(tests.Context(t), txRequest) + require.NoError(t, err) + _, err = txStore.UpdateUnstartedTransactionWithNonce(tests.Context(t), address, 1) // Create nonce gap + require.NoError(t, err) + + // During backfill we observe nonce has changed. The transaction with nonce = 1 should be marked unconfirmed. + // For nonce = 0 there are no transactions stored in txStore, which results in a nonce gap. + // TXM creates a new empty transaction and fills the gap. + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() + attempt := &types.Attempt{ + TxID: 1, + Fee: gas.EvmFee{GasPrice: assets.NewWeiI(1)}, + GasLimit: 22000, + } + ab.On("NewAttempt", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(attempt, nil).Once() + client.On("SendTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + bo, err := txm.backfillTransactions(ctx, address) + require.NoError(t, err) + assert.False(t, bo) + tests.AssertLogEventually(t, observedLogs, fmt.Sprintf("Nonce gap at nonce: %d - address: %v. Creating a new transaction", 0, address)) + _, count, err := txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, address) + require.NoError(t, err) + assert.Equal(t, 2, count) + }) + + t.Run("retries attempt after threshold", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address)) + ab := mocks.NewAttemptBuilder(t) + c := Config{EIP1559: false, BlockTime: 1 * time.Second, RetryBlockThreshold: 1, EmptyTxLimitDefault: 22000} + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, c, keystore) + emptyMetrics, err := NewTxmMetrics(testutils.FixtureChainID) + require.NoError(t, err) + txm.metrics = emptyMetrics + + IDK := "IDK" + txRequest := &types.TxRequest{ + Data: []byte{100, 200}, + IdempotencyKey: &IDK, + ChainID: testutils.FixtureChainID, + FromAddress: address, + ToAddress: testutils.NewAddress(), + SpecifiedGasLimit: 22000, + } + tx, err := txm.CreateTransaction(tests.Context(t), txRequest) + require.NoError(t, err) + _, err = txStore.UpdateUnstartedTransactionWithNonce(tests.Context(t), address, 0) + require.NoError(t, err) + + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{GasPrice: assets.NewWeiI(1)}, + GasLimit: 22000, + } + ab.On("NewAttempt", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(attempt, nil).Once() + + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() + client.On("SendTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + _, err = txm.backfillTransactions(tests.Context(t), address) + require.NoError(t, err) + tests.AssertLogEventually(t, observedLogs, fmt.Sprintf("Rebroadcasting attempt for txID: %d", attempt.TxID)) + }) +} diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go new file mode 100644 index 00000000000..1d1104545d4 --- /dev/null +++ b/core/chains/evm/txm/types/transaction.go @@ -0,0 +1,192 @@ +package types + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "time" + + "github.com/google/uuid" + "gopkg.in/guregu/null.v4" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" + clnull "github.com/smartcontractkit/chainlink-common/pkg/utils/null" + + commontypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" +) + +type Transaction struct { + ID uint64 + IdempotencyKey *string + ChainID *big.Int + Nonce *uint64 + FromAddress common.Address + ToAddress common.Address + Value *big.Int + Data []byte + SpecifiedGasLimit uint64 + + CreatedAt time.Time + InitialBroadcastAt *time.Time + LastBroadcastAt *time.Time + + State commontypes.TxState + IsPurgeable bool + Attempts []*Attempt + AttemptCount uint16 // AttempCount is strictly kept in memory and prevents indefinite retrying + Meta *sqlutil.JSON + Subject uuid.NullUUID + + // Pipeline variables - if you aren't calling this from chain tx task within + // the pipeline, you don't need these variables + PipelineTaskRunID uuid.NullUUID + MinConfirmations clnull.Uint32 + SignalCallback bool + CallbackCompleted bool +} + +func (t *Transaction) String() string { + return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%s, FromAddress:%v, ToAddress:%v, Value:%v, `+ + `Data:%s, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ + `Meta:%v, Subject:%v}`, + t.ID, stringOrNull(t.IdempotencyKey), t.ChainID, stringOrNull(t.Nonce), t.FromAddress, t.ToAddress, t.Value, + base64.StdEncoding.EncodeToString(t.Data), t.SpecifiedGasLimit, t.CreatedAt, stringOrNull(t.InitialBroadcastAt), stringOrNull(t.LastBroadcastAt), + t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject) +} + +func stringOrNull[T any](t *T) string { + if t != nil { + return fmt.Sprintf("%v", *t) + } + return "null" +} + +func (t *Transaction) PrintWithAttempts() string { + attempts := " Attempts: [" + for _, a := range t.Attempts { + attempts += a.String() + ", " + } + attempts += "]" + + return t.String() + attempts +} + +func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, error) { + for _, a := range t.Attempts { + if a.Hash == attemptHash { + return a, nil + } + } + return nil, fmt.Errorf("attempt with hash: %v was not found", attemptHash) +} + +func (t *Transaction) DeepCopy() *Transaction { + txCopy := *t + attemptsCopy := make([]*Attempt, 0, len(t.Attempts)) + for _, attempt := range t.Attempts { + attemptsCopy = append(attemptsCopy, attempt.DeepCopy()) + } + txCopy.Attempts = attemptsCopy + return &txCopy +} + +func (t *Transaction) GetMeta() (*TxMeta, error) { + if t.Meta == nil { + return nil, nil + } + var m TxMeta + if err := json.Unmarshal(*t.Meta, &m); err != nil { + return nil, fmt.Errorf("unmarshalling meta: %w", err) + } + return &m, nil +} + +type Attempt struct { + ID uint64 + TxID uint64 + Hash common.Hash + Fee gas.EvmFee + GasLimit uint64 + Type byte + SignedTransaction *types.Transaction + + CreatedAt time.Time + BroadcastAt *time.Time +} + +func (a *Attempt) DeepCopy() *Attempt { + txCopy := *a + if a.SignedTransaction != nil { + txCopy.SignedTransaction = a.SignedTransaction.WithoutBlobTxSidecar() + } + return &txCopy +} + +func (a *Attempt) String() string { + return fmt.Sprintf(`{ID:%d, TxID:%d, Hash:%v, Fee:%v, GasLimit:%d, Type:%v, CreatedAt:%v, BroadcastAt:%v}`, + a.ID, a.TxID, a.Hash, a.Fee, a.GasLimit, a.Type, a.CreatedAt, stringOrNull(a.BroadcastAt)) +} + +type TxRequest struct { + IdempotencyKey *string + ChainID *big.Int + FromAddress common.Address + ToAddress common.Address + Value *big.Int + Data []byte + SpecifiedGasLimit uint64 + + Meta *sqlutil.JSON // TODO: *TxMeta after migration + ForwarderAddress common.Address + + // Pipeline variables - if you aren't calling this from chain tx task within + // the pipeline, you don't need these variables + PipelineTaskRunID uuid.NullUUID + MinConfirmations clnull.Uint32 + SignalCallback bool +} + +type TxMeta struct { + // Pipeline + JobID *int32 `json:"JobID,omitempty"` + FailOnRevert null.Bool `json:"FailOnRevert,omitempty"` + + // VRF + RequestID *common.Hash `json:"RequestID,omitempty"` + RequestTxHash *common.Hash `json:"RequestTxHash,omitempty"` + RequestIDs []common.Hash `json:"RequestIDs,omitempty"` + RequestTxHashes []common.Hash `json:"RequestTxHashes,omitempty"` + MaxLink *string `json:"MaxLink,omitempty"` + SubID *uint64 `json:"SubId,omitempty"` + GlobalSubID *string `json:"GlobalSubId,omitempty"` + MaxEth *string `json:"MaxEth,omitempty"` + ForceFulfilled *bool `json:"ForceFulfilled,omitempty"` + ForceFulfillmentAttempt *uint64 `json:"ForceFulfillmentAttempt,omitempty"` + + // Used for keepers + UpkeepID *string `json:"UpkeepID,omitempty"` + + // Used for Keystone Workflows + WorkflowExecutionID *string `json:"WorkflowExecutionID,omitempty"` + + // Forwarders + FwdrDestAddress *common.Address `json:"ForwarderDestAddress,omitempty"` + + // CCIP + MessageIDs []string `json:"MessageIDs,omitempty"` + SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` + + // Dual Broadcast + DualBroadcast *bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams *string `json:"DualBroadcastParams,omitempty"` +} + +type QueueingTxStrategy struct { + QueueSize uint32 + Subject uuid.NullUUID +} diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 73c5614aba3..4a09d16d214 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -18,6 +18,9 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/clientwrappers" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) @@ -91,6 +94,56 @@ func NewEvmTxm( return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError) } +func NewTxmV2( + ds sqlutil.DataSource, + chainConfig ChainConfig, + fCfg FeeConfig, + txConfig config.Transactions, + txmV2Config config.TransactionManagerV2, + client client.Client, + lggr logger.Logger, + logPoller logpoller.LogPoller, + keyStore keystore.Eth, + estimator gas.EvmFeeEstimator, +) (TxManager, error) { + var fwdMgr *forwarders.FwdMgr + if txConfig.ForwardersEnabled() { + fwdMgr = forwarders.NewFwdMgr(ds, client, logPoller, lggr, chainConfig) + } else { + lggr.Info("ForwarderManager: Disabled") + } + + chainID := client.ConfiguredChainID() + + var stuckTxDetector txm.StuckTxDetector + if txConfig.AutoPurge().Enabled() { + stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ + BlockTime: *txmV2Config.BlockTime(), + StuckTxBlockThreshold: *txConfig.AutoPurge().Threshold(), + DetectionURL: txConfig.AutoPurge().DetectionApiUrl().String(), + } + stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) + } + + attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMaxKey, estimator, keyStore) + inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID) + config := txm.Config{ + EIP1559: fCfg.EIP1559DynamicFees(), + BlockTime: *txmV2Config.BlockTime(), + //nolint:gosec // reuse existing config until migration + RetryBlockThreshold: uint16(fCfg.BumpThreshold()), + EmptyTxLimitDefault: fCfg.LimitDefault(), + } + var c txm.Client + if txmV2Config.DualBroadcast() != nil && *txmV2Config.DualBroadcast() { + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomURL()) + } else { + c = clientwrappers.NewChainClient(client) + } + t := txm.NewTxm(lggr, chainID, c, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, keyStore) + return txm.NewTxmOrchestrator(lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil +} + // NewEvmResender creates a new concrete EvmResender func NewEvmResender( lggr logger.Logger, diff --git a/core/chains/legacyevm/evm_txm.go b/core/chains/legacyevm/evm_txm.go index 17dbce79e84..15731790f67 100644 --- a/core/chains/legacyevm/evm_txm.go +++ b/core/chains/legacyevm/evm_txm.go @@ -40,20 +40,35 @@ func newEvmTxm( ) if opts.GenTxManager == nil { - txm, err = txmgr.NewTxm( - ds, - cfg, - txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), - cfg.Transactions(), - cfg.NodePool().Errors(), - databaseConfig, - listenerConfig, - client, - lggr, - logPoller, - opts.KeyStore, - estimator, - headTracker) + if cfg.Transactions().TransactionManagerV2().Enabled() { + txm, err = txmgr.NewTxmV2( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.Transactions().TransactionManagerV2(), + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + ) + } else { + txm, err = txmgr.NewTxm( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.NodePool().Errors(), + databaseConfig, + listenerConfig, + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + headTracker) + } } else { txm = opts.GenTxManager(chainID) } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index e8adb3d611c..bdcb0e8c5d4 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -141,6 +141,16 @@ Threshold = 5 # Example # MinAttempts configures the minimum number of broadcasted attempts a transaction has to have before it is evaluated further for being terminally stuck. This threshold is only applied if there is no custom API to identify stuck transactions provided by the chain. Ensure the gas estimator configs take more bump attempts before reaching the configured max gas price. MinAttempts = 3 # Example +[EVM.Transactions.TransactionManagerV2] +# Enabled enables TransactionManagerV2. +Enabled = false # Default +# BlockTime controls the frequency of the backfill loop of TransactionManagerV2. +BlockTime = '10s' # Example +# CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomURL = 'https://example.api.io' # Example +# DualBroadcast enables DualBroadcast functionality. +DualBroadcast = false # Example + [EVM.BalanceMonitor] # Enabled balance monitoring for all keys. Enabled = true # Default diff --git a/core/config/docs/docs_test.go b/core/config/docs/docs_test.go index 9fca08ee99b..bc171caf8c1 100644 --- a/core/config/docs/docs_test.go +++ b/core/config/docs/docs_test.go @@ -97,6 +97,11 @@ func TestDoc(t *testing.T) { docDefaults.Transactions.AutoPurge.Threshold = nil docDefaults.Transactions.AutoPurge.MinAttempts = nil + // TransactionManagerV2 configs are only set if the feature is enabled + docDefaults.Transactions.TransactionManagerV2.BlockTime = nil + docDefaults.Transactions.TransactionManagerV2.CustomURL = nil + docDefaults.Transactions.TransactionManagerV2.DualBroadcast = nil + // Fallback DA oracle is not set docDefaults.GasEstimator.DAOracle = evmcfg.DAOracle{} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 0e64fa0444e..764830409fc 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -664,6 +664,9 @@ func TestConfig_Marshal(t *testing.T) { AutoPurge: evmcfg.AutoPurgeConfig{ Enabled: ptr(false), }, + TransactionManagerV2: evmcfg.TransactionManagerV2Config{ + Enabled: ptr(false), + }, }, HeadTracker: evmcfg.HeadTracker{ @@ -1132,6 +1135,9 @@ ResendAfterThreshold = '1h0m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true @@ -1408,6 +1414,15 @@ func TestConfig_full(t *testing.T) { got.EVM[c].Nodes[n].Order = ptr(int32(100)) } } + if got.EVM[c].Transactions.TransactionManagerV2.BlockTime == nil { + got.EVM[c].Transactions.TransactionManagerV2.BlockTime = new(commoncfg.Duration) + } + if got.EVM[c].Transactions.TransactionManagerV2.CustomURL == nil { + got.EVM[c].Transactions.TransactionManagerV2.CustomURL = new(commoncfg.URL) + } + if got.EVM[c].Transactions.TransactionManagerV2.DualBroadcast == nil { + got.EVM[c].Transactions.TransactionManagerV2.DualBroadcast = ptr(false) + } if got.EVM[c].Transactions.AutoPurge.Threshold == nil { got.EVM[c].Transactions.AutoPurge.Threshold = ptr(uint32(0)) } diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index a3fcea9bf73..b57f9a76cf2 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -349,6 +349,9 @@ ResendAfterThreshold = '1h0m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/core/services/chainlink/testdata/config-invalid.toml b/core/services/chainlink/testdata/config-invalid.toml index 347530cec53..2a3506da45b 100644 --- a/core/services/chainlink/testdata/config-invalid.toml +++ b/core/services/chainlink/testdata/config-invalid.toml @@ -58,6 +58,9 @@ FinalizedBlockOffset = 64 [EVM.Transactions.AutoPurge] Enabled = true +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.GasEstimator] Mode = 'FixedPrice' BumpThreshold = 0 @@ -112,6 +115,9 @@ ChainType = 'scroll' Enabled = true DetectionApiUrl = '' +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [[EVM.Nodes]] Name = 'scroll node' WSURL = 'ws://foo.bar' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index a659223134a..dbaafbe67d1 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -332,6 +332,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true @@ -443,6 +446,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true @@ -548,6 +554,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 4672d176915..d2d4b90e3ca 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -349,6 +349,9 @@ ResendAfterThreshold = '1h0m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 35974ea1ac8..82b82216371 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -332,6 +332,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true @@ -443,6 +446,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true @@ -548,6 +554,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 18a44e30fa5..e90b37d09c7 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2056,6 +2056,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2161,6 +2164,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2266,6 +2272,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2371,6 +2380,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2477,6 +2489,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2586,6 +2601,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2691,6 +2709,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2797,6 +2818,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -2902,6 +2926,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3006,6 +3033,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3110,6 +3140,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3215,6 +3248,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3321,6 +3357,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3426,6 +3465,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3531,6 +3573,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3635,6 +3680,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3741,6 +3789,9 @@ ResendAfterThreshold = '3m0s' Enabled = true MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3847,6 +3898,9 @@ ResendAfterThreshold = '3m0s' Enabled = true MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -3952,6 +4006,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4061,6 +4118,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4166,6 +4226,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4275,6 +4338,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4383,6 +4449,9 @@ ResendAfterThreshold = '2m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4488,6 +4557,9 @@ ResendAfterThreshold = '2m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4593,6 +4665,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4701,6 +4776,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4810,6 +4888,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -4919,6 +5000,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5028,6 +5112,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5132,6 +5219,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5237,6 +5327,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5343,6 +5436,9 @@ ResendAfterThreshold = '3m0s' Enabled = true MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5448,6 +5544,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5553,6 +5652,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5658,6 +5760,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5767,6 +5872,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5875,6 +5983,9 @@ ResendAfterThreshold = '0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -5981,6 +6092,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6091,6 +6205,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6200,6 +6317,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6305,6 +6425,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6410,6 +6533,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6520,6 +6646,9 @@ ResendAfterThreshold = '3m0s' Enabled = true MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6624,6 +6753,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6729,6 +6861,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6834,6 +6969,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -6943,6 +7081,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7053,6 +7194,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7162,6 +7306,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7267,6 +7414,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7376,6 +7526,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7482,6 +7635,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7591,6 +7747,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7700,6 +7859,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7808,6 +7970,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -7913,6 +8078,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8018,6 +8186,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8123,6 +8294,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8231,6 +8405,9 @@ Enabled = true Threshold = 90 MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8343,6 +8520,9 @@ Enabled = true Threshold = 90 MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8451,6 +8631,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8556,6 +8739,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8664,6 +8850,9 @@ ResendAfterThreshold = '3m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8770,6 +8959,9 @@ Enabled = true Threshold = 50 MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8876,6 +9068,9 @@ Enabled = true Threshold = 50 MinAttempts = 3 +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -8981,6 +9176,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9086,6 +9284,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9195,6 +9396,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9299,6 +9503,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9403,6 +9610,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9508,6 +9718,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9617,6 +9830,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9727,6 +9943,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9836,6 +10055,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -9944,6 +10166,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10053,6 +10278,9 @@ ResendAfterThreshold = '1m0s' Enabled = true DetectionApiUrl = 'https://sepolia-venus.scroll.io' +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10163,6 +10391,9 @@ ResendAfterThreshold = '1m0s' Enabled = true DetectionApiUrl = 'https://venus.scroll.io' +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10272,6 +10503,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10381,6 +10615,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10490,6 +10727,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10595,6 +10835,9 @@ ResendAfterThreshold = '30s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10704,6 +10947,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -10809,6 +11055,9 @@ ResendAfterThreshold = '1m0s' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -11180,6 +11429,40 @@ MinAttempts = 3 # Example ``` MinAttempts configures the minimum number of broadcasted attempts a transaction has to have before it is evaluated further for being terminally stuck. This threshold is only applied if there is no custom API to identify stuck transactions provided by the chain. Ensure the gas estimator configs take more bump attempts before reaching the configured max gas price. +## EVM.Transactions.TransactionManagerV2 +```toml +[EVM.Transactions.TransactionManagerV2] +Enabled = false # Default +BlockTime = '10s' # Example +CustomURL = 'https://example.api.io' # Example +DualBroadcast = false # Example +``` + + +### Enabled +```toml +Enabled = false # Default +``` +Enabled enables TransactionManagerV2. + +### BlockTime +```toml +BlockTime = '10s' # Example +``` +BlockTime controls the frequency of the backfill loop of TransactionManagerV2. + +### CustomURL +```toml +CustomURL = 'https://example.api.io' # Example +``` +CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. + +### DualBroadcast +```toml +DualBroadcast = false # Example +``` +DualBroadcast enables DualBroadcast functionality. + ## EVM.BalanceMonitor ```toml [EVM.BalanceMonitor] diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 44f24dda47f..2acde4e13b6 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -405,6 +405,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 79fee5634d8..d061e4468ec 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -388,6 +388,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 2a64da5c274..b5609655977 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -388,6 +388,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index b6450111b71..217a78a7c3c 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -388,6 +388,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 58580d8203a..5cc001850d6 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -40,6 +40,9 @@ ResendAfterThreshold = '1m' [Transactions.AutoPurge] Enabled = false +[Transactions.TransactionManagerV2] +Enabled = false + [BalanceMonitor] Enabled = true @@ -479,6 +482,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 2247cf66e87..4d42750c96d 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -378,6 +378,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index f431e853454..eef870f2280 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -385,6 +385,9 @@ ResendAfterThreshold = '1m0s' [EVM.Transactions.AutoPurge] Enabled = false +[EVM.Transactions.TransactionManagerV2] +Enabled = false + [EVM.BalanceMonitor] Enabled = true