From 10e27f716a1e9804ab3b24dbaad3cc1162261fa8 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 11 Oct 2024 12:52:45 +0300 Subject: [PATCH 01/73] TXMv2 alpha version --- .mockery.yaml | 5 + core/chains/evm/txm/attempt_builder.go | 144 ++++ core/chains/evm/txm/dummy_keystore.go | 32 + core/chains/evm/txm/mocks/attempt_builder.go | 161 ++++ core/chains/evm/txm/mocks/client.go | 252 +++++++ core/chains/evm/txm/mocks/storage.go | 699 ++++++++++++++++++ core/chains/evm/txm/orchestrator.go | 38 + core/chains/evm/txm/storage/inmemory_store.go | 238 ++++++ .../evm/txm/storage/inmemory_store_test.go | 426 +++++++++++ core/chains/evm/txm/txm.go | 351 +++++++++ core/chains/evm/txm/txm_test.go | 162 ++++ core/chains/evm/txm/types/transaction.go | 84 +++ 12 files changed, 2592 insertions(+) create mode 100644 core/chains/evm/txm/attempt_builder.go create mode 100644 core/chains/evm/txm/dummy_keystore.go create mode 100644 core/chains/evm/txm/mocks/attempt_builder.go create mode 100644 core/chains/evm/txm/mocks/client.go create mode 100644 core/chains/evm/txm/mocks/storage.go create mode 100644 core/chains/evm/txm/orchestrator.go create mode 100644 core/chains/evm/txm/storage/inmemory_store.go create mode 100644 core/chains/evm/txm/storage/inmemory_store_test.go create mode 100644 core/chains/evm/txm/txm.go create mode 100644 core/chains/evm/txm/txm_test.go create mode 100644 core/chains/evm/txm/types/transaction.go diff --git a/.mockery.yaml b/.mockery.yaml index 709134b05bd..ad452a9d9cd 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -95,6 +95,11 @@ packages: BalanceMonitor: config: dir: "{{ .InterfaceDir }}/../mocks" + github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm: + interfaces: + Client: + Storage: + AttemptBuilder: github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr: interfaces: ChainConfig: diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go new file mode 100644 index 00000000000..de5848e71a7 --- /dev/null +++ b/core/chains/evm/txm/attempt_builder.go @@ -0,0 +1,144 @@ +package txm + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + evmtypes "github.com/ethereum/go-ethereum/core/types" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type Keystore interface { + SignTx(ctx context.Context, fromAddress common.Address, tx *evmtypes.Transaction, chainID *big.Int) (*evmtypes.Transaction, error) +} + +type attemptBuilder struct { + chainID *big.Int + priceMax *assets.Wei // TODO: PriceMax per key level + estimator gas.EvmFeeEstimator + keystore Keystore +} + +func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore Keystore) *attemptBuilder { + return &attemptBuilder{ + chainID: chainID, + estimator: estimator, + keystore: keystore, + } +} + +func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { + fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) + if err != nil { + return nil, err + } + txType := evmtypes.LegacyTxType + if dynamic { + txType = evmtypes.DynamicFeeTxType + } + return a.newCustomAttempt(ctx, tx, fee, estimatedGasLimit, byte(txType), lggr) +} + +func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { + bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) + if err != nil { + return nil, err + } + return a.newCustomAttempt(ctx, tx, bumpedFee, bumpedFeeLimit, previousAttempt.Type, lggr) +} + +func (a *attemptBuilder) newCustomAttempt( + ctx context.Context, + tx *types.Transaction, + fee gas.EvmFee, + estimatedGasLimit uint64, + txType byte, + lggr logger.Logger, +) (attempt *types.Attempt, err error) { + switch txType { + case 0x0: + if fee.GasPrice == nil { + err = fmt.Errorf("attemptID: %v of txID: %v, is a type 0 transaction but estimator did not return legacy fee", tx.ID, attempt.ID) + logger.Sugared(lggr).AssumptionViolation(err.Error()) + return + } + return a.newLegacyAttempt(ctx, tx, fee.GasPrice, estimatedGasLimit) + case 0x2: + if !fee.ValidDynamic() { + err = fmt.Errorf("attemptID %v of txID: %v, is a type 2 transaction but estimator did not return dynamic fee", tx.ID, attempt.ID) + logger.Sugared(lggr).AssumptionViolation(err.Error()) + return + } + return a.newDynamicFeeAttempt(ctx, tx, fee.DynamicFee, estimatedGasLimit) + default: + return nil, fmt.Errorf("cannot build attempt, unrecognized transaction type: %v", txType) + } +} + +func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transaction, gasPrice *assets.Wei, estimatedGasLimit uint64) (*types.Attempt, error) { + var data []byte + if !tx.IsPurgeable { + data = tx.Data + } + legacyTx := evmtypes.LegacyTx{ + Nonce: tx.Nonce, + To: &tx.ToAddress, + Value: tx.Value, + Gas: estimatedGasLimit, + GasPrice: gasPrice.ToInt(), + Data: data, + } + + signedTx, err := a.keystore.SignTx(ctx, tx.FromAddress, evmtypes.NewTx(&legacyTx), a.chainID) + if err != nil { + return nil, fmt.Errorf("failed to sign attempt for txID: %v, err: %w", tx.ID, err) + } + + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{GasPrice: gasPrice}, + Hash: signedTx.Hash(), + GasLimit: estimatedGasLimit, + SignedTransaction: signedTx, + } + + return attempt, nil +} + +func (a *attemptBuilder) newDynamicFeeAttempt(ctx context.Context, tx *types.Transaction, dynamicFee gas.DynamicFee, estimatedGasLimit uint64) (*types.Attempt, error) { + var data []byte + if !tx.IsPurgeable { + data = tx.Data + } + dynamicTx := evmtypes.DynamicFeeTx{ + Nonce: tx.Nonce, + To: &tx.ToAddress, + Value: tx.Value, + Gas: estimatedGasLimit, + GasFeeCap: dynamicFee.GasFeeCap.ToInt(), + GasTipCap: dynamicFee.GasTipCap.ToInt(), + Data: data, + } + + signedTx, err := a.keystore.SignTx(ctx, tx.FromAddress, evmtypes.NewTx(&dynamicTx), a.chainID) + if err != nil { + return nil, fmt.Errorf("failed to sign attempt for txID: %v, err: %w", tx.ID, err) + } + + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{DynamicFee: gas.DynamicFee{GasFeeCap: dynamicFee.GasFeeCap, GasTipCap: dynamicFee.GasTipCap}}, + Hash: signedTx.Hash(), + GasLimit: estimatedGasLimit, + SignedTransaction: signedTx, + } + + return attempt, nil +} diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go new file mode 100644 index 00000000000..795d4fa186c --- /dev/null +++ b/core/chains/evm/txm/dummy_keystore.go @@ -0,0 +1,32 @@ +package txm + +import ( + "context" + "crypto/ecdsa" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +type DummyKeystore struct { + privateKey *ecdsa.PrivateKey +} + +func NewKeystore(privateKeyString string) *DummyKeystore { + return &DummyKeystore{} +} + +func (k *DummyKeystore) Add(privateKeyString string) error { + privateKey, err := crypto.HexToECDSA(privateKeyString) + if err != nil { + return err + } + k.privateKey = privateKey + return nil +} + +func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + return types.SignTx(tx, types.NewEIP155Signer(chainID), k.privateKey) +} diff --git a/core/chains/evm/txm/mocks/attempt_builder.go b/core/chains/evm/txm/mocks/attempt_builder.go new file mode 100644 index 00000000000..ef02397cf6b --- /dev/null +++ b/core/chains/evm/txm/mocks/attempt_builder.go @@ -0,0 +1,161 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + context "context" + + logger "github.com/smartcontractkit/chainlink-common/pkg/logger" + mock "github.com/stretchr/testify/mock" + + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +// AttemptBuilder is an autogenerated mock type for the AttemptBuilder type +type AttemptBuilder struct { + mock.Mock +} + +type AttemptBuilder_Expecter struct { + mock *mock.Mock +} + +func (_m *AttemptBuilder) EXPECT() *AttemptBuilder_Expecter { + return &AttemptBuilder_Expecter{mock: &_m.Mock} +} + +// NewAttempt provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *AttemptBuilder) NewAttempt(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 bool) (*types.Attempt, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for NewAttempt") + } + + var r0 *types.Attempt + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, bool) (*types.Attempt, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, bool) *types.Attempt); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Attempt) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, logger.Logger, *types.Transaction, bool) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AttemptBuilder_NewAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewAttempt' +type AttemptBuilder_NewAttempt_Call struct { + *mock.Call +} + +// NewAttempt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 logger.Logger +// - _a2 *types.Transaction +// - _a3 bool +func (_e *AttemptBuilder_Expecter) NewAttempt(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *AttemptBuilder_NewAttempt_Call { + return &AttemptBuilder_NewAttempt_Call{Call: _e.mock.On("NewAttempt", _a0, _a1, _a2, _a3)} +} + +func (_c *AttemptBuilder_NewAttempt_Call) Run(run func(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 bool)) *AttemptBuilder_NewAttempt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(logger.Logger), args[2].(*types.Transaction), args[3].(bool)) + }) + return _c +} + +func (_c *AttemptBuilder_NewAttempt_Call) Return(_a0 *types.Attempt, _a1 error) *AttemptBuilder_NewAttempt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AttemptBuilder_NewAttempt_Call) RunAndReturn(run func(context.Context, logger.Logger, *types.Transaction, bool) (*types.Attempt, error)) *AttemptBuilder_NewAttempt_Call { + _c.Call.Return(run) + return _c +} + +// NewBumpAttempt provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *AttemptBuilder) NewBumpAttempt(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 types.Attempt) (*types.Attempt, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for NewBumpAttempt") + } + + var r0 *types.Attempt + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, types.Attempt) (*types.Attempt, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, logger.Logger, *types.Transaction, types.Attempt) *types.Attempt); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Attempt) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, logger.Logger, *types.Transaction, types.Attempt) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AttemptBuilder_NewBumpAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewBumpAttempt' +type AttemptBuilder_NewBumpAttempt_Call struct { + *mock.Call +} + +// NewBumpAttempt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 logger.Logger +// - _a2 *types.Transaction +// - _a3 types.Attempt +func (_e *AttemptBuilder_Expecter) NewBumpAttempt(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *AttemptBuilder_NewBumpAttempt_Call { + return &AttemptBuilder_NewBumpAttempt_Call{Call: _e.mock.On("NewBumpAttempt", _a0, _a1, _a2, _a3)} +} + +func (_c *AttemptBuilder_NewBumpAttempt_Call) Run(run func(_a0 context.Context, _a1 logger.Logger, _a2 *types.Transaction, _a3 types.Attempt)) *AttemptBuilder_NewBumpAttempt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(logger.Logger), args[2].(*types.Transaction), args[3].(types.Attempt)) + }) + return _c +} + +func (_c *AttemptBuilder_NewBumpAttempt_Call) Return(_a0 *types.Attempt, _a1 error) *AttemptBuilder_NewBumpAttempt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AttemptBuilder_NewBumpAttempt_Call) RunAndReturn(run func(context.Context, logger.Logger, *types.Transaction, types.Attempt) (*types.Attempt, error)) *AttemptBuilder_NewBumpAttempt_Call { + _c.Call.Return(run) + return _c +} + +// NewAttemptBuilder creates a new instance of AttemptBuilder. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAttemptBuilder(t interface { + mock.TestingT + Cleanup(func()) +}) *AttemptBuilder { + mock := &AttemptBuilder{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go new file mode 100644 index 00000000000..f90f362a83d --- /dev/null +++ b/core/chains/evm/txm/mocks/client.go @@ -0,0 +1,252 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + context "context" + big "math/big" + + common "github.com/ethereum/go-ethereum/common" + + mock "github.com/stretchr/testify/mock" + + rpc "github.com/ethereum/go-ethereum/rpc" + + types "github.com/ethereum/go-ethereum/core/types" +) + +// Client is an autogenerated mock type for the Client type +type Client struct { + mock.Mock +} + +type Client_Expecter struct { + mock *mock.Mock +} + +func (_m *Client) EXPECT() *Client_Expecter { + return &Client_Expecter{mock: &_m.Mock} +} + +// BatchCallContext provides a mock function with given fields: _a0, _a1 +func (_m *Client) BatchCallContext(_a0 context.Context, _a1 []rpc.BatchElem) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for BatchCallContext") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []rpc.BatchElem) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Client_BatchCallContext_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BatchCallContext' +type Client_BatchCallContext_Call struct { + *mock.Call +} + +// BatchCallContext is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 []rpc.BatchElem +func (_e *Client_Expecter) BatchCallContext(_a0 interface{}, _a1 interface{}) *Client_BatchCallContext_Call { + return &Client_BatchCallContext_Call{Call: _e.mock.On("BatchCallContext", _a0, _a1)} +} + +func (_c *Client_BatchCallContext_Call) Run(run func(_a0 context.Context, _a1 []rpc.BatchElem)) *Client_BatchCallContext_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]rpc.BatchElem)) + }) + return _c +} + +func (_c *Client_BatchCallContext_Call) Return(_a0 error) *Client_BatchCallContext_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Client_BatchCallContext_Call) RunAndReturn(run func(context.Context, []rpc.BatchElem) error) *Client_BatchCallContext_Call { + _c.Call.Return(run) + return _c +} + +// NonceAt provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Client) NonceAt(_a0 context.Context, _a1 common.Address, _a2 *big.Int) (uint64, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for NonceAt") + } + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) (uint64, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) uint64); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Client_NonceAt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NonceAt' +type Client_NonceAt_Call struct { + *mock.Call +} + +// NonceAt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +// - _a2 *big.Int +func (_e *Client_Expecter) NonceAt(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Client_NonceAt_Call { + return &Client_NonceAt_Call{Call: _e.mock.On("NonceAt", _a0, _a1, _a2)} +} + +func (_c *Client_NonceAt_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 *big.Int)) *Client_NonceAt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].(*big.Int)) + }) + return _c +} + +func (_c *Client_NonceAt_Call) Return(_a0 uint64, _a1 error) *Client_NonceAt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Client_NonceAt_Call) RunAndReturn(run func(context.Context, common.Address, *big.Int) (uint64, error)) *Client_NonceAt_Call { + _c.Call.Return(run) + return _c +} + +// PendingNonceAt provides a mock function with given fields: _a0, _a1 +func (_m *Client) PendingNonceAt(_a0 context.Context, _a1 common.Address) (uint64, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for PendingNonceAt") + } + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address) (uint64, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address) uint64); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Client_PendingNonceAt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PendingNonceAt' +type Client_PendingNonceAt_Call struct { + *mock.Call +} + +// PendingNonceAt is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +func (_e *Client_Expecter) PendingNonceAt(_a0 interface{}, _a1 interface{}) *Client_PendingNonceAt_Call { + return &Client_PendingNonceAt_Call{Call: _e.mock.On("PendingNonceAt", _a0, _a1)} +} + +func (_c *Client_PendingNonceAt_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *Client_PendingNonceAt_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address)) + }) + return _c +} + +func (_c *Client_PendingNonceAt_Call) Return(_a0 uint64, _a1 error) *Client_PendingNonceAt_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Client_PendingNonceAt_Call) RunAndReturn(run func(context.Context, common.Address) (uint64, error)) *Client_PendingNonceAt_Call { + _c.Call.Return(run) + return _c +} + +// SendTransaction provides a mock function with given fields: _a0, _a1 +func (_m *Client) SendTransaction(_a0 context.Context, _a1 *types.Transaction) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for SendTransaction") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Client_SendTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SendTransaction' +type Client_SendTransaction_Call struct { + *mock.Call +} + +// SendTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *types.Transaction +func (_e *Client_Expecter) SendTransaction(_a0 interface{}, _a1 interface{}) *Client_SendTransaction_Call { + return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", _a0, _a1)} +} + +func (_c *Client_SendTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Client_SendTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.Transaction)) + }) + return _c +} + +func (_c *Client_SendTransaction_Call) Return(_a0 error) *Client_SendTransaction_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction) error) *Client_SendTransaction_Call { + _c.Call.Return(run) + return _c +} + +// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewClient(t interface { + mock.TestingT + Cleanup(func()) +}) *Client { + mock := &Client{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/mocks/storage.go b/core/chains/evm/txm/mocks/storage.go new file mode 100644 index 00000000000..4962fabd471 --- /dev/null +++ b/core/chains/evm/txm/mocks/storage.go @@ -0,0 +1,699 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + context "context" + big "math/big" + + common "github.com/ethereum/go-ethereum/common" + + mock "github.com/stretchr/testify/mock" + + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +// Storage is an autogenerated mock type for the Storage type +type Storage struct { + mock.Mock +} + +type Storage_Expecter struct { + mock *mock.Mock +} + +func (_m *Storage) EXPECT() *Storage_Expecter { + return &Storage_Expecter{mock: &_m.Mock} +} + +// AbandonPendingTransactions provides a mock function with given fields: _a0, _a1 +func (_m *Storage) AbandonPendingTransactions(_a0 context.Context, _a1 common.Address) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for AbandonPendingTransactions") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_AbandonPendingTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbandonPendingTransactions' +type Storage_AbandonPendingTransactions_Call struct { + *mock.Call +} + +// AbandonPendingTransactions is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +func (_e *Storage_Expecter) AbandonPendingTransactions(_a0 interface{}, _a1 interface{}) *Storage_AbandonPendingTransactions_Call { + return &Storage_AbandonPendingTransactions_Call{Call: _e.mock.On("AbandonPendingTransactions", _a0, _a1)} +} + +func (_c *Storage_AbandonPendingTransactions_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *Storage_AbandonPendingTransactions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address)) + }) + return _c +} + +func (_c *Storage_AbandonPendingTransactions_Call) Return(_a0 error) *Storage_AbandonPendingTransactions_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_AbandonPendingTransactions_Call) RunAndReturn(run func(context.Context, common.Address) error) *Storage_AbandonPendingTransactions_Call { + _c.Call.Return(run) + return _c +} + +// AppendAttemptToTransaction provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Storage) AppendAttemptToTransaction(_a0 context.Context, _a1 uint64, _a2 *types.Attempt) error { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for AppendAttemptToTransaction") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, *types.Attempt) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_AppendAttemptToTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AppendAttemptToTransaction' +type Storage_AppendAttemptToTransaction_Call struct { + *mock.Call +} + +// AppendAttemptToTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 *types.Attempt +func (_e *Storage_Expecter) AppendAttemptToTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_AppendAttemptToTransaction_Call { + return &Storage_AppendAttemptToTransaction_Call{Call: _e.mock.On("AppendAttemptToTransaction", _a0, _a1, _a2)} +} + +func (_c *Storage_AppendAttemptToTransaction_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 *types.Attempt)) *Storage_AppendAttemptToTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(*types.Attempt)) + }) + return _c +} + +func (_c *Storage_AppendAttemptToTransaction_Call) Return(_a0 error) *Storage_AppendAttemptToTransaction_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_AppendAttemptToTransaction_Call) RunAndReturn(run func(context.Context, uint64, *types.Attempt) error) *Storage_AppendAttemptToTransaction_Call { + _c.Call.Return(run) + return _c +} + +// CountUnstartedTransactions provides a mock function with given fields: _a0, _a1 +func (_m *Storage) CountUnstartedTransactions(_a0 context.Context, _a1 common.Address) (int, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for CountUnstartedTransactions") + } + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address) (int, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address) int); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Storage_CountUnstartedTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountUnstartedTransactions' +type Storage_CountUnstartedTransactions_Call struct { + *mock.Call +} + +// CountUnstartedTransactions is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +func (_e *Storage_Expecter) CountUnstartedTransactions(_a0 interface{}, _a1 interface{}) *Storage_CountUnstartedTransactions_Call { + return &Storage_CountUnstartedTransactions_Call{Call: _e.mock.On("CountUnstartedTransactions", _a0, _a1)} +} + +func (_c *Storage_CountUnstartedTransactions_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *Storage_CountUnstartedTransactions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address)) + }) + return _c +} + +func (_c *Storage_CountUnstartedTransactions_Call) Return(_a0 int, _a1 error) *Storage_CountUnstartedTransactions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Storage_CountUnstartedTransactions_Call) RunAndReturn(run func(context.Context, common.Address) (int, error)) *Storage_CountUnstartedTransactions_Call { + _c.Call.Return(run) + return _c +} + +// CreateEmptyUnconfirmedTransaction provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *Storage) CreateEmptyUnconfirmedTransaction(_a0 context.Context, _a1 common.Address, _a2 *big.Int, _a3 uint64, _a4 uint64) (*types.Transaction, error) { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) + + if len(ret) == 0 { + panic("no return value specified for CreateEmptyUnconfirmedTransaction") + } + + var r0 *types.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error)); ok { + return rf(_a0, _a1, _a2, _a3, _a4) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int, uint64, uint64) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int, uint64, uint64) error); ok { + r1 = rf(_a0, _a1, _a2, _a3, _a4) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Storage_CreateEmptyUnconfirmedTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateEmptyUnconfirmedTransaction' +type Storage_CreateEmptyUnconfirmedTransaction_Call struct { + *mock.Call +} + +// CreateEmptyUnconfirmedTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +// - _a2 *big.Int +// - _a3 uint64 +// - _a4 uint64 +func (_e *Storage_Expecter) CreateEmptyUnconfirmedTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}, _a4 interface{}) *Storage_CreateEmptyUnconfirmedTransaction_Call { + return &Storage_CreateEmptyUnconfirmedTransaction_Call{Call: _e.mock.On("CreateEmptyUnconfirmedTransaction", _a0, _a1, _a2, _a3, _a4)} +} + +func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 *big.Int, _a3 uint64, _a4 uint64)) *Storage_CreateEmptyUnconfirmedTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].(*big.Int), args[3].(uint64), args[4].(uint64)) + }) + return _c +} + +func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *Storage_CreateEmptyUnconfirmedTransaction_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) RunAndReturn(run func(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error)) *Storage_CreateEmptyUnconfirmedTransaction_Call { + _c.Call.Return(run) + return _c +} + +// CreateTransaction provides a mock function with given fields: _a0, _a1 +func (_m *Storage) CreateTransaction(_a0 context.Context, _a1 *types.Transaction) (uint64, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for CreateTransaction") + } + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) (uint64, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) uint64); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context, *types.Transaction) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Storage_CreateTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTransaction' +type Storage_CreateTransaction_Call struct { + *mock.Call +} + +// CreateTransaction is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *types.Transaction +func (_e *Storage_Expecter) CreateTransaction(_a0 interface{}, _a1 interface{}) *Storage_CreateTransaction_Call { + return &Storage_CreateTransaction_Call{Call: _e.mock.On("CreateTransaction", _a0, _a1)} +} + +func (_c *Storage_CreateTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Storage_CreateTransaction_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.Transaction)) + }) + return _c +} + +func (_c *Storage_CreateTransaction_Call) Return(_a0 uint64, _a1 error) *Storage_CreateTransaction_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Storage_CreateTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction) (uint64, error)) *Storage_CreateTransaction_Call { + _c.Call.Return(run) + return _c +} + +// DeleteAttemptForUnconfirmedTx provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Storage) DeleteAttemptForUnconfirmedTx(_a0 context.Context, _a1 uint64, _a2 *types.Attempt) error { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for DeleteAttemptForUnconfirmedTx") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, *types.Attempt) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_DeleteAttemptForUnconfirmedTx_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteAttemptForUnconfirmedTx' +type Storage_DeleteAttemptForUnconfirmedTx_Call struct { + *mock.Call +} + +// DeleteAttemptForUnconfirmedTx is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 *types.Attempt +func (_e *Storage_Expecter) DeleteAttemptForUnconfirmedTx(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_DeleteAttemptForUnconfirmedTx_Call { + return &Storage_DeleteAttemptForUnconfirmedTx_Call{Call: _e.mock.On("DeleteAttemptForUnconfirmedTx", _a0, _a1, _a2)} +} + +func (_c *Storage_DeleteAttemptForUnconfirmedTx_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 *types.Attempt)) *Storage_DeleteAttemptForUnconfirmedTx_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(*types.Attempt)) + }) + return _c +} + +func (_c *Storage_DeleteAttemptForUnconfirmedTx_Call) Return(_a0 error) *Storage_DeleteAttemptForUnconfirmedTx_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_DeleteAttemptForUnconfirmedTx_Call) RunAndReturn(run func(context.Context, uint64, *types.Attempt) error) *Storage_DeleteAttemptForUnconfirmedTx_Call { + _c.Call.Return(run) + return _c +} + +// FetchUnconfirmedTransactionAtNonceWithCount provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Storage) FetchUnconfirmedTransactionAtNonceWithCount(_a0 context.Context, _a1 uint64, _a2 common.Address) (*types.Transaction, int, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for FetchUnconfirmedTransactionAtNonceWithCount") + } + + var r0 *types.Transaction + var r1 int + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) (*types.Transaction, int, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64, common.Address) int); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(context.Context, uint64, common.Address) error); ok { + r2 = rf(_a0, _a1, _a2) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchUnconfirmedTransactionAtNonceWithCount' +type Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call struct { + *mock.Call +} + +// FetchUnconfirmedTransactionAtNonceWithCount is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 common.Address +func (_e *Storage_Expecter) FetchUnconfirmedTransactionAtNonceWithCount(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { + return &Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call{Call: _e.mock.On("FetchUnconfirmedTransactionAtNonceWithCount", _a0, _a1, _a2)} +} + +func (_c *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) + }) + return _c +} + +func (_c *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call) Return(_a0 *types.Transaction, _a1 int, _a2 error) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call) RunAndReturn(run func(context.Context, uint64, common.Address) (*types.Transaction, int, error)) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { + _c.Call.Return(run) + return _c +} + +// MarkTransactionsConfirmed provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Storage) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]uint64, []uint64, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for MarkTransactionsConfirmed") + } + + var r0 []uint64 + var r1 []uint64 + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) ([]uint64, []uint64, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) []uint64); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]uint64) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64, common.Address) []uint64); ok { + r1 = rf(_a0, _a1, _a2) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]uint64) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, uint64, common.Address) error); ok { + r2 = rf(_a0, _a1, _a2) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// Storage_MarkTransactionsConfirmed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTransactionsConfirmed' +type Storage_MarkTransactionsConfirmed_Call struct { + *mock.Call +} + +// MarkTransactionsConfirmed is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 common.Address +func (_e *Storage_Expecter) MarkTransactionsConfirmed(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_MarkTransactionsConfirmed_Call { + return &Storage_MarkTransactionsConfirmed_Call{Call: _e.mock.On("MarkTransactionsConfirmed", _a0, _a1, _a2)} +} + +func (_c *Storage_MarkTransactionsConfirmed_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *Storage_MarkTransactionsConfirmed_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) + }) + return _c +} + +func (_c *Storage_MarkTransactionsConfirmed_Call) Return(_a0 []uint64, _a1 []uint64, _a2 error) *Storage_MarkTransactionsConfirmed_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *Storage_MarkTransactionsConfirmed_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]uint64, []uint64, error)) *Storage_MarkTransactionsConfirmed_Call { + _c.Call.Return(run) + return _c +} + +// MarkTxFatal provides a mock function with given fields: _a0, _a1 +func (_m *Storage) MarkTxFatal(_a0 context.Context, _a1 *types.Transaction) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for MarkTxFatal") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_MarkTxFatal_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTxFatal' +type Storage_MarkTxFatal_Call struct { + *mock.Call +} + +// MarkTxFatal is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *types.Transaction +func (_e *Storage_Expecter) MarkTxFatal(_a0 interface{}, _a1 interface{}) *Storage_MarkTxFatal_Call { + return &Storage_MarkTxFatal_Call{Call: _e.mock.On("MarkTxFatal", _a0, _a1)} +} + +func (_c *Storage_MarkTxFatal_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Storage_MarkTxFatal_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.Transaction)) + }) + return _c +} + +func (_c *Storage_MarkTxFatal_Call) Return(_a0 error) *Storage_MarkTxFatal_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_MarkTxFatal_Call) RunAndReturn(run func(context.Context, *types.Transaction) error) *Storage_MarkTxFatal_Call { + _c.Call.Return(run) + return _c +} + +// MarkUnconfirmedTransactionPurgeable provides a mock function with given fields: _a0, _a1 +func (_m *Storage) MarkUnconfirmedTransactionPurgeable(_a0 context.Context, _a1 uint64) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for MarkUnconfirmedTransactionPurgeable") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_MarkUnconfirmedTransactionPurgeable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkUnconfirmedTransactionPurgeable' +type Storage_MarkUnconfirmedTransactionPurgeable_Call struct { + *mock.Call +} + +// MarkUnconfirmedTransactionPurgeable is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +func (_e *Storage_Expecter) MarkUnconfirmedTransactionPurgeable(_a0 interface{}, _a1 interface{}) *Storage_MarkUnconfirmedTransactionPurgeable_Call { + return &Storage_MarkUnconfirmedTransactionPurgeable_Call{Call: _e.mock.On("MarkUnconfirmedTransactionPurgeable", _a0, _a1)} +} + +func (_c *Storage_MarkUnconfirmedTransactionPurgeable_Call) Run(run func(_a0 context.Context, _a1 uint64)) *Storage_MarkUnconfirmedTransactionPurgeable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64)) + }) + return _c +} + +func (_c *Storage_MarkUnconfirmedTransactionPurgeable_Call) Return(_a0 error) *Storage_MarkUnconfirmedTransactionPurgeable_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_MarkUnconfirmedTransactionPurgeable_Call) RunAndReturn(run func(context.Context, uint64) error) *Storage_MarkUnconfirmedTransactionPurgeable_Call { + _c.Call.Return(run) + return _c +} + +// UpdateTransactionBroadcast provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *Storage) UpdateTransactionBroadcast(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash) error { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for UpdateTransactionBroadcast") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, uint64, common.Hash) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_UpdateTransactionBroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTransactionBroadcast' +type Storage_UpdateTransactionBroadcast_Call struct { + *mock.Call +} + +// UpdateTransactionBroadcast is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 uint64 +// - _a2 uint64 +// - _a3 common.Hash +func (_e *Storage_Expecter) UpdateTransactionBroadcast(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *Storage_UpdateTransactionBroadcast_Call { + return &Storage_UpdateTransactionBroadcast_Call{Call: _e.mock.On("UpdateTransactionBroadcast", _a0, _a1, _a2, _a3)} +} + +func (_c *Storage_UpdateTransactionBroadcast_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash)) *Storage_UpdateTransactionBroadcast_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(uint64), args[2].(uint64), args[3].(common.Hash)) + }) + return _c +} + +func (_c *Storage_UpdateTransactionBroadcast_Call) Return(_a0 error) *Storage_UpdateTransactionBroadcast_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_UpdateTransactionBroadcast_Call) RunAndReturn(run func(context.Context, uint64, uint64, common.Hash) error) *Storage_UpdateTransactionBroadcast_Call { + _c.Call.Return(run) + return _c +} + +// UpdateUnstartedTransactionWithNonce provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Storage) UpdateUnstartedTransactionWithNonce(_a0 context.Context, _a1 common.Address, _a2 uint64) (*types.Transaction, error) { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for UpdateUnstartedTransactionWithNonce") + } + + var r0 *types.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64) (*types.Transaction, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, uint64) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Storage_UpdateUnstartedTransactionWithNonce_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateUnstartedTransactionWithNonce' +type Storage_UpdateUnstartedTransactionWithNonce_Call struct { + *mock.Call +} + +// UpdateUnstartedTransactionWithNonce is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 common.Address +// - _a2 uint64 +func (_e *Storage_Expecter) UpdateUnstartedTransactionWithNonce(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_UpdateUnstartedTransactionWithNonce_Call { + return &Storage_UpdateUnstartedTransactionWithNonce_Call{Call: _e.mock.On("UpdateUnstartedTransactionWithNonce", _a0, _a1, _a2)} +} + +func (_c *Storage_UpdateUnstartedTransactionWithNonce_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 uint64)) *Storage_UpdateUnstartedTransactionWithNonce_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].(uint64)) + }) + return _c +} + +func (_c *Storage_UpdateUnstartedTransactionWithNonce_Call) Return(_a0 *types.Transaction, _a1 error) *Storage_UpdateUnstartedTransactionWithNonce_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Storage_UpdateUnstartedTransactionWithNonce_Call) RunAndReturn(run func(context.Context, common.Address, uint64) (*types.Transaction, error)) *Storage_UpdateUnstartedTransactionWithNonce_Call { + _c.Call.Return(run) + return _c +} + +// NewStorage creates a new instance of Storage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewStorage(t interface { + mock.TestingT + Cleanup(func()) +}) *Storage { + mock := &Storage{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go new file mode 100644 index 00000000000..c89f73b8899 --- /dev/null +++ b/core/chains/evm/txm/orchestrator.go @@ -0,0 +1,38 @@ +package txm + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" + nullv4 "gopkg.in/guregu/null.v4" + + "github.com/smartcontractkit/chainlink-common/pkg/services" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type TxmOrchestrator interface { + services.Service + Trigger(addr common.Address) + CreateTransaction(ctx context.Context, txRequest *types.Transaction) (id int64, err error) + GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) + GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) + RegisterResumeCallback(fn ResumeCallback) + SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value *big.Int, gasLimit uint64) (tx *types.Transaction, err error) + CountTransactionsByState(ctx context.Context, state types.TxState) (count int, err error) + GetTransactionStatus(ctx context.Context, idempotencyKey string) (state commontypes.TransactionStatus, err error) + //Reset(addr ADDR, abandon bool) error // Potentially will be replaced by Abandon + + // Testing methods(?) + FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) + FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) + FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*types.Transaction, err error) + FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []types.TxState, chainID *big.Int) (txes []*types.Transaction, err error) + FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) + FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) +} + +type ResumeCallback func(ctx context.Context, id uuid.UUID, result interface{}, err error) error diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go new file mode 100644 index 00000000000..12df2130502 --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -0,0 +1,238 @@ +package storage + +import ( + "context" + "fmt" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type InMemoryStore struct { + sync.RWMutex + lggr logger.Logger + txIDCount uint64 + + UnstartedTransactions []*types.Transaction + UnconfirmedTransactions map[uint64]*types.Transaction + ConfirmedTransactions map[uint64]*types.Transaction + FatalTransactions []*types.Transaction +} + +func NewInMemoryStore(lggr logger.Logger) *InMemoryStore { + return &InMemoryStore{ + lggr: logger.Named(lggr, "InMemoryStore"), + UnconfirmedTransactions: make(map[uint64]*types.Transaction), + ConfirmedTransactions: make(map[uint64]*types.Transaction), + } +} + +func (m *InMemoryStore) AbandonPendingTransactions(context.Context, common.Address) error { + m.Lock() + defer m.Unlock() + + for _, tx := range m.UnstartedTransactions { + tx.State = types.TxFatalError + } + m.FatalTransactions = m.UnstartedTransactions + m.UnstartedTransactions = []*types.Transaction{} + + for _, tx := range m.UnconfirmedTransactions { + tx.State = types.TxFatalError + m.FatalTransactions = append(m.FatalTransactions, tx) + } + m.UnconfirmedTransactions = make(map[uint64]*types.Transaction) + + return nil +} + +func (m *InMemoryStore) AppendAttemptToTransaction(_ context.Context, txNonce uint64, attempt *types.Attempt) error { + m.Lock() + defer m.Unlock() + + tx, exists := m.UnconfirmedTransactions[txNonce] + if !exists { + return fmt.Errorf("unconfirmed tx was not found for nonce: %d - txID: %v", txNonce, attempt.TxID) + } + + if tx.ID != attempt.TxID { + return fmt.Errorf("unconfirmed tx with nonce exists but attempt points to a different txID. Found Tx: %v - txID: %v", m.UnconfirmedTransactions[txNonce], attempt.TxID) + } + + attempt.CreatedAt = time.Now() + m.UnconfirmedTransactions[txNonce].Attempts = append(m.UnconfirmedTransactions[txNonce].Attempts, attempt.DeepCopy()) + + return nil +} + +func (m *InMemoryStore) CountUnstartedTransactions(context.Context, common.Address) (int, error) { + m.RLock() + defer m.RUnlock() + + return len(m.UnstartedTransactions), nil +} + +func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(ctx context.Context, fromAddress common.Address, chainID *big.Int, nonce uint64, limit uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + emptyTx := &types.Transaction{ + ID: m.txIDCount, + ChainID: chainID, + Nonce: nonce, + FromAddress: fromAddress, + ToAddress: common.Address{}, + Value: big.NewInt(0), + SpecifiedGasLimit: limit, + CreatedAt: time.Now(), + State: types.TxUnconfirmed, + } + + if _, exists := m.UnconfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) + } + + m.UnconfirmedTransactions[nonce] = emptyTx + + return emptyTx.DeepCopy(), nil +} + +func (m *InMemoryStore) CreateTransaction(_ context.Context, tx *types.Transaction) (uint64, error) { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + + tx.ID = m.txIDCount + tx.CreatedAt = time.Now() + tx.State = types.TxUnstarted + + m.UnstartedTransactions = append(m.UnstartedTransactions, tx.DeepCopy()) + return tx.ID, nil +} + +func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(_ context.Context, latestNonce uint64, _ common.Address) (txCopy *types.Transaction, unconfirmedCount int, err error) { + m.RLock() + defer m.RUnlock() + + tx := m.UnconfirmedTransactions[latestNonce] + if tx != nil { + txCopy = tx.DeepCopy() + } + unconfirmedCount = len(m.UnconfirmedTransactions) + return +} + +func (m *InMemoryStore) MarkTransactionsConfirmed(_ context.Context, latestNonce uint64, _ common.Address) ([]uint64, []uint64, error) { + m.Lock() + defer m.Unlock() + + var confirmedTransactionIDs []uint64 + for _, tx := range m.UnconfirmedTransactions { + if tx.Nonce < latestNonce { + tx.State = types.TxConfirmed + confirmedTransactionIDs = append(confirmedTransactionIDs, tx.ID) + m.ConfirmedTransactions[tx.Nonce] = tx + delete(m.UnconfirmedTransactions, tx.Nonce) + } + } + + var unconfirmedTransactionIDs []uint64 + for _, tx := range m.ConfirmedTransactions { + if tx.Nonce >= latestNonce { + tx.State = types.TxUnconfirmed + unconfirmedTransactionIDs = append(unconfirmedTransactionIDs, tx.ID) + m.UnconfirmedTransactions[tx.Nonce] = tx + delete(m.ConfirmedTransactions, tx.Nonce) + } + } + return confirmedTransactionIDs, unconfirmedTransactionIDs, nil +} + +func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(_ context.Context, nonce uint64) error { + m.Lock() + defer m.Unlock() + + tx, exists := m.UnconfirmedTransactions[nonce] + if !exists { + return fmt.Errorf("unconfirmed tx with nonce: %d was not found", nonce) + } + + tx.IsPurgeable = true + + return nil +} + +func (m *InMemoryStore) UpdateTransactionBroadcast(_ context.Context, txID uint64, txNonce uint64, attemptHash common.Hash) error { + m.Lock() + defer m.Unlock() + + unconfirmedTx, exists := m.UnconfirmedTransactions[txNonce] + if !exists { + return fmt.Errorf("unconfirmed tx was not found for nonce: %d - txID: %v", txNonce, txID) + } + + // Set the same time for both the tx and its attempt + now := time.Now() + unconfirmedTx.LastBroadcastAt = now + a, err := unconfirmedTx.FindAttemptByHash(attemptHash) + if err != nil { + return err + } + a.BroadcastAt = now + + return nil +} + +func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(_ context.Context, _ common.Address, nonce uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + if len(m.UnstartedTransactions) == 0 { + m.lggr.Debug("Unstarted transaction queue is empty") + return nil, nil + } + + if _, exists := m.UnconfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) + } + + tx := m.UnstartedTransactions[0] + tx.Nonce = nonce + tx.State = types.TxUnconfirmed + + m.UnstartedTransactions = m.UnstartedTransactions[1:] + m.UnconfirmedTransactions[nonce] = tx + + return tx.DeepCopy(), nil +} + +// Error Handler +func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(_ context.Context, transactionNonce uint64, attempt *types.Attempt) error { + m.Lock() + defer m.Unlock() + + tx, exists := m.UnconfirmedTransactions[transactionNonce] + if !exists { + return fmt.Errorf("unconfirmed tx was not found for nonce: %d - txID: %v", transactionNonce, attempt.TxID) + } + + for i, a := range tx.Attempts { + if a.Hash == attempt.Hash { + tx.Attempts = append(tx.Attempts[:i], tx.Attempts[i+1:]...) + return nil + } + } + + return fmt.Errorf("attempt with hash: %v for txID: %v was not found", attempt.Hash, attempt.TxID) +} + +func (m *InMemoryStore) MarkTxFatal(context.Context, *types.Transaction) error { + return fmt.Errorf("not implemented") +} diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go new file mode 100644 index 00000000000..7eb2be0864a --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -0,0 +1,426 @@ +package storage + +import ( + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +func TestAbandonPendingTransactions(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + t.Run("abandons unstarted and unconfirmed transactions", func(t *testing.T) { + // Unstarted + tx1 := insertUnstartedTransaction(m, fromAddress) + tx2 := insertUnstartedTransaction(m, fromAddress) + + // Unconfirmed + tx3, err := insertUnconfirmedTransaction(m, fromAddress, 3) + assert.NoError(t, err) + tx4, err := insertUnconfirmedTransaction(m, fromAddress, 4) + assert.NoError(t, err) + + assert.NoError(t, m.AbandonPendingTransactions(tests.Context(t), fromAddress)) + + assert.Equal(t, types.TxFatalError, tx1.State) + assert.Equal(t, types.TxFatalError, tx2.State) + assert.Equal(t, types.TxFatalError, tx3.State) + assert.Equal(t, types.TxFatalError, tx4.State) + }) + + t.Run("skips all types apart from unstarted and unconfirmed transactions", func(t *testing.T) { + // Fatal + tx1 := insertFataTransaction(m, fromAddress) + tx2 := insertFataTransaction(m, fromAddress) + + // Confirmed + tx3, err := insertConfirmedTransaction(m, fromAddress, 3) + assert.NoError(t, err) + tx4, err := insertConfirmedTransaction(m, fromAddress, 4) + assert.NoError(t, err) + + assert.NoError(t, m.AbandonPendingTransactions(tests.Context(t), fromAddress)) + + assert.Equal(t, types.TxFatalError, tx1.State) + assert.Equal(t, types.TxFatalError, tx2.State) + assert.Equal(t, types.TxConfirmed, tx3.State) + assert.Equal(t, types.TxConfirmed, tx4.State) + + }) + +} + +func TestAppendAttemptToTransaction(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + + _, err := insertUnconfirmedTransaction(m, fromAddress, 0) // txID = 1 + assert.NoError(t, err) + _, err = insertConfirmedTransaction(m, fromAddress, 2) // txID = 1 + assert.NoError(t, err) + + t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { + var nonce uint64 = 1 + newAttempt := &types.Attempt{ + TxID: 1, + } + assert.Error(t, m.AppendAttemptToTransaction(tests.Context(t), nonce, newAttempt)) + }) + + t.Run("fails if unconfirmed transaction was found but has doesn't match the txID", func(t *testing.T) { + var nonce uint64 = 0 + newAttempt := &types.Attempt{ + TxID: 2, + } + assert.Error(t, m.AppendAttemptToTransaction(tests.Context(t), nonce, newAttempt)) + }) + + t.Run("appends attempt to transaction", func(t *testing.T) { + var nonce uint64 = 0 + newAttempt := &types.Attempt{ + TxID: 1, + } + assert.NoError(t, m.AppendAttemptToTransaction(tests.Context(t), nonce, newAttempt)) + + }) +} + +func TestCountUnstartedTransactions(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + n, _ := m.CountUnstartedTransactions(tests.Context(t), fromAddress) + assert.Equal(t, 0, n) + + insertUnstartedTransaction(m, fromAddress) + n, _ = m.CountUnstartedTransactions(tests.Context(t), fromAddress) + assert.Equal(t, 1, n) + +} + +func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + insertUnconfirmedTransaction(m, fromAddress, 0) + + t.Run("fails if unconfirmed transaction with the same nonce exists", func(t *testing.T) { + _, err := m.CreateEmptyUnconfirmedTransaction(tests.Context(t), fromAddress, testutils.FixtureChainID, 0, 0) + assert.Error(t, err) + }) + + t.Run("creates a new empty unconfirmed transaction", func(t *testing.T) { + tx, err := m.CreateEmptyUnconfirmedTransaction(tests.Context(t), fromAddress, testutils.FixtureChainID, 1, 0) + assert.NoError(t, err) + assert.Equal(t, types.TxUnconfirmed, tx.State) + }) + +} + +func TestCreateTransaction(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + + tx1 := &types.Transaction{} + tx2 := &types.Transaction{} + id1, err := m.CreateTransaction(tests.Context(t), tx1) + assert.NoError(t, err) + assert.Equal(t, uint64(1), id1) + + id2, err := m.CreateTransaction(tests.Context(t), tx2) + assert.NoError(t, err) + assert.Equal(t, uint64(2), id2) + + count, _ := m.CountUnstartedTransactions(tests.Context(t), fromAddress) + assert.Equal(t, count, 2) + +} + +func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + + tx, count, _ := m.FetchUnconfirmedTransactionAtNonceWithCount(tests.Context(t), 0, fromAddress) + assert.Nil(t, tx) + assert.Equal(t, 0, count) + + var nonce uint64 = 0 + insertUnconfirmedTransaction(m, fromAddress, nonce) + tx, count, _ = m.FetchUnconfirmedTransactionAtNonceWithCount(tests.Context(t), nonce, fromAddress) + assert.Equal(t, tx.Nonce, nonce) + assert.Equal(t, 1, count) + +} + +func TestMarkTransactionsConfirmed(t *testing.T) { + + fromAddress := testutils.NewAddress() + + t.Run("returns 0 if there are no transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + un, cn, err := m.MarkTransactionsConfirmed(tests.Context(t), 100, fromAddress) + assert.NoError(t, err) + assert.Equal(t, len(un), 0) + assert.Equal(t, len(cn), 0) + }) + + t.Run("confirms transaction with nonce lower than the latest", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + ctx1, err := insertUnconfirmedTransaction(m, fromAddress, 0) + assert.NoError(t, err) + + ctx2, err := insertUnconfirmedTransaction(m, fromAddress, 1) + assert.NoError(t, err) + + ctxs, utxs, err := m.MarkTransactionsConfirmed(tests.Context(t), 1, fromAddress) + assert.NoError(t, err) + assert.Equal(t, types.TxConfirmed, ctx1.State) + assert.Equal(t, types.TxUnconfirmed, ctx2.State) + assert.Equal(t, ctxs[0], ctx1.ID) + assert.Equal(t, 0, len(utxs)) + }) + + t.Run("unconfirms transaction with nonce equal to or higher than the latest", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + ctx1, err := insertConfirmedTransaction(m, fromAddress, 0) + assert.NoError(t, err) + + ctx2, err := insertConfirmedTransaction(m, fromAddress, 1) + assert.NoError(t, err) + + ctxs, utxs, err := m.MarkTransactionsConfirmed(tests.Context(t), 1, fromAddress) + assert.NoError(t, err) + assert.Equal(t, types.TxConfirmed, ctx1.State) + assert.Equal(t, types.TxUnconfirmed, ctx2.State) + assert.Equal(t, utxs[0], ctx2.ID) + assert.Equal(t, 0, len(ctxs)) + }) +} + +func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { + + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + + // fails if tx was not found + err := m.MarkUnconfirmedTransactionPurgeable(tests.Context(t), 0) + assert.Error(t, err) + + tx, err := insertUnconfirmedTransaction(m, fromAddress, 0) + assert.NoError(t, err) + err = m.MarkUnconfirmedTransactionPurgeable(tests.Context(t), 0) + assert.NoError(t, err) + assert.Equal(t, true, tx.IsPurgeable) +} + +func TestUpdateTransactionBroadcast(t *testing.T) { + + fromAddress := testutils.NewAddress() + hash := testutils.NewHash() + t.Run("fails if unconfirmed transaction was not found", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + var nonce uint64 = 0 + assert.Error(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + }) + + t.Run("fails if attempt was not found for a given transaction", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + var nonce uint64 = 0 + tx, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + assert.NoError(t, err) + assert.Error(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + + // Attempt with different hash + attempt := &types.Attempt{TxID: tx.ID, Hash: testutils.NewHash()} + tx.Attempts = append(tx.Attempts, attempt) + assert.Error(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + }) + + t.Run("updates transaction's and attempt's broadcast times", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + var nonce uint64 = 0 + tx, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + assert.NoError(t, err) + attempt := &types.Attempt{TxID: tx.ID, Hash: hash} + tx.Attempts = append(tx.Attempts, attempt) + assert.NoError(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + assert.False(t, tx.LastBroadcastAt.IsZero()) + assert.False(t, attempt.BroadcastAt.IsZero()) + }) +} + +func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { + + fromAddress := testutils.NewAddress() + t.Run("returns nil if there are no unstarted transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + tx, err := m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, 0) + assert.NoError(t, err) + assert.Nil(t, tx) + }) + + t.Run("fails if there is already another unstarted transaction with the same nonce", func(t *testing.T) { + var nonce uint64 = 0 + m := NewInMemoryStore(logger.Test(t)) + insertUnstartedTransaction(m, fromAddress) + _, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + assert.NoError(t, err) + + _, err = m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, nonce) + assert.Error(t, err) + }) + + t.Run("updates unstarted transaction to unconfirmed and assigns a nonce", func(t *testing.T) { + var nonce uint64 = 0 + m := NewInMemoryStore(logger.Test(t)) + insertUnstartedTransaction(m, fromAddress) + + tx, err := m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, nonce) + assert.NoError(t, err) + assert.Equal(t, nonce, tx.Nonce) + assert.Equal(t, types.TxUnconfirmed, tx.State) + }) +} + +func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { + + fromAddress := testutils.NewAddress() + t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + tx := &types.Transaction{Nonce: 0} + attempt := &types.Attempt{TxID: 0} + err := m.DeleteAttemptForUnconfirmedTx(tests.Context(t), tx.Nonce, attempt) + assert.Error(t, err) + }) + + t.Run("fails if corresponding unconfirmed attempt for txID was not found", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + _, err := insertUnconfirmedTransaction(m, fromAddress, 0) + assert.NoError(t, err) + + attempt := &types.Attempt{TxID: 2, Hash: testutils.NewHash()} + err = m.DeleteAttemptForUnconfirmedTx(tests.Context(t), 0, attempt) + + assert.Error(t, err) + }) + + t.Run("deletes attempt of unconfirmed transaction", func(t *testing.T) { + hash := testutils.NewHash() + var nonce uint64 = 0 + m := NewInMemoryStore(logger.Test(t)) + tx, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + assert.NoError(t, err) + + attempt := &types.Attempt{TxID: 0, Hash: hash} + tx.Attempts = append(tx.Attempts, attempt) + err = m.DeleteAttemptForUnconfirmedTx(tests.Context(t), nonce, attempt) + assert.NoError(t, err) + + assert.Equal(t, 0, len(tx.Attempts)) + }) +} + +func insertUnstartedTransaction(m *InMemoryStore, fromAddress common.Address) *types.Transaction { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: 0, + FromAddress: fromAddress, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: types.TxUnstarted, + } + + m.UnstartedTransactions = append(m.UnstartedTransactions, tx) + return tx +} + +func insertUnconfirmedTransaction(m *InMemoryStore, fromAddress common.Address, nonce uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: nonce, + FromAddress: fromAddress, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: types.TxUnconfirmed, + } + + if _, exists := m.UnconfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) + } + + m.UnconfirmedTransactions[nonce] = tx + return tx, nil +} + +func insertConfirmedTransaction(m *InMemoryStore, fromAddress common.Address, nonce uint64) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: nonce, + FromAddress: fromAddress, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: types.TxConfirmed, + } + + if _, exists := m.ConfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("a confirmed tx with the same nonce already exists: %v", m.ConfirmedTransactions[nonce]) + } + + m.ConfirmedTransactions[nonce] = tx + return tx, nil +} + +func insertFataTransaction(m *InMemoryStore, fromAddress common.Address) *types.Transaction { + m.Lock() + defer m.Unlock() + + m.txIDCount++ + tx := &types.Transaction{ + ID: m.txIDCount, + ChainID: testutils.FixtureChainID, + Nonce: 0, + FromAddress: fromAddress, + ToAddress: testutils.NewAddress(), + Value: big.NewInt(0), + SpecifiedGasLimit: 0, + CreatedAt: time.Now(), + State: types.TxFatalError, + } + + m.FatalTransactions = append(m.FatalTransactions, tx) + return tx +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go new file mode 100644 index 00000000000..bb345b50227 --- /dev/null +++ b/core/chains/evm/txm/txm.go @@ -0,0 +1,351 @@ +package txm + +import ( + "context" + "fmt" + "math/big" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + evmtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/utils" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +const ( + broadcastInterval time.Duration = 30 * time.Second + maxInFlightTransactions uint64 = 16 + maxAllowedAttempts uint16 = 10 +) + +type Client interface { + PendingNonceAt(context.Context, common.Address) (uint64, error) + NonceAt(context.Context, common.Address, *big.Int) (uint64, error) + SendTransaction(context.Context, *evmtypes.Transaction) error + BatchCallContext(context.Context, []rpc.BatchElem) error +} + +type Storage interface { + AbandonPendingTransactions(context.Context, common.Address) error + AppendAttemptToTransaction(context.Context, uint64, *types.Attempt) error + CountUnstartedTransactions(context.Context, common.Address) (int, error) + CreateEmptyUnconfirmedTransaction(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error) + CreateTransaction(context.Context, *types.Transaction) (uint64, error) + FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*types.Transaction, int, error) + MarkTransactionsConfirmed(context.Context, uint64, common.Address) ([]uint64, []uint64, error) + MarkUnconfirmedTransactionPurgeable(context.Context, uint64) error + UpdateTransactionBroadcast(context.Context, uint64, uint64, common.Hash) error + UpdateUnstartedTransactionWithNonce(context.Context, common.Address, uint64) (*types.Transaction, error) + + // ErrorHandler + DeleteAttemptForUnconfirmedTx(context.Context, uint64, *types.Attempt) error + MarkTxFatal(context.Context, *types.Transaction) error +} + +type AttemptBuilder interface { + NewAttempt(context.Context, logger.Logger, *types.Transaction, bool) (*types.Attempt, error) + NewBumpAttempt(context.Context, logger.Logger, *types.Transaction, types.Attempt) (*types.Attempt, error) +} + +type ErrorHandler interface { + HandleError(tx *types.Transaction, message error, attemptBuilder AttemptBuilder, client Client, storage Storage) (err error) +} + +type StuckTxDetector interface { + DetectStuckTransactions(tx *types.Transaction) (bool, error) +} + +type Config struct { + EIP1559 bool + BlockTime time.Duration + RetryBlockThreshold uint16 + EmptyTxLimitDefault uint64 +} + +type Txm struct { + services.StateMachine + lggr logger.SugaredLogger + address common.Address + chainID *big.Int + client Client + attemptBuilder AttemptBuilder + errorHandler ErrorHandler + stuckTxDetector StuckTxDetector + storage Storage + config Config + nonce atomic.Uint64 + + triggerCh chan struct{} + broadcastStopCh services.StopChan + backfillStopCh services.StopChan + wg *sync.WaitGroup +} + +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, storage Storage, config Config, address common.Address) *Txm { + return &Txm{ + lggr: logger.Sugared(logger.Named(lggr, "Txm")), + address: address, + chainID: chainID, + client: client, + attemptBuilder: attemptBuilder, + storage: storage, + config: config, + triggerCh: make(chan struct{}), + broadcastStopCh: make(chan struct{}), + backfillStopCh: make(chan struct{}), + } +} + +func (t *Txm) Start(context.Context) error { + return t.StartOnce("Txm", func() error { + pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + if err != nil { + return err + } + t.nonce.Store(pendingNonce) + t.wg.Add(2) + go t.broadcastLoop() + go t.backfillLoop() + + return nil + }) +} + +func (t *Txm) Close() error { + return t.StopOnce("Txm", func() error { + close(t.broadcastStopCh) + close(t.backfillStopCh) + t.wg.Wait() + return nil + }) +} + +func (t *Txm) Trigger() error { + if !t.IfStarted(func() { + t.triggerCh <- struct{}{} + }) { + return fmt.Errorf("Txm unstarted") + } + return nil +} + +func (t *Txm) Abandon() error { + return t.storage.AbandonPendingTransactions(context.TODO(), t.address) +} + +func (t *Txm) broadcastLoop() { + defer t.wg.Done() + broadcasterTicker := time.NewTicker(utils.WithJitter(broadcastInterval) * time.Second) + defer broadcasterTicker.Stop() + + for { + select { + case <-t.broadcastStopCh: + return + case <-t.triggerCh: + start := time.Now() + if err := t.broadcastTransaction(); err != nil { + t.lggr.Errorf("Error during triggered transaction broadcasting %w", err) + } else { + t.lggr.Debug("Triggered transaction broadcasting time elapsed: ", time.Since(start)) + } + broadcasterTicker.Reset(utils.WithJitter(broadcastInterval) * time.Second) + case <-broadcasterTicker.C: + start := time.Now() + if err := t.broadcastTransaction(); err != nil { + t.lggr.Errorf("Error during transaction broadcasting: %w", err) + } else { + t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) + } + } + } +} + +func (t *Txm) backfillLoop() { + defer t.wg.Done() + backfillTicker := time.NewTicker(utils.WithJitter(t.config.BlockTime) * time.Second) + defer backfillTicker.Stop() + + for { + select { + case <-t.backfillStopCh: + return + case <-backfillTicker.C: + start := time.Now() + if err := t.backfillTransactions(); err != nil { + t.lggr.Errorf("Error during backfill: %w", err) + } else { + t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) + } + } + } +} + +func (t *Txm) broadcastTransaction() (err error) { + pendingNonce, latestNonce, err := t.pendingAndLatestNonce(context.TODO(), t.address) + if err != nil { + return + } + + // Some clients allow out-of-order nonce filling, but it's safer to disable it. + if pendingNonce-latestNonce > maxInFlightTransactions || t.nonce.Load() > pendingNonce { + t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, LatestNonce: %d, maxInFlightTransactions: %d", + t.nonce.Load(), pendingNonce, latestNonce, maxInFlightTransactions) + return + } + + tx, err := t.storage.UpdateUnstartedTransactionWithNonce(context.TODO(), t.address, t.nonce.Load()) + if err != nil { + return + } + if tx == nil { + return + } + tx.Nonce = t.nonce.Load() + tx.State = types.TxUnconfirmed + t.nonce.Add(1) + + return t.createAndSendAttempt(tx) +} + +func (t *Txm) createAndSendAttempt(tx *types.Transaction) error { + attempt, err := t.attemptBuilder.NewAttempt(context.TODO(), t.lggr, tx, t.config.EIP1559) + if err != nil { + return err + } + + if err = t.storage.AppendAttemptToTransaction(context.TODO(), tx.Nonce, attempt); err != nil { + return err + } + + return t.sendTransactionWithError(tx, attempt) +} + +func (t *Txm) sendTransactionWithError(tx *types.Transaction, attempt *types.Attempt) (err error) { + txErr := t.client.SendTransaction(context.TODO(), attempt.SignedTransaction) + tx.AttemptCount++ + t.lggr.Infof("Broadcasted attempt", "tx", tx, "attempt", attempt, "txErr: ", txErr) + if txErr != nil && t.errorHandler != nil { + if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.storage); err != nil { + return + } + } else if txErr != nil { + pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + if err != nil { + return err + } + if pendingNonce > tx.Nonce { + return nil + } + t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, tx.Nonce) + return nil + } + + return t.storage.UpdateTransactionBroadcast(context.TODO(), attempt.TxID, tx.Nonce, attempt.Hash) +} + +func (t *Txm) backfillTransactions() error { + latestNonce, err := t.client.NonceAt(context.TODO(), t.address, nil) + if err != nil { + return err + } + + // TODO: Update LastBroadcast(?) + confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.storage.MarkTransactionsConfirmed(context.TODO(), latestNonce, t.address) + if err != nil { + return err + } + t.lggr.Infof("Confirmed transactions: %v . Re-orged transactions: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) + + tx, unconfirmedCount, err := t.storage.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), latestNonce, t.address) + if err != nil { + return err + } + if unconfirmedCount == 0 { + pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + if err != nil { + return err + } + // if local nonce is incorrect, we need to fill the gap to start new transactions + count, err := t.storage.CountUnstartedTransactions(context.TODO(), t.address) + if err != nil { + return err + } + if t.nonce.Load() <= pendingNonce && count == 0 { + t.lggr.Debugf("All transactions confirmed for address: %v", t.address) + return nil + } + } + + if tx == nil || tx.Nonce != latestNonce { + t.lggr.Warn("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, t.address) + return t.createAndSendEmptyTx(latestNonce) + } else { + if !tx.IsPurgeable && t.stuckTxDetector != nil { + isStuck, err := t.stuckTxDetector.DetectStuckTransactions(tx) + if err != nil { + return err + } + if isStuck { + tx.IsPurgeable = true + t.storage.MarkUnconfirmedTransactionPurgeable(context.TODO(), tx.Nonce) + t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for tx: ", tx.ID, tx) + return t.createAndSendAttempt(tx) + } + } + if (time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero()) && + tx.AttemptCount < maxAllowedAttempts { + // TODO: add graceful bumping + t.lggr.Infow("Rebroadcasting attempt for tx: ", tx) + return t.createAndSendAttempt(tx) + } + } + return nil +} + +func (t *Txm) createAndSendEmptyTx(latestNonce uint64) error { + tx, err := t.storage.CreateEmptyUnconfirmedTransaction(context.TODO(), t.address, t.chainID, latestNonce, t.config.EmptyTxLimitDefault) + if err != nil { + return err + } + return t.createAndSendAttempt(tx) +} + +func (t *Txm) pendingAndLatestNonce(ctx context.Context, fromAddress common.Address) (pending uint64, latest uint64, err error) { + pendingS, latestS := new(string), new(string) + reqs := []rpc.BatchElem{ + {Method: "eth_getTransactionCount", Args: []interface{}{fromAddress, "pending"}, Result: &pendingS}, + {Method: "eth_getTransactionCount", Args: []interface{}{fromAddress, "latest"}, Result: &latestS}, + } + + if err = t.client.BatchCallContext(ctx, reqs); err != nil { + return + } + + for _, response := range reqs { + if response.Error != nil { + return 0, 0, response.Error + } + } + + if pending, err = hexutil.DecodeUint64(*pendingS); err != nil { + return + } + if latest, err = hexutil.DecodeUint64(*latestS); err != nil { + return + } + + if pending < latest { + return 0, 0, fmt.Errorf("RPC nonce state out of sync. Pending: %d, Latest: %d", pending, latest) + } + + return pending, latest, err +} diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go new file mode 100644 index 00000000000..e8f77a85d5f --- /dev/null +++ b/core/chains/evm/txm/txm_test.go @@ -0,0 +1,162 @@ +package txm + +import ( + "errors" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.uber.org/zap" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/mocks" +) + +func TestLifecycle(t *testing.T) { + t.Parallel() + + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + storage := mocks.NewStorage(t) + config := Config{} + address := testutils.NewAddress() + + t.Run("fails to start if pending nonce call fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), errors.New("error")).Once() + assert.Error(t, txm.Start(tests.Context(t))) + }) + + t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, storage, config, address) + var nonce uint64 = 0 + // Start + client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() + // broadcast loop (may or may not be executed multiple times) + client.On("BatchCallContext", mock.Anything, mock.Anything).Return(nil) + storage.On("UpdateUnstartedTransactionWithNonce", mock.Anything, address, mock.Anything).Return(nil, nil) + // backfill loop (may or may not be executed multiple times) + client.On("NonceAt", mock.Anything, address, nil).Return(nonce, nil) + storage.On("MarkTransactionsConfirmed", mock.Anything, nonce, address).Return([]uint64{}, []uint64{}, nil) + storage.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, nonce, address).Return(nil, 0) + client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil) + storage.On("CountUnstartedTransactions", mock.Anything, address).Return(0) + + assert.NoError(t, txm.Start(tests.Context(t))) + assert.NoError(t, txm.Close()) + }) + +} + +func TestTrigger(t *testing.T) { + t.Parallel() + + t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { + txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, common.Address{}) + txm.Trigger() + assert.Error(t, txm.Trigger(), "Txm unstarted") + }) +} + + +func TestBroadcastTransaction(t *testing.T) { + t.Parallel() + + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + storage := mocks.NewStorage(t) + config := Config{} + address := testutils.NewAddress() + + t.Run("fails if batch call for pending and latest nonce fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + client.On("BatchCallContext", mock.Anything, mock.Anything).Return(errors.New("batch call error")).Once() + err := txm.broadcastTransaction() + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch call error") + }) + + t.Run("fails if batch call for pending and latest nonce fails for one of them", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + //pending nonce + client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Error = errors.New("pending nonce failed") + }).Return(nil).Once() + err := txm.broadcastTransaction() + assert.Error(t, err) + assert.Contains(t, err.Error(), "pending nonce failed") + + // latest nonce + client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[1].Error = errors.New("latest nonce failed") + }).Return(nil).Once() + err = txm.broadcastTransaction() + assert.Error(t, err) + assert.Contains(t, err.Error(), "latest nonce failed") + }) + + t.Run("throws a warning if maxInFlightTransactions are reached", func(t *testing.T) { + pending := "0x100" + latest := "0x0" + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &pending // pending + elems[1].Result = &latest // latest + }).Return(nil).Once() + err := txm.broadcastTransaction() + assert.Error(t, err) + assert.Contains(t, err.Error(), "Reached transaction limit") + + }) + t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { + pending := "0x8" + latest := "0x0" + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + txm.nonce.Store(0) + client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &pending // pending + elems[1].Result = &latest // latest + }).Return(nil).Once() + storage.On("UpdateUnstartedTransactionWithNonce", mock.Anything, address, mock.Anything).Return(nil, errors.New("update failed")) + err := txm.broadcastTransaction() + assert.Error(t, err) + assert.Contains(t, err.Error(), "update failed") + }) +} + +func TestBackfillTransactions(t *testing.T) { + t.Parallel() + + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + storage := mocks.NewStorage(t) + config := Config{} + address := testutils.NewAddress() + + t.Run("fails if latest nonce fetching fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() + err := txm.backfillTransactions() + assert.Error(t, err) + assert.Contains(t, err.Error(), "latest nonce fail") + }) + + t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil) + storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")) + err := txm.backfillTransactions() + assert.Error(t, err) + assert.Contains(t, err.Error(), "marking transactions confirmed failed") + }) +} diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go new file mode 100644 index 00000000000..9a3ddb0db9f --- /dev/null +++ b/core/chains/evm/txm/types/transaction.go @@ -0,0 +1,84 @@ +package types + +import ( + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" +) + +type TxState string + +const ( + TxUnstarted = TxState("unstarted") + TxUnconfirmed = TxState("unconfirmed") + TxConfirmed = TxState("confirmed") + + TxFatalError = TxState("fatal") + TxFinalized = TxState("finalized") +) + +type Transaction struct { + ID uint64 + IdempotencyKey *string + ChainID *big.Int + Nonce uint64 + FromAddress common.Address + ToAddress common.Address + Value *big.Int + Data []byte + SpecifiedGasLimit uint64 + + CreatedAt time.Time + LastBroadcastAt time.Time + + State TxState + IsPurgeable bool + Attempts []*Attempt + AttemptCount uint16 // AttempCount is strictly kept inMemory and prevents indefinite retrying + // Meta, ForwarderAddress, Strategy +} + +func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, error) { + for _, a := range t.Attempts { + if a.Hash == attemptHash { + return a, nil + } + } + return nil, fmt.Errorf("attempt with hash: %v was not found", attemptHash) +} + +func (t *Transaction) DeepCopy() *Transaction { + copy := *t + var attemptsCopy []*Attempt + for _, attempt := range t.Attempts { + attemptsCopy = append(attemptsCopy, attempt.DeepCopy()) + } + copy.Attempts = attemptsCopy + return © +} + +type Attempt struct { + ID uint64 + TxID uint64 + Hash common.Hash + Fee gas.EvmFee + GasLimit uint64 + Type byte + SignedTransaction *types.Transaction + + CreatedAt time.Time + BroadcastAt time.Time +} + +func (a *Attempt) DeepCopy() *Attempt { + copy := *a + if a.SignedTransaction != nil { + signedTransactionCopy := *a.SignedTransaction + copy.SignedTransaction = &signedTransactionCopy + } + return © +} From 174c605982e0a2bd9ca202603530fee2e9371855 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 14 Oct 2024 15:57:00 +0300 Subject: [PATCH 02/73] TXM fixes --- core/chains/evm/txm/txm.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index bb345b50227..808fbd9de8c 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -128,6 +128,10 @@ func (t *Txm) Close() error { }) } +func (t *Txm) CreateTransaction(ctx context.Context, tx *types.Transaction) (uint64, error) { + return t.storage.CreateTransaction(ctx, tx) +} + func (t *Txm) Trigger() error { if !t.IfStarted(func() { t.triggerCh <- struct{}{} From adecb3c6e9dae8e299c3ea5bf9a5a375439f610d Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 14 Oct 2024 16:51:24 +0300 Subject: [PATCH 03/73] Fix --- core/chains/evm/txm/attempt_builder.go | 1 + core/chains/evm/txm/dummy_keystore.go | 4 +- core/chains/evm/txm/mocks/client.go | 49 --- core/chains/evm/txm/mocks/storage.go | 24 +- core/chains/evm/txm/orchestrator.go | 326 ++++++++++++++++-- core/chains/evm/txm/storage/inmemory_store.go | 95 ++++- .../evm/txm/storage/inmemory_store_test.go | 83 ++++- core/chains/evm/txm/txm.go | 166 ++++----- core/chains/evm/txm/types/transaction.go | 87 ++++- 9 files changed, 636 insertions(+), 199 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index de5848e71a7..e5be133d25b 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -29,6 +29,7 @@ type attemptBuilder struct { func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore Keystore) *attemptBuilder { return &attemptBuilder{ chainID: chainID, + priceMax: priceMax, estimator: estimator, keystore: keystore, } diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 795d4fa186c..5e895bc15fa 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -14,7 +14,7 @@ type DummyKeystore struct { privateKey *ecdsa.PrivateKey } -func NewKeystore(privateKeyString string) *DummyKeystore { +func NewKeystore() *DummyKeystore { return &DummyKeystore{} } @@ -28,5 +28,5 @@ func (k *DummyKeystore) Add(privateKeyString string) error { } func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - return types.SignTx(tx, types.NewEIP155Signer(chainID), k.privateKey) + return types.SignTx(tx, types.LatestSignerForChainID(chainID), k.privateKey) } diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go index f90f362a83d..334580587e5 100644 --- a/core/chains/evm/txm/mocks/client.go +++ b/core/chains/evm/txm/mocks/client.go @@ -10,8 +10,6 @@ import ( mock "github.com/stretchr/testify/mock" - rpc "github.com/ethereum/go-ethereum/rpc" - types "github.com/ethereum/go-ethereum/core/types" ) @@ -28,53 +26,6 @@ func (_m *Client) EXPECT() *Client_Expecter { return &Client_Expecter{mock: &_m.Mock} } -// BatchCallContext provides a mock function with given fields: _a0, _a1 -func (_m *Client) BatchCallContext(_a0 context.Context, _a1 []rpc.BatchElem) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for BatchCallContext") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []rpc.BatchElem) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Client_BatchCallContext_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BatchCallContext' -type Client_BatchCallContext_Call struct { - *mock.Call -} - -// BatchCallContext is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 []rpc.BatchElem -func (_e *Client_Expecter) BatchCallContext(_a0 interface{}, _a1 interface{}) *Client_BatchCallContext_Call { - return &Client_BatchCallContext_Call{Call: _e.mock.On("BatchCallContext", _a0, _a1)} -} - -func (_c *Client_BatchCallContext_Call) Run(run func(_a0 context.Context, _a1 []rpc.BatchElem)) *Client_BatchCallContext_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].([]rpc.BatchElem)) - }) - return _c -} - -func (_c *Client_BatchCallContext_Call) Return(_a0 error) *Client_BatchCallContext_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Client_BatchCallContext_Call) RunAndReturn(run func(context.Context, []rpc.BatchElem) error) *Client_BatchCallContext_Call { - _c.Call.Return(run) - return _c -} - // NonceAt provides a mock function with given fields: _a0, _a1, _a2 func (_m *Client) NonceAt(_a0 context.Context, _a1 common.Address, _a2 *big.Int) (uint64, error) { ret := _m.Called(_a0, _a1, _a2) diff --git a/core/chains/evm/txm/mocks/storage.go b/core/chains/evm/txm/mocks/storage.go index 4962fabd471..18b180cd617 100644 --- a/core/chains/evm/txm/mocks/storage.go +++ b/core/chains/evm/txm/mocks/storage.go @@ -241,25 +241,27 @@ func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) RunAndReturn(run func( } // CreateTransaction provides a mock function with given fields: _a0, _a1 -func (_m *Storage) CreateTransaction(_a0 context.Context, _a1 *types.Transaction) (uint64, error) { +func (_m *Storage) CreateTransaction(_a0 context.Context, _a1 *types.TxRequest) (*types.Transaction, error) { ret := _m.Called(_a0, _a1) if len(ret) == 0 { panic("no return value specified for CreateTransaction") } - var r0 uint64 + var r0 *types.Transaction var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) (uint64, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *types.TxRequest) (*types.Transaction, error)); ok { return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) uint64); ok { + if rf, ok := ret.Get(0).(func(context.Context, *types.TxRequest) *types.Transaction); ok { r0 = rf(_a0, _a1) } else { - r0 = ret.Get(0).(uint64) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Transaction) + } } - if rf, ok := ret.Get(1).(func(context.Context, *types.Transaction) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *types.TxRequest) error); ok { r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) @@ -275,24 +277,24 @@ type Storage_CreateTransaction_Call struct { // CreateTransaction is a helper method to define mock.On call // - _a0 context.Context -// - _a1 *types.Transaction +// - _a1 *types.TxRequest func (_e *Storage_Expecter) CreateTransaction(_a0 interface{}, _a1 interface{}) *Storage_CreateTransaction_Call { return &Storage_CreateTransaction_Call{Call: _e.mock.On("CreateTransaction", _a0, _a1)} } -func (_c *Storage_CreateTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Storage_CreateTransaction_Call { +func (_c *Storage_CreateTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.TxRequest)) *Storage_CreateTransaction_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Transaction)) + run(args[0].(context.Context), args[1].(*types.TxRequest)) }) return _c } -func (_c *Storage_CreateTransaction_Call) Return(_a0 uint64, _a1 error) *Storage_CreateTransaction_Call { +func (_c *Storage_CreateTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *Storage_CreateTransaction_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Storage_CreateTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction) (uint64, error)) *Storage_CreateTransaction_Call { +func (_c *Storage_CreateTransaction_Call) RunAndReturn(run func(context.Context, *types.TxRequest) (*types.Transaction, error)) *Storage_CreateTransaction_Call { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index c89f73b8899..9ef48018b7d 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -2,37 +2,317 @@ package txm import ( "context" + "encoding/json" + "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" nullv4 "gopkg.in/guregu/null.v4" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-common/pkg/utils" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + "github.com/smartcontractkit/chainlink/v2/common/txmgr" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/common/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/forwarders" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + txmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) -type TxmOrchestrator interface { - services.Service - Trigger(addr common.Address) - CreateTransaction(ctx context.Context, txRequest *types.Transaction) (id int64, err error) - GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) - GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) - RegisterResumeCallback(fn ResumeCallback) - SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value *big.Int, gasLimit uint64) (tx *types.Transaction, err error) - CountTransactionsByState(ctx context.Context, state types.TxState) (count int, err error) - GetTransactionStatus(ctx context.Context, idempotencyKey string) (state commontypes.TransactionStatus, err error) - //Reset(addr ADDR, abandon bool) error // Potentially will be replaced by Abandon - - // Testing methods(?) - FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) - FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) - FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*types.Transaction, err error) - FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []types.TxState, chainID *big.Int) (txes []*types.Transaction, err error) - FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) - FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) -} - -type ResumeCallback func(ctx context.Context, id uuid.UUID, result interface{}, err error) error +// TODO: use this after the migration +//type TxmOrchestrator interface { +// services.Service +// Trigger(addr common.Address) +// CreateTransaction(ctx context.Context, txRequest *types.Transaction) (id int64, err error) +// GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) +// GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) +// RegisterResumeCallback(fn ResumeCallback) +// SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value *big.Int, gasLimit uint64) (tx *types.Transaction, err error) +// CountTransactionsByState(ctx context.Context, state types.TxState) (count int, err error) +// GetTransactionStatus(ctx context.Context, idempotencyKey string) (state commontypes.TransactionStatus, err error) +// //Reset(addr ADDR, abandon bool) error // Potentially will be replaced by Abandon +// +// // Testing methods(?) +// FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) +// FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) +// FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*types.Transaction, err error) +// FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []types.TxState, chainID *big.Int) (txes []*types.Transaction, err error) +// FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) +// FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) +//} + +type OrchestratorTxStore interface { + FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*txmtypes.Transaction, int, error) + FindTxWithIdempotencyKey(context.Context, *string) (*txmtypes.Transaction, error) +} + +type Orchestrator struct { + services.StateMachine + lggr logger.SugaredLogger + chainID *big.Int + txm *Txm + txStore OrchestratorTxStore + fwdMgr *forwarders.FwdMgr + resumeCallback txmgr.ResumeCallback +} + +func NewTxmOrchestrator( + lggr logger.Logger, + chainID *big.Int, + txm *Txm, + txStore OrchestratorTxStore, + fwdMgr *forwarders.FwdMgr, +) txmgr.TxManager[*big.Int, types.Head[common.Hash], common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] { + return &Orchestrator{ + lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), + chainID: chainID, + txm: txm, + txStore: txStore, + fwdMgr: fwdMgr, + } +} + +func (o *Orchestrator) Start(ctx context.Context) error { + return o.StartOnce("Orchestrator", func() error { + var ms services.MultiStart + if err := ms.Start(ctx, o.txm); err != nil { + return fmt.Errorf("Orchestrator: Txm failed to start: %w", err) + } + if o.fwdMgr != nil { + if err := ms.Start(ctx, o.fwdMgr); err != nil { + return fmt.Errorf("Orchestrator: ForwarderManager failed to start: %w", err) + } + } + return nil + }) +} + +func (o *Orchestrator) Close() (merr error) { + return o.StopOnce("Orchestrator", func() error { + if o.fwdMgr != nil { + if err := o.fwdMgr.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop ForwarderManager: %w", err)) + } + } + if err := o.txm.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop Txm: %w", err)) + } + return merr + }) +} + +func (o *Orchestrator) Trigger(addr common.Address) { + if err := o.txm.Trigger(); err != nil { + o.lggr.Error(err) + } +} + +func (o *Orchestrator) Name() string { + return o.lggr.Name() +} + +func (o *Orchestrator) HealthReport() map[string]error { + return map[string]error{o.Name(): o.Healthy()} +} + +func (o *Orchestrator) RegisterResumeCallback(fn txmgr.ResumeCallback) { + o.resumeCallback = fn +} + +func (o *Orchestrator) Reset(addr common.Address, abandon bool) error { + ok := o.IfStarted(func() { + if err := o.txm.Abandon(); err != nil { + o.lggr.Error(err) + } + }) + if !ok { + return fmt.Errorf("Orchestrator not started yet") + } + return nil +} + +func (o *Orchestrator) OnNewLongestChain(ctx context.Context, head types.Head[common.Hash]) { +} + +func (o *Orchestrator) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + // TODO: Idempotency + var wrappedTx *txmtypes.Transaction + wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(context.TODO(), request.IdempotencyKey) + if err != nil { + return + } + + if wrappedTx != nil { + o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", *wrappedTx.IdempotencyKey) + } else { + var pipelineTaskRunID uuid.NullUUID + if request.PipelineTaskRunID != nil { + pipelineTaskRunID.UUID = *request.PipelineTaskRunID + pipelineTaskRunID.Valid = true + } + + if o.fwdMgr != nil && (!utils.IsZero(request.ForwarderAddress)) { + fwdPayload, fwdErr := o.fwdMgr.ConvertPayload(request.ToAddress, request.EncodedPayload) + if fwdErr == nil { + // Handling meta not set at caller. + if request.Meta != nil { + request.Meta.FwdrDestAddress = &request.ToAddress + } else { + request.Meta = &txmgrtypes.TxMeta[common.Address, common.Hash]{ + FwdrDestAddress: &request.ToAddress, + } + } + request.ToAddress = request.ForwarderAddress + request.EncodedPayload = fwdPayload + } else { + o.lggr.Errorf("Failed to use forwarder set upstream: %v", fwdErr.Error()) + } + } + + var meta *sqlutil.JSON + if request.Meta != nil { + raw, err := json.Marshal(request.Meta) + if err != nil { + return tx, err + } + m := sqlutil.JSON(raw) + meta = &m + } + + wrappedTxRequest := &txmtypes.TxRequest{ + IdempotencyKey: request.IdempotencyKey, + ChainID: o.chainID, + FromAddress: request.FromAddress, + ToAddress: request.ToAddress, + Value: &request.Value, + Data: request.EncodedPayload, + SpecifiedGasLimit: request.FeeLimit, + Meta: meta, + ForwarderAddress: request.ForwarderAddress, + + PipelineTaskRunID: pipelineTaskRunID, + MinConfirmations: request.MinConfirmations, + SignalCallback: request.SignalCallback, + } + + wrappedTx, err = o.txm.CreateTransaction(ctx, wrappedTxRequest) + if err != nil { + return + } + o.txm.Trigger() + } + + sequence := evmtypes.Nonce(wrappedTx.Nonce) + tx = txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]{ + ID: int64(wrappedTx.ID), + IdempotencyKey: wrappedTx.IdempotencyKey, + Sequence: &sequence, + FromAddress: wrappedTx.FromAddress, + ToAddress: wrappedTx.ToAddress, + EncodedPayload: wrappedTx.Data, + Value: *wrappedTx.Value, + FeeLimit: wrappedTx.SpecifiedGasLimit, + CreatedAt: wrappedTx.CreatedAt, + Meta: wrappedTx.Meta, + //Subject: wrappedTx.Subject, + + //TransmitChecker: wrappedTx.TransmitChecker, + ChainID: wrappedTx.ChainID, + + PipelineTaskRunID: wrappedTx.PipelineTaskRunID, + MinConfirmations: wrappedTx.MinConfirmations, + SignalCallback: wrappedTx.SignalCallback, + CallbackCompleted: wrappedTx.CallbackCompleted, + } + return +} + +func (o *Orchestrator) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (uint32, error) { + _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, common.Address{}) + return uint32(count), err +} + +func (o *Orchestrator) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { + return +} + +func (o *Orchestrator) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (time nullv4.Int, err error) { + return +} + +func (o *Orchestrator) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} + +func (o *Orchestrator) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} +func (o *Orchestrator) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} +func (o *Orchestrator) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + return +} + +func (o *Orchestrator) GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) { + if o.fwdMgr != nil { + forwarder, err = o.fwdMgr.ForwarderFor(ctx, eoa) + } + return +} + +func (o *Orchestrator) GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) { + if o.fwdMgr != nil { + forwarder, err = o.fwdMgr.ForwarderForOCR2Feeds(ctx, eoa, ocr2AggregatorID) + } + return +} + +func (o *Orchestrator) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { + // Loads attempts and receipts in the transaction + tx, err := o.txStore.FindTxWithIdempotencyKey(ctx, &transactionID) + if err != nil || tx == nil { + return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s: %w", transactionID, err) + } + + switch tx.State { + case txmtypes.TxUnconfirmed: + return commontypes.Pending, nil + case txmtypes.TxConfirmed: + // Return unconfirmed for confirmed transactions because they are not yet finalized + return commontypes.Unconfirmed, nil + case txmtypes.TxFinalized: + return commontypes.Finalized, nil + case txmtypes.TxFatalError: + return commontypes.Fatal, nil + default: + return commontypes.Unknown, nil + } +} + +func (o *Orchestrator) SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value big.Int, gasLimit uint64) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + txRequest := txmgrtypes.TxRequest[common.Address, common.Hash]{ + FromAddress: from, + ToAddress: to, + EncodedPayload: []byte{}, + Value: value, + FeeLimit: gasLimit, + //Strategy: NewSendEveryStrategy(), + } + + tx, err = o.CreateTransaction(ctx, txRequest) + if err != nil { + return + } + + // Trigger the Txm to check for new transaction + err = o.txm.Trigger() + return tx, err +} diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 12df2130502..9b88a5ccbde 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "sort" "sync" "time" @@ -13,6 +14,11 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) +const ( + maxQueuedTransactions = 250 + pruneSubset = 3 +) + type InMemoryStore struct { sync.RWMutex lggr logger.Logger @@ -22,6 +28,8 @@ type InMemoryStore struct { UnconfirmedTransactions map[uint64]*types.Transaction ConfirmedTransactions map[uint64]*types.Transaction FatalTransactions []*types.Transaction + + Transactions map[uint64]*types.Transaction } func NewInMemoryStore(lggr logger.Logger) *InMemoryStore { @@ -29,6 +37,7 @@ func NewInMemoryStore(lggr logger.Logger) *InMemoryStore { lggr: logger.Named(lggr, "InMemoryStore"), UnconfirmedTransactions: make(map[uint64]*types.Transaction), ConfirmedTransactions: make(map[uint64]*types.Transaction), + Transactions: make(map[uint64]*types.Transaction), } } @@ -65,6 +74,8 @@ func (m *InMemoryStore) AppendAttemptToTransaction(_ context.Context, txNonce ui } attempt.CreatedAt = time.Now() + attempt.ID = uint64(len(tx.Attempts)) // Attempts are not collectively tracked by the in-memory store so attemptIDs are not unique between transactions and can be reused. + tx.AttemptCount++ m.UnconfirmedTransactions[txNonce].Attempts = append(m.UnconfirmedTransactions[txNonce].Attempts, attempt.DeepCopy()) return nil @@ -99,22 +110,45 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(ctx context.Context, f } m.UnconfirmedTransactions[nonce] = emptyTx + m.Transactions[emptyTx.ID] = emptyTx return emptyTx.DeepCopy(), nil } -func (m *InMemoryStore) CreateTransaction(_ context.Context, tx *types.Transaction) (uint64, error) { +func (m *InMemoryStore) CreateTransaction(_ context.Context, txRequest *types.TxRequest) (*types.Transaction, error) { m.Lock() defer m.Unlock() m.txIDCount++ - tx.ID = m.txIDCount - tx.CreatedAt = time.Now() - tx.State = types.TxUnstarted + tx := &types.Transaction{ + ID: m.txIDCount, + IdempotencyKey: txRequest.IdempotencyKey, + ChainID: txRequest.ChainID, + FromAddress: txRequest.FromAddress, + ToAddress: txRequest.ToAddress, + Value: txRequest.Value, + Data: txRequest.Data, + SpecifiedGasLimit: txRequest.SpecifiedGasLimit, + CreatedAt: time.Now(), + State: types.TxUnstarted, + Meta: txRequest.Meta, + MinConfirmations: txRequest.MinConfirmations, + PipelineTaskRunID: txRequest.PipelineTaskRunID, + SignalCallback: txRequest.SignalCallback, + } - m.UnstartedTransactions = append(m.UnstartedTransactions, tx.DeepCopy()) - return tx.ID, nil + if len(m.UnstartedTransactions) == maxQueuedTransactions { + m.lggr.Warnf("Unstarted transactions queue reached max limit of: %d. Dropping oldest transaction: %v.", + maxQueuedTransactions, m.UnstartedTransactions[0]) + delete(m.Transactions, m.UnstartedTransactions[0].ID) + m.UnstartedTransactions = m.UnstartedTransactions[1:maxQueuedTransactions] + } + + copy := tx.DeepCopy() + m.Transactions[copy.ID] = copy + m.UnstartedTransactions = append(m.UnstartedTransactions, copy) + return tx, nil } func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(_ context.Context, latestNonce uint64, _ common.Address) (txCopy *types.Transaction, unconfirmedCount int, err error) { @@ -147,11 +181,19 @@ func (m *InMemoryStore) MarkTransactionsConfirmed(_ context.Context, latestNonce for _, tx := range m.ConfirmedTransactions { if tx.Nonce >= latestNonce { tx.State = types.TxUnconfirmed + tx.LastBroadcastAt = time.Time{} // Mark reorged transaction as if it wasn't broadcasted before unconfirmedTransactionIDs = append(unconfirmedTransactionIDs, tx.ID) m.UnconfirmedTransactions[tx.Nonce] = tx delete(m.ConfirmedTransactions, tx.Nonce) } } + + if len(m.ConfirmedTransactions) >= maxQueuedTransactions { + prunedTxIDs := m.pruneConfirmedTransactions() + m.lggr.Debugf("Confirmed transactions map reached max limit of: %d. Pruned 1/3 of the oldest confirmed transactions. TxIDs: %v", maxQueuedTransactions, prunedTxIDs) + } + sort.Slice(confirmedTransactionIDs, func(i, j int) bool { return confirmedTransactionIDs[i] < confirmedTransactionIDs[j] }) + sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) return confirmedTransactionIDs, unconfirmedTransactionIDs, nil } @@ -213,6 +255,31 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(_ context.Context, _ return tx.DeepCopy(), nil } +// Shouldn't call lock because it's being called by a method that already has the lock +func (m *InMemoryStore) pruneConfirmedTransactions() []uint64 { + var noncesToPrune []uint64 + for nonce := range m.ConfirmedTransactions { + noncesToPrune = append(noncesToPrune, nonce) + } + if len(noncesToPrune) <= 0 { + return nil + } + sort.Slice(noncesToPrune, func(i, j int) bool { return noncesToPrune[i] < noncesToPrune[j] }) + minNonce := noncesToPrune[len(noncesToPrune)/pruneSubset] + + var txIDsToPrune []uint64 + for nonce, tx := range m.ConfirmedTransactions { + if nonce < minNonce { + txIDsToPrune = append(txIDsToPrune, tx.ID) + delete(m.Transactions, tx.ID) + delete(m.ConfirmedTransactions, nonce) + } + } + + sort.Slice(txIDsToPrune, func(i, j int) bool { return txIDsToPrune[i] < txIDsToPrune[j] }) + return txIDsToPrune +} + // Error Handler func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(_ context.Context, transactionNonce uint64, attempt *types.Attempt) error { m.Lock() @@ -236,3 +303,19 @@ func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(_ context.Context, transac func (m *InMemoryStore) MarkTxFatal(context.Context, *types.Transaction) error { return fmt.Errorf("not implemented") } + +// Orchestrator +func (m *InMemoryStore) FindTxWithIdempotencyKey(_ context.Context, idempotencyKey *string) (*types.Transaction, error) { + m.Lock() + defer m.Unlock() + + if idempotencyKey != nil { + for _, tx := range m.Transactions { + if tx.IdempotencyKey != nil && tx.IdempotencyKey == idempotencyKey { + return tx.DeepCopy(), nil + } + } + } + + return nil, nil +} diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 7eb2be0864a..95d01c39663 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -16,6 +16,7 @@ import ( ) func TestAbandonPendingTransactions(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t)) @@ -61,6 +62,7 @@ func TestAbandonPendingTransactions(t *testing.T) { } func TestAppendAttemptToTransaction(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t)) @@ -97,6 +99,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { } func TestCountUnstartedTransactions(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t)) @@ -110,6 +113,7 @@ func TestCountUnstartedTransactions(t *testing.T) { } func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t)) @@ -129,26 +133,52 @@ func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { } func TestCreateTransaction(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) - tx1 := &types.Transaction{} - tx2 := &types.Transaction{} - id1, err := m.CreateTransaction(tests.Context(t), tx1) - assert.NoError(t, err) - assert.Equal(t, uint64(1), id1) + t.Run("creates new transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + now := time.Now() + txR1 := &types.TxRequest{} + txR2 := &types.TxRequest{} + tx1, err := m.CreateTransaction(tests.Context(t), txR1) + assert.NoError(t, err) + assert.Equal(t, uint64(1), tx1.ID) + assert.Less(t, now, tx1.CreatedAt) - id2, err := m.CreateTransaction(tests.Context(t), tx2) - assert.NoError(t, err) - assert.Equal(t, uint64(2), id2) + tx2, err := m.CreateTransaction(tests.Context(t), txR2) + assert.NoError(t, err) + assert.Equal(t, uint64(2), tx2.ID) + assert.Less(t, now, tx2.CreatedAt) - count, _ := m.CountUnstartedTransactions(tests.Context(t), fromAddress) - assert.Equal(t, count, 2) + count, _ := m.CountUnstartedTransactions(tests.Context(t), fromAddress) + assert.Equal(t, count, 2) + }) + + t.Run("prunes oldest unstarted transactions if limit is reached", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + overshot := 5 + for i := 1; i < maxQueuedTransactions+overshot; i++ { + r := &types.TxRequest{} + tx, err := m.CreateTransaction(tests.Context(t), r) + assert.NoError(t, err) + assert.Equal(t, uint64(i), tx.ID) + } + // total shouldn't exceed maxQueuedTransactions + total, err := m.CountUnstartedTransactions(tests.Context(t), fromAddress) + assert.NoError(t, err) + assert.Equal(t, maxQueuedTransactions, total) + // earliest tx ID should be the same amount of the number of transactions that we dropped + tx, err := m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, 0) + assert.NoError(t, err) + assert.Equal(t, uint64(overshot), tx.ID) + }) } func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t)) @@ -166,6 +196,7 @@ func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { } func TestMarkTransactionsConfirmed(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() @@ -208,9 +239,21 @@ func TestMarkTransactionsConfirmed(t *testing.T) { assert.Equal(t, utxs[0], ctx2.ID) assert.Equal(t, 0, len(ctxs)) }) + t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t)) + for i := 0; i < maxQueuedTransactions; i++ { + _, err := insertConfirmedTransaction(m, fromAddress, uint64(i)) + assert.NoError(t, err) + } + assert.Equal(t, maxQueuedTransactions, len(m.ConfirmedTransactions)) + _, _, err := m.MarkTransactionsConfirmed(tests.Context(t), maxQueuedTransactions, fromAddress) + assert.NoError(t, err) + assert.Equal(t, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset), len(m.ConfirmedTransactions)) + }) } func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t)) @@ -227,6 +270,7 @@ func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { } func TestUpdateTransactionBroadcast(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() hash := testutils.NewHash() @@ -263,6 +307,7 @@ func TestUpdateTransactionBroadcast(t *testing.T) { } func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() t.Run("returns nil if there are no unstarted transactions", func(t *testing.T) { @@ -296,6 +341,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { } func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { + t.Parallel() fromAddress := testutils.NewAddress() t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { @@ -333,6 +379,21 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { }) } +func TestPruneConfirmedTransactions(t *testing.T) { + t.Parallel() + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t)) + total := 5 + for i := 0; i < total; i++ { + _, err := insertConfirmedTransaction(m, fromAddress, uint64(i)) + assert.NoError(t, err) + } + prunedTxIDs := m.pruneConfirmedTransactions() + left := total - total/pruneSubset + assert.Equal(t, left, len(m.ConfirmedTransactions)) + assert.Equal(t, total/pruneSubset, len(prunedTxIDs)) +} + func insertUnstartedTransaction(m *InMemoryStore, fromAddress common.Address) *types.Transaction { m.Lock() defer m.Unlock() diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 808fbd9de8c..9bb9ded92fb 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -9,9 +9,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" evmtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rpc" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -21,7 +19,7 @@ import ( ) const ( - broadcastInterval time.Duration = 30 * time.Second + broadcastInterval time.Duration = 15 * time.Second maxInFlightTransactions uint64 = 16 maxAllowedAttempts uint16 = 10 ) @@ -30,15 +28,13 @@ type Client interface { PendingNonceAt(context.Context, common.Address) (uint64, error) NonceAt(context.Context, common.Address, *big.Int) (uint64, error) SendTransaction(context.Context, *evmtypes.Transaction) error - BatchCallContext(context.Context, []rpc.BatchElem) error } -type Storage interface { +type TxStore interface { AbandonPendingTransactions(context.Context, common.Address) error AppendAttemptToTransaction(context.Context, uint64, *types.Attempt) error - CountUnstartedTransactions(context.Context, common.Address) (int, error) CreateEmptyUnconfirmedTransaction(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error) - CreateTransaction(context.Context, *types.Transaction) (uint64, error) + CreateTransaction(context.Context, *types.TxRequest) (*types.Transaction, error) FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*types.Transaction, int, error) MarkTransactionsConfirmed(context.Context, uint64, common.Address) ([]uint64, []uint64, error) MarkUnconfirmedTransactionPurgeable(context.Context, uint64) error @@ -56,7 +52,7 @@ type AttemptBuilder interface { } type ErrorHandler interface { - HandleError(tx *types.Transaction, message error, attemptBuilder AttemptBuilder, client Client, storage Storage) (err error) + HandleError(tx *types.Transaction, message error, attemptBuilder AttemptBuilder, client Client, txStore TxStore) (err error) } type StuckTxDetector interface { @@ -79,24 +75,24 @@ type Txm struct { attemptBuilder AttemptBuilder errorHandler ErrorHandler stuckTxDetector StuckTxDetector - storage Storage + txStore TxStore config Config nonce atomic.Uint64 triggerCh chan struct{} broadcastStopCh services.StopChan backfillStopCh services.StopChan - wg *sync.WaitGroup + wg sync.WaitGroup } -func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, storage Storage, config Config, address common.Address) *Txm { +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, address common.Address) *Txm { return &Txm{ lggr: logger.Sugared(logger.Named(lggr, "Txm")), address: address, chainID: chainID, client: client, attemptBuilder: attemptBuilder, - storage: storage, + txStore: txStore, config: config, triggerCh: make(chan struct{}), broadcastStopCh: make(chan struct{}), @@ -128,8 +124,12 @@ func (t *Txm) Close() error { }) } -func (t *Txm) CreateTransaction(ctx context.Context, tx *types.Transaction) (uint64, error) { - return t.storage.CreateTransaction(ctx, tx) +func (t *Txm) CreateTransaction(ctx context.Context, txRequest *types.TxRequest) (tx *types.Transaction, err error) { + tx, err = t.txStore.CreateTransaction(ctx, txRequest) + if err == nil { + t.lggr.Infow("Created transaction", "tx", tx) + } + return } func (t *Txm) Trigger() error { @@ -142,12 +142,12 @@ func (t *Txm) Trigger() error { } func (t *Txm) Abandon() error { - return t.storage.AbandonPendingTransactions(context.TODO(), t.address) + return t.txStore.AbandonPendingTransactions(context.TODO(), t.address) } func (t *Txm) broadcastLoop() { defer t.wg.Done() - broadcasterTicker := time.NewTicker(utils.WithJitter(broadcastInterval) * time.Second) + broadcasterTicker := time.NewTicker(utils.WithJitter(broadcastInterval)) defer broadcasterTicker.Stop() for { @@ -157,15 +157,15 @@ func (t *Txm) broadcastLoop() { case <-t.triggerCh: start := time.Now() if err := t.broadcastTransaction(); err != nil { - t.lggr.Errorf("Error during triggered transaction broadcasting %w", err) + t.lggr.Errorf("Error during triggered transaction broadcasting: %v", err) } else { t.lggr.Debug("Triggered transaction broadcasting time elapsed: ", time.Since(start)) } - broadcasterTicker.Reset(utils.WithJitter(broadcastInterval) * time.Second) + broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) case <-broadcasterTicker.C: start := time.Now() if err := t.broadcastTransaction(); err != nil { - t.lggr.Errorf("Error during transaction broadcasting: %w", err) + t.lggr.Errorf("Error during transaction broadcasting: %v", err) } else { t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) } @@ -175,7 +175,7 @@ func (t *Txm) broadcastLoop() { func (t *Txm) backfillLoop() { defer t.wg.Done() - backfillTicker := time.NewTicker(utils.WithJitter(t.config.BlockTime) * time.Second) + backfillTicker := time.NewTicker(utils.WithJitter(t.config.BlockTime)) defer backfillTicker.Stop() for { @@ -185,7 +185,7 @@ func (t *Txm) backfillLoop() { case <-backfillTicker.C: start := time.Now() if err := t.backfillTransactions(); err != nil { - t.lggr.Errorf("Error during backfill: %w", err) + t.lggr.Errorf("Error during backfill: %v", err) } else { t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) } @@ -193,25 +193,37 @@ func (t *Txm) backfillLoop() { } } -func (t *Txm) broadcastTransaction() (err error) { - pendingNonce, latestNonce, err := t.pendingAndLatestNonce(context.TODO(), t.address) +func (t *Txm) broadcastTransaction() error { + _, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), 0, t.address) if err != nil { - return + return err } - // Some clients allow out-of-order nonce filling, but it's safer to disable it. - if pendingNonce-latestNonce > maxInFlightTransactions || t.nonce.Load() > pendingNonce { - t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, LatestNonce: %d, maxInFlightTransactions: %d", - t.nonce.Load(), pendingNonce, latestNonce, maxInFlightTransactions) - return + // Optimistically send up to 1/3 of the maxInFlightTransactions. After that threshold, broadcast more cautiously + // by checking the pending nonce so no more than maxInFlightTransactions/3 can get stuck simultaneously i.e. due + // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary + // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. + if unconfirmedCount >= int(maxInFlightTransactions)/3 { + if unconfirmedCount > int(maxInFlightTransactions) { + t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) + return nil + } + pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + if err != nil { + return err + } + if t.nonce.Load() > pendingNonce { + t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", + t.nonce.Load(), pendingNonce, unconfirmedCount) + } } - tx, err := t.storage.UpdateUnstartedTransactionWithNonce(context.TODO(), t.address, t.nonce.Load()) + tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(context.TODO(), t.address, t.nonce.Load()) if err != nil { - return + return err } if tx == nil { - return + return err } tx.Nonce = t.nonce.Load() tx.State = types.TxUnconfirmed @@ -226,7 +238,7 @@ func (t *Txm) createAndSendAttempt(tx *types.Transaction) error { return err } - if err = t.storage.AppendAttemptToTransaction(context.TODO(), tx.Nonce, attempt); err != nil { + if err = t.txStore.AppendAttemptToTransaction(context.TODO(), tx.Nonce, attempt); err != nil { return err } @@ -236,9 +248,9 @@ func (t *Txm) createAndSendAttempt(tx *types.Transaction) error { func (t *Txm) sendTransactionWithError(tx *types.Transaction, attempt *types.Attempt) (err error) { txErr := t.client.SendTransaction(context.TODO(), attempt.SignedTransaction) tx.AttemptCount++ - t.lggr.Infof("Broadcasted attempt", "tx", tx, "attempt", attempt, "txErr: ", txErr) + t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { - if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.storage); err != nil { + if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore); err != nil { return } } else if txErr != nil { @@ -246,14 +258,13 @@ func (t *Txm) sendTransactionWithError(tx *types.Transaction, attempt *types.Att if err != nil { return err } - if pendingNonce > tx.Nonce { + if pendingNonce <= tx.Nonce { + t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, tx.Nonce) return nil } - t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, tx.Nonce) - return nil } - return t.storage.UpdateTransactionBroadcast(context.TODO(), attempt.TxID, tx.Nonce, attempt.Hash) + return t.txStore.UpdateTransactionBroadcast(context.TODO(), attempt.TxID, tx.Nonce, attempt.Hash) } func (t *Txm) backfillTransactions() error { @@ -262,35 +273,25 @@ func (t *Txm) backfillTransactions() error { return err } - // TODO: Update LastBroadcast(?) - confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.storage.MarkTransactionsConfirmed(context.TODO(), latestNonce, t.address) + confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(context.TODO(), latestNonce, t.address) if err != nil { return err } - t.lggr.Infof("Confirmed transactions: %v . Re-orged transactions: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) + if len(confirmedTransactionIDs) > 0 || len(unconfirmedTransactionIDs) > 0 { + t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) + } - tx, unconfirmedCount, err := t.storage.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), latestNonce, t.address) + tx, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), latestNonce, t.address) if err != nil { return err } if unconfirmedCount == 0 { - pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) - if err != nil { - return err - } - // if local nonce is incorrect, we need to fill the gap to start new transactions - count, err := t.storage.CountUnstartedTransactions(context.TODO(), t.address) - if err != nil { - return err - } - if t.nonce.Load() <= pendingNonce && count == 0 { - t.lggr.Debugf("All transactions confirmed for address: %v", t.address) - return nil - } + t.lggr.Debugf("All transactions confirmed for address: %v", t.address) + return nil } if tx == nil || tx.Nonce != latestNonce { - t.lggr.Warn("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, t.address) + t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, t.address) return t.createAndSendEmptyTx(latestNonce) } else { if !tx.IsPurgeable && t.stuckTxDetector != nil { @@ -300,15 +301,21 @@ func (t *Txm) backfillTransactions() error { } if isStuck { tx.IsPurgeable = true - t.storage.MarkUnconfirmedTransactionPurgeable(context.TODO(), tx.Nonce) - t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for tx: ", tx.ID, tx) + t.txStore.MarkUnconfirmedTransactionPurgeable(context.TODO(), tx.Nonce) + t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: ", tx.ID) return t.createAndSendAttempt(tx) } } - if (time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero()) && - tx.AttemptCount < maxAllowedAttempts { - // TODO: add graceful bumping - t.lggr.Infow("Rebroadcasting attempt for tx: ", tx) + + if tx.AttemptCount >= maxAllowedAttempts { + return fmt.Errorf("reached max allowed attempts for txID: %d. TXM won't broadcast any more attempts."+ + "If this error persists, it means the transaction won't be confirmed and the TXM needs to be restarted."+ + "Look for any error messages from previous attempts that may indicate why this happened, i.e. wallet is out of funds. Tx: %v", tx.ID, tx) + } + + if time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero() { + // TODO: add optional graceful bumping strategy + t.lggr.Info("Rebroadcasting attempt for txID: ", tx.ID) return t.createAndSendAttempt(tx) } } @@ -316,40 +323,9 @@ func (t *Txm) backfillTransactions() error { } func (t *Txm) createAndSendEmptyTx(latestNonce uint64) error { - tx, err := t.storage.CreateEmptyUnconfirmedTransaction(context.TODO(), t.address, t.chainID, latestNonce, t.config.EmptyTxLimitDefault) + tx, err := t.txStore.CreateEmptyUnconfirmedTransaction(context.TODO(), t.address, t.chainID, latestNonce, t.config.EmptyTxLimitDefault) if err != nil { return err } return t.createAndSendAttempt(tx) } - -func (t *Txm) pendingAndLatestNonce(ctx context.Context, fromAddress common.Address) (pending uint64, latest uint64, err error) { - pendingS, latestS := new(string), new(string) - reqs := []rpc.BatchElem{ - {Method: "eth_getTransactionCount", Args: []interface{}{fromAddress, "pending"}, Result: &pendingS}, - {Method: "eth_getTransactionCount", Args: []interface{}{fromAddress, "latest"}, Result: &latestS}, - } - - if err = t.client.BatchCallContext(ctx, reqs); err != nil { - return - } - - for _, response := range reqs { - if response.Error != nil { - return 0, 0, response.Error - } - } - - if pending, err = hexutil.DecodeUint64(*pendingS); err != nil { - return - } - if latest, err = hexutil.DecodeUint64(*latestS); err != nil { - return - } - - if pending < latest { - return 0, 0, fmt.Errorf("RPC nonce state out of sync. Pending: %d, Latest: %d", pending, latest) - } - - return pending, latest, err -} diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 9a3ddb0db9f..ff380206267 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -1,12 +1,20 @@ package types import ( + "encoding/json" "fmt" "math/big" "time" + "github.com/google/uuid" + "gopkg.in/guregu/null.v4" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" + clnull "github.com/smartcontractkit/chainlink-common/pkg/utils/null" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" ) @@ -38,8 +46,16 @@ type Transaction struct { State TxState IsPurgeable bool Attempts []*Attempt - AttemptCount uint16 // AttempCount is strictly kept inMemory and prevents indefinite retrying - // Meta, ForwarderAddress, Strategy + AttemptCount uint16 // AttempCount is strictly kept in memory and prevents indefinite retrying + Meta *sqlutil.JSON + Subject uuid.NullUUID + + // Pipeline variables - if you aren't calling this from chain tx task within + // the pipeline, you don't need these variables + PipelineTaskRunID uuid.NullUUID + MinConfirmations clnull.Uint32 + SignalCallback bool + CallbackCompleted bool } func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, error) { @@ -61,6 +77,17 @@ func (t *Transaction) DeepCopy() *Transaction { return © } +func (t *Transaction) GetMeta() (*TxMeta, error) { + if t.Meta != nil { + return nil, nil + } + var m TxMeta + if err := json.Unmarshal(*t.Meta, &m); err != nil { + return nil, fmt.Errorf("unmarshalling meta: %w", err) + } + return &m, nil +} + type Attempt struct { ID uint64 TxID uint64 @@ -82,3 +109,59 @@ func (a *Attempt) DeepCopy() *Attempt { } return © } + +type TxRequest struct { + IdempotencyKey *string + ChainID *big.Int + FromAddress common.Address + ToAddress common.Address + Value *big.Int + Data []byte + SpecifiedGasLimit uint64 + + Meta *sqlutil.JSON // TODO: *TxMeta after migration + ForwarderAddress common.Address + //QueueingTxStrategy QueueingTxStrategy + + // Pipeline variables - if you aren't calling this from chain tx task within + // the pipeline, you don't need these variables + PipelineTaskRunID uuid.NullUUID + MinConfirmations clnull.Uint32 + SignalCallback bool +} + +type TxMeta struct { + // Pipeline + JobID *int32 `json:"JobID,omitempty"` + FailOnRevert null.Bool `json:"FailOnRevert,omitempty"` + + // VRF + RequestID *common.Hash `json:"RequestID,omitempty"` + RequestTxHash *common.Hash `json:"RequestTxHash,omitempty"` + RequestIDs []common.Hash `json:"RequestIDs,omitempty"` + RequestTxHashes []common.Hash `json:"RequestTxHashes,omitempty"` + MaxLink *string `json:"MaxLink,omitempty"` + SubID *uint64 `json:"SubId,omitempty"` + GlobalSubID *string `json:"GlobalSubId,omitempty"` + MaxEth *string `json:"MaxEth,omitempty"` + ForceFulfilled *bool `json:"ForceFulfilled,omitempty"` + ForceFulfillmentAttempt *uint64 `json:"ForceFulfillmentAttempt,omitempty"` + + // Used for keepers + UpkeepID *string `json:"UpkeepID,omitempty"` + + // Used for Keystone Workflows + WorkflowExecutionID *string `json:"WorkflowExecutionID,omitempty"` + + // Forwarders + FwdrDestAddress *common.Address `json:"ForwarderDestAddress,omitempty"` + + // CCIP + MessageIDs []string `json:"MessageIDs,omitempty"` + SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` +} + +type QueueingTxStrategy struct { + QueueSize uint32 + Subject uuid.NullUUID +} From b92c3e4225e6cb1e6a1d2ee0f34c202834176551 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 21 Oct 2024 16:31:23 +0300 Subject: [PATCH 04/73] Update orchestrator --- core/chains/evm/txm/orchestrator.go | 52 ++++++++++++++++------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 9ef48018b7d..554d7160457 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -53,7 +53,11 @@ type OrchestratorTxStore interface { FindTxWithIdempotencyKey(context.Context, *string) (*txmtypes.Transaction, error) } -type Orchestrator struct { +// Generics are necessary to keep TXMv2 backwards compatible +type Orchestrator[ + BLOCK_HASH types.Hashable, + HEAD types.Head[BLOCK_HASH], +] struct { services.StateMachine lggr logger.SugaredLogger chainID *big.Int @@ -63,14 +67,14 @@ type Orchestrator struct { resumeCallback txmgr.ResumeCallback } -func NewTxmOrchestrator( +func NewTxmOrchestrator[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]]( lggr logger.Logger, chainID *big.Int, txm *Txm, txStore OrchestratorTxStore, fwdMgr *forwarders.FwdMgr, -) txmgr.TxManager[*big.Int, types.Head[common.Hash], common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] { - return &Orchestrator{ +) *Orchestrator[BLOCK_HASH, HEAD] { + return &Orchestrator[BLOCK_HASH, HEAD]{ lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), chainID: chainID, txm: txm, @@ -79,7 +83,7 @@ func NewTxmOrchestrator( } } -func (o *Orchestrator) Start(ctx context.Context) error { +func (o *Orchestrator[BLOCK_HASH, HEAD]) Start(ctx context.Context) error { return o.StartOnce("Orchestrator", func() error { var ms services.MultiStart if err := ms.Start(ctx, o.txm); err != nil { @@ -94,7 +98,7 @@ func (o *Orchestrator) Start(ctx context.Context) error { }) } -func (o *Orchestrator) Close() (merr error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) Close() (merr error) { return o.StopOnce("Orchestrator", func() error { if o.fwdMgr != nil { if err := o.fwdMgr.Close(); err != nil { @@ -108,25 +112,25 @@ func (o *Orchestrator) Close() (merr error) { }) } -func (o *Orchestrator) Trigger(addr common.Address) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) Trigger(addr common.Address) { if err := o.txm.Trigger(); err != nil { o.lggr.Error(err) } } -func (o *Orchestrator) Name() string { +func (o *Orchestrator[BLOCK_HASH, HEAD]) Name() string { return o.lggr.Name() } -func (o *Orchestrator) HealthReport() map[string]error { +func (o *Orchestrator[BLOCK_HASH, HEAD]) HealthReport() map[string]error { return map[string]error{o.Name(): o.Healthy()} } -func (o *Orchestrator) RegisterResumeCallback(fn txmgr.ResumeCallback) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) RegisterResumeCallback(fn txmgr.ResumeCallback) { o.resumeCallback = fn } -func (o *Orchestrator) Reset(addr common.Address, abandon bool) error { +func (o *Orchestrator[BLOCK_HASH, HEAD]) Reset(addr common.Address, abandon bool) error { ok := o.IfStarted(func() { if err := o.txm.Abandon(); err != nil { o.lggr.Error(err) @@ -138,10 +142,10 @@ func (o *Orchestrator) Reset(addr common.Address, abandon bool) error { return nil } -func (o *Orchestrator) OnNewLongestChain(ctx context.Context, head types.Head[common.Hash]) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) OnNewLongestChain(ctx context.Context, head HEAD) { } -func (o *Orchestrator) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { // TODO: Idempotency var wrappedTx *txmtypes.Transaction wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(context.TODO(), request.IdempotencyKey) @@ -234,48 +238,48 @@ func (o *Orchestrator) CreateTransaction(ctx context.Context, request txmgrtypes return } -func (o *Orchestrator) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (uint32, error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (uint32, error) { _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, common.Address{}) return uint32(count), err } -func (o *Orchestrator) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { return } -func (o *Orchestrator) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (time nullv4.Int, err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (time nullv4.Int, err error) { return } -func (o *Orchestrator) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } -func (o *Orchestrator) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } -func (o *Orchestrator) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } -func (o *Orchestrator) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } -func (o *Orchestrator) GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) { if o.fwdMgr != nil { forwarder, err = o.fwdMgr.ForwarderFor(ctx, eoa) } return } -func (o *Orchestrator) GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) { if o.fwdMgr != nil { forwarder, err = o.fwdMgr.ForwarderForOCR2Feeds(ctx, eoa, ocr2AggregatorID) } return } -func (o *Orchestrator) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { // Loads attempts and receipts in the transaction tx, err := o.txStore.FindTxWithIdempotencyKey(ctx, &transactionID) if err != nil || tx == nil { @@ -297,7 +301,7 @@ func (o *Orchestrator) GetTransactionStatus(ctx context.Context, transactionID s } } -func (o *Orchestrator) SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value big.Int, gasLimit uint64) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { +func (o *Orchestrator[BLOCK_HASH, HEAD]) SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value big.Int, gasLimit uint64) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { txRequest := txmgrtypes.TxRequest[common.Address, common.Hash]{ FromAddress: from, ToAddress: to, From 610228f54a13902118c08a7359fa09f1a00747c4 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 22 Oct 2024 13:22:16 +0300 Subject: [PATCH 05/73] Add builder --- core/chains/evm/txm/attempt_builder.go | 4 +-- core/chains/evm/txmgr/builder.go | 41 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index e5be133d25b..0aa8a7a97cb 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -66,14 +66,14 @@ func (a *attemptBuilder) newCustomAttempt( switch txType { case 0x0: if fee.GasPrice == nil { - err = fmt.Errorf("attemptID: %v of txID: %v, is a type 0 transaction but estimator did not return legacy fee", tx.ID, attempt.ID) + err = fmt.Errorf("tried to create attempt of type %v for txID: %v but estimator did not return legacy fee", txType, tx.ID) logger.Sugared(lggr).AssumptionViolation(err.Error()) return } return a.newLegacyAttempt(ctx, tx, fee.GasPrice, estimatedGasLimit) case 0x2: if !fee.ValidDynamic() { - err = fmt.Errorf("attemptID %v of txID: %v, is a type 2 transaction but estimator did not return dynamic fee", tx.ID, attempt.ID) + err = fmt.Errorf("tried to create attempt of type %v for txID: %v but estimator did not return dynamic fee", txType, tx.ID) logger.Sugared(lggr).AssumptionViolation(err.Error()) return } diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index cbfb8775cfb..1a7e2d2d615 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -5,6 +5,7 @@ import ( "math/big" "time" + "github.com/ethereum/go-ethereum/common" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink/v2/common/txmgr" @@ -16,6 +17,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) @@ -89,6 +92,44 @@ func NewEvmTxm( return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError) } +func NewTxmv2( + ds sqlutil.DataSource, + chainConfig ChainConfig, + fCfg FeeConfig, + blockTime time.Duration, + fwdEnabled bool, + client client.Client, + lggr logger.Logger, + logPoller logpoller.LogPoller, + keyStore keystore.Eth, + estimator gas.EvmFeeEstimator, +) (TxManager, error) { + var fwdMgr *forwarders.FwdMgr + if fwdEnabled { + fwdMgr = forwarders.NewFwdMgr(ds, client, logPoller, lggr, chainConfig) + } else { + lggr.Info("ForwarderManager: Disabled") + } + + chainID := client.ConfiguredChainID() + addresses, err := keyStore.EnabledAddressesForChain(context.TODO(), chainID) + if err != nil { + return nil, err + } + // TXMv2 only supports 1 address for now + address := addresses[0] + attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMaxKey(address), estimator, keyStore) + inMemoryStore := storage.NewInMemoryStore(lggr) + config := txm.Config{ + EIP1559: fCfg.EIP1559DynamicFees(), + BlockTime: blockTime, //TODO: create new config + RetryBlockThreshold: uint16(fCfg.BumpThreshold()), + EmptyTxLimitDefault: fCfg.LimitDefault(), + } + t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStore, config, address) + return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStore, fwdMgr), nil +} + // NewEvmResender creates a new concrete EvmResender func NewEvmResender( lggr logger.Logger, From 117d8f1935350297e78faac5f644d22d427b706a Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 23 Oct 2024 13:36:46 +0300 Subject: [PATCH 06/73] Add txmv2 tests --- core/chains/evm/txm/txm.go | 1 + core/chains/evm/txm/txm_test.go | 135 ++++++++++++++++---------------- 2 files changed, 70 insertions(+), 66 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 9bb9ded92fb..840c52c862f 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -215,6 +215,7 @@ func (t *Txm) broadcastTransaction() error { if t.nonce.Load() > pendingNonce { t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", t.nonce.Load(), pendingNonce, unconfirmedCount) + return nil } } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index e8f77a85d5f..c7101fceccb 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -3,18 +3,20 @@ package txm import ( "errors" "testing" + "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rpc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "go.uber.org/zap" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/mocks" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" ) func TestLifecycle(t *testing.T) { @@ -22,34 +24,27 @@ func TestLifecycle(t *testing.T) { client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) - storage := mocks.NewStorage(t) - config := Config{} + config := Config{BlockTime: 10 * time.Millisecond} address := testutils.NewAddress() - t.Run("fails to start if pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, address) client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), errors.New("error")).Once() assert.Error(t, txm.Start(tests.Context(t))) }) t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, storage, config, address) + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + txStore := storage.NewInMemoryStore(lggr) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) var nonce uint64 = 0 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() - // broadcast loop (may or may not be executed multiple times) - client.On("BatchCallContext", mock.Anything, mock.Anything).Return(nil) - storage.On("UpdateUnstartedTransactionWithNonce", mock.Anything, address, mock.Anything).Return(nil, nil) // backfill loop (may or may not be executed multiple times) - client.On("NonceAt", mock.Anything, address, nil).Return(nonce, nil) - storage.On("MarkTransactionsConfirmed", mock.Anything, nonce, address).Return([]uint64{}, []uint64{}, nil) - storage.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, nonce, address).Return(nil, 0) - client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil) - storage.On("CountUnstartedTransactions", mock.Anything, address).Return(0) - - assert.NoError(t, txm.Start(tests.Context(t))) - assert.NoError(t, txm.Close()) + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(nonce, nil) + + servicetest.Run(t, txm) + tests.AssertLogEventually(t, observedLogs, "Backfill time elapsed") }) } @@ -62,75 +57,83 @@ func TestTrigger(t *testing.T) { txm.Trigger() assert.Error(t, txm.Trigger(), "Txm unstarted") }) -} + t.Run("executes Trigger", func(t *testing.T) { + lggr := logger.Test(t) + address := testutils.NewAddress() + txStore := storage.NewInMemoryStore(lggr) + client := mocks.NewClient(t) + ab := mocks.NewAttemptBuilder(t) + config := Config{BlockTime: 10 * time.Second} + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) + var nonce uint64 = 0 + // Start + client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() + servicetest.Run(t, txm) + assert.NoError(t, txm.Trigger()) + }) +} func TestBroadcastTransaction(t *testing.T) { t.Parallel() client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) - storage := mocks.NewStorage(t) config := Config{} address := testutils.NewAddress() - t.Run("fails if batch call for pending and latest nonce fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) - client.On("BatchCallContext", mock.Anything, mock.Anything).Return(errors.New("batch call error")).Once() + t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { + mTxStore := mocks.NewStorage(t) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, address) err := txm.broadcastTransaction() assert.Error(t, err) - assert.Contains(t, err.Error(), "batch call error") + assert.Contains(t, err.Error(), "call failed") }) - t.Run("fails if batch call for pending and latest nonce fails for one of them", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) - //pending nonce - client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Error = errors.New("pending nonce failed") - }).Return(nil).Once() - err := txm.broadcastTransaction() - assert.Error(t, err) - assert.Contains(t, err.Error(), "pending nonce failed") - - // latest nonce - client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[1].Error = errors.New("latest nonce failed") - }).Return(nil).Once() - err = txm.broadcastTransaction() - assert.Error(t, err) - assert.Contains(t, err.Error(), "latest nonce failed") + t.Run("throws a warning and returns if unconfirmed transactions exceed maxInFlightTransactions", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + mTxStore := mocks.NewStorage(t) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions+1), nil).Once() + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, address) + txm.broadcastTransaction() + tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) - t.Run("throws a warning if maxInFlightTransactions are reached", func(t *testing.T) { - pending := "0x100" - latest := "0x0" - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) - client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Result = &pending // pending - elems[1].Result = &latest // latest - }).Return(nil).Once() - err := txm.broadcastTransaction() - assert.Error(t, err) - assert.Contains(t, err.Error(), "Reached transaction limit") + t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + mTxStore := mocks.NewStorage(t) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, address) + txm.nonce.Store(1) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions/3), nil).Twice() + + client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), nil).Once() // LocalNonce: 1, PendingNonce: 0 + txm.broadcastTransaction() + + client.On("PendingNonceAt", mock.Anything, address).Return(uint64(1), nil).Once() // LocalNonce: 1, PendingNonce: 1 + mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() + txm.broadcastTransaction() + tests.AssertLogCountEventually(t, observedLogs, "Reached transaction limit.", 1) }) + t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { - pending := "0x8" - latest := "0x0" - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) - txm.nonce.Store(0) - client.On("BatchCallContext", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Result = &pending // pending - elems[1].Result = &latest // latest - }).Return(nil).Once() - storage.On("UpdateUnstartedTransactionWithNonce", mock.Anything, address, mock.Anything).Return(nil, errors.New("update failed")) + mTxStore := mocks.NewStorage(t) + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, address) + mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() err := txm.broadcastTransaction() assert.Error(t, err) - assert.Contains(t, err.Error(), "update failed") + assert.Contains(t, err.Error(), "call failed") + }) + + t.Run("returns if there are no unstarted transactions", func(t *testing.T) { + lggr := logger.Test(t) + txStore := storage.NewInMemoryStore(lggr) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) + err := txm.broadcastTransaction() + assert.NoError(t, err) + assert.Equal(t, uint64(0), txm.nonce.Load()) }) } From bfdb9a4e7df3eded79bcfc72c319582390cb8220 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 29 Oct 2024 12:04:17 +0200 Subject: [PATCH 07/73] Update retry logic --- core/chains/evm/txm/txm.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 840c52c862f..6519dc9da07 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -151,24 +151,18 @@ func (t *Txm) broadcastLoop() { defer broadcasterTicker.Stop() for { + start := time.Now() + if err := t.broadcastTransaction(); err != nil { + t.lggr.Errorf("Error during transaction broadcasting: %v", err) + } else { + t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) + } select { case <-t.broadcastStopCh: return case <-t.triggerCh: - start := time.Now() - if err := t.broadcastTransaction(); err != nil { - t.lggr.Errorf("Error during triggered transaction broadcasting: %v", err) - } else { - t.lggr.Debug("Triggered transaction broadcasting time elapsed: ", time.Since(start)) - } broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) case <-broadcasterTicker.C: - start := time.Now() - if err := t.broadcastTransaction(); err != nil { - t.lggr.Errorf("Error during transaction broadcasting: %v", err) - } else { - t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) - } } } } @@ -215,7 +209,7 @@ func (t *Txm) broadcastTransaction() error { if t.nonce.Load() > pendingNonce { t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", t.nonce.Load(), pendingNonce, unconfirmedCount) - return nil + return nil } } @@ -247,9 +241,10 @@ func (t *Txm) createAndSendAttempt(tx *types.Transaction) error { } func (t *Txm) sendTransactionWithError(tx *types.Transaction, attempt *types.Attempt) (err error) { + start := time.Now() txErr := t.client.SendTransaction(context.TODO(), attempt.SignedTransaction) tx.AttemptCount++ - t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "txErr: ", txErr) + t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore); err != nil { return From 1c18bd784482a031202d9ff6ab668b64797e297e Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 29 Oct 2024 13:30:18 +0200 Subject: [PATCH 08/73] Add backoff mechanism --- core/chains/evm/txm/txm.go | 96 +++++++++++++++++++++------------ core/chains/evm/txm/txm_test.go | 9 ++-- 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 6519dc9da07..1dff60ef3eb 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" evmtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/jpillora/backoff" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -19,9 +20,10 @@ import ( ) const ( - broadcastInterval time.Duration = 15 * time.Second - maxInFlightTransactions uint64 = 16 - maxAllowedAttempts uint16 = 10 + broadcastInterval time.Duration = 30 * time.Second + inFlightTransactionRecheckInterval time.Duration = 1 * time.Second + maxInFlightTransactions uint64 = 16 + maxAllowedAttempts uint16 = 10 ) type Client interface { @@ -145,24 +147,44 @@ func (t *Txm) Abandon() error { return t.txStore.AbandonPendingTransactions(context.TODO(), t.address) } +func newBackoff() backoff.Backoff { + return backoff.Backoff{ + Min: 1 * time.Second, + Max: 30 * time.Second, + Jitter: true, + } +} + func (t *Txm) broadcastLoop() { defer t.wg.Done() broadcasterTicker := time.NewTicker(utils.WithJitter(broadcastInterval)) defer broadcasterTicker.Stop() + backoffT := newBackoff() + var backOffCh <-chan time.Time for { start := time.Now() - if err := t.broadcastTransaction(); err != nil { + bo, err := t.broadcastTransaction() + if err != nil { t.lggr.Errorf("Error during transaction broadcasting: %v", err) } else { t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) } + if bo { + backOffCh = time.After(backoffT.Duration()) + } else { + backoffT = newBackoff() + backOffCh = nil + } select { case <-t.broadcastStopCh: return case <-t.triggerCh: broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) + case <-backOffCh: + broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) case <-broadcasterTicker.C: + continue } } } @@ -187,44 +209,48 @@ func (t *Txm) backfillLoop() { } } -func (t *Txm) broadcastTransaction() error { - _, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), 0, t.address) - if err != nil { - return err - } +func (t *Txm) broadcastTransaction() (bool, error) { + for { + _, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), 0, t.address) + if err != nil { + return false, err + } - // Optimistically send up to 1/3 of the maxInFlightTransactions. After that threshold, broadcast more cautiously - // by checking the pending nonce so no more than maxInFlightTransactions/3 can get stuck simultaneously i.e. due - // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary - // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. - if unconfirmedCount >= int(maxInFlightTransactions)/3 { - if unconfirmedCount > int(maxInFlightTransactions) { - t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) - return nil + // Optimistically send up to 1/3 of the maxInFlightTransactions. After that threshold, broadcast more cautiously + // by checking the pending nonce so no more than maxInFlightTransactions/3 can get stuck simultaneously i.e. due + // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary + // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. + if unconfirmedCount >= int(maxInFlightTransactions)/3 { + if unconfirmedCount > int(maxInFlightTransactions) { + t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) + return true, nil + } + pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + if err != nil { + return false, err + } + if t.nonce.Load() > pendingNonce { + t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", + t.nonce.Load(), pendingNonce, unconfirmedCount) + return true, nil + } } - pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + + tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(context.TODO(), t.address, t.nonce.Load()) if err != nil { - return err + return false, err } - if t.nonce.Load() > pendingNonce { - t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", - t.nonce.Load(), pendingNonce, unconfirmedCount) - return nil + if tx == nil { + return false, nil } - } + tx.Nonce = t.nonce.Load() + tx.State = types.TxUnconfirmed + t.nonce.Add(1) - tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(context.TODO(), t.address, t.nonce.Load()) - if err != nil { - return err - } - if tx == nil { - return err + if err := t.createAndSendAttempt(tx); err != nil { + return true, err + } } - tx.Nonce = t.nonce.Load() - tx.State = types.TxUnconfirmed - t.nonce.Add(1) - - return t.createAndSendAttempt(tx) } func (t *Txm) createAndSendAttempt(tx *types.Transaction) error { diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index c7101fceccb..8630a57b08c 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -86,8 +86,9 @@ func TestBroadcastTransaction(t *testing.T) { mTxStore := mocks.NewStorage(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, address) - err := txm.broadcastTransaction() + bo, err := txm.broadcastTransaction() assert.Error(t, err) + assert.False(t, bo) assert.Contains(t, err.Error(), "call failed") }) @@ -122,7 +123,8 @@ func TestBroadcastTransaction(t *testing.T) { mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, address) mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() - err := txm.broadcastTransaction() + bo, err := txm.broadcastTransaction() + assert.False(t, bo) assert.Error(t, err) assert.Contains(t, err.Error(), "call failed") }) @@ -131,8 +133,9 @@ func TestBroadcastTransaction(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStore(lggr) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) - err := txm.broadcastTransaction() + bo, err := txm.broadcastTransaction() assert.NoError(t, err) + assert.False(t, bo) assert.Equal(t, uint64(0), txm.nonce.Load()) }) } From 93889aed7d5d960ec6dee88b727d6c4122d5f629 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 1 Nov 2024 11:48:25 +0200 Subject: [PATCH 09/73] Add multi address support --- .mockery.yaml | 2 +- core/chains/evm/txm/attempt_builder.go | 23 +- core/chains/evm/txm/dummy_keystore.go | 10 +- .../evm/txm/mocks/{storage.go => tx_store.go} | 330 ++++++++---------- core/chains/evm/txm/orchestrator.go | 9 +- core/chains/evm/txm/storage/inmemory_store.go | 57 +-- .../evm/txm/storage/inmemory_store_manager.go | 124 +++++++ .../evm/txm/storage/inmemory_store_test.go | 197 +++++------ core/chains/evm/txm/txm.go | 204 ++++++----- core/chains/evm/txm/txm_test.go | 86 +++-- core/chains/evm/txmgr/builder.go | 13 +- 11 files changed, 578 insertions(+), 477 deletions(-) rename core/chains/evm/txm/mocks/{storage.go => tx_store.go} (50%) create mode 100644 core/chains/evm/txm/storage/inmemory_store_manager.go diff --git a/.mockery.yaml b/.mockery.yaml index 33f5abd3e39..5e219c5d228 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -96,7 +96,7 @@ packages: github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm: interfaces: Client: - Storage: + TxStore: AttemptBuilder: github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr: interfaces: diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index 0aa8a7a97cb..4879b91ccdf 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -20,23 +20,23 @@ type Keystore interface { } type attemptBuilder struct { - chainID *big.Int - priceMax *assets.Wei // TODO: PriceMax per key level - estimator gas.EvmFeeEstimator - keystore Keystore + chainID *big.Int + priceMaxMap map[common.Address]*assets.Wei + estimator gas.EvmFeeEstimator + keystore Keystore } -func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore Keystore) *attemptBuilder { +func NewAttemptBuilder(chainID *big.Int, priceMaxMap map[common.Address]*assets.Wei, estimator gas.EvmFeeEstimator, keystore Keystore) *attemptBuilder { return &attemptBuilder{ - chainID: chainID, - priceMax: priceMax, - estimator: estimator, - keystore: keystore, + chainID: chainID, + priceMaxMap: priceMaxMap, + estimator: estimator, + keystore: keystore, } } func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { - fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) + fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMaxMap[tx.FromAddress], &tx.FromAddress, &tx.ToAddress) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx } func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { - bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) + bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMaxMap[tx.FromAddress], nil) if err != nil { return nil, err } @@ -138,6 +138,7 @@ func (a *attemptBuilder) newDynamicFeeAttempt(ctx context.Context, tx *types.Tra Fee: gas.EvmFee{DynamicFee: gas.DynamicFee{GasFeeCap: dynamicFee.GasFeeCap, GasTipCap: dynamicFee.GasTipCap}}, Hash: signedTx.Hash(), GasLimit: estimatedGasLimit, + Type: evmtypes.DynamicFeeTxType, SignedTransaction: signedTx, } diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 5e895bc15fa..800f9aeff70 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -11,22 +11,22 @@ import ( ) type DummyKeystore struct { - privateKey *ecdsa.PrivateKey + privateKeyMap map[common.Address]*ecdsa.PrivateKey } func NewKeystore() *DummyKeystore { - return &DummyKeystore{} + return &DummyKeystore{privateKeyMap: make(map[common.Address]*ecdsa.PrivateKey)} } -func (k *DummyKeystore) Add(privateKeyString string) error { +func (k *DummyKeystore) Add(privateKeyString string, address common.Address) error { privateKey, err := crypto.HexToECDSA(privateKeyString) if err != nil { return err } - k.privateKey = privateKey + k.privateKeyMap[address] = privateKey return nil } func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - return types.SignTx(tx, types.LatestSignerForChainID(chainID), k.privateKey) + return types.SignTx(tx, types.LatestSignerForChainID(chainID), k.privateKeyMap[fromAddress]) } diff --git a/core/chains/evm/txm/mocks/storage.go b/core/chains/evm/txm/mocks/tx_store.go similarity index 50% rename from core/chains/evm/txm/mocks/storage.go rename to core/chains/evm/txm/mocks/tx_store.go index 18b180cd617..a475bd996ad 100644 --- a/core/chains/evm/txm/mocks/storage.go +++ b/core/chains/evm/txm/mocks/tx_store.go @@ -4,7 +4,6 @@ package mocks import ( context "context" - big "math/big" common "github.com/ethereum/go-ethereum/common" @@ -13,21 +12,21 @@ import ( types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) -// Storage is an autogenerated mock type for the Storage type -type Storage struct { +// TxStore is an autogenerated mock type for the TxStore type +type TxStore struct { mock.Mock } -type Storage_Expecter struct { +type TxStore_Expecter struct { mock *mock.Mock } -func (_m *Storage) EXPECT() *Storage_Expecter { - return &Storage_Expecter{mock: &_m.Mock} +func (_m *TxStore) EXPECT() *TxStore_Expecter { + return &TxStore_Expecter{mock: &_m.Mock} } // AbandonPendingTransactions provides a mock function with given fields: _a0, _a1 -func (_m *Storage) AbandonPendingTransactions(_a0 context.Context, _a1 common.Address) error { +func (_m *TxStore) AbandonPendingTransactions(_a0 context.Context, _a1 common.Address) error { ret := _m.Called(_a0, _a1) if len(ret) == 0 { @@ -44,46 +43,46 @@ func (_m *Storage) AbandonPendingTransactions(_a0 context.Context, _a1 common.Ad return r0 } -// Storage_AbandonPendingTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbandonPendingTransactions' -type Storage_AbandonPendingTransactions_Call struct { +// TxStore_AbandonPendingTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbandonPendingTransactions' +type TxStore_AbandonPendingTransactions_Call struct { *mock.Call } // AbandonPendingTransactions is a helper method to define mock.On call // - _a0 context.Context // - _a1 common.Address -func (_e *Storage_Expecter) AbandonPendingTransactions(_a0 interface{}, _a1 interface{}) *Storage_AbandonPendingTransactions_Call { - return &Storage_AbandonPendingTransactions_Call{Call: _e.mock.On("AbandonPendingTransactions", _a0, _a1)} +func (_e *TxStore_Expecter) AbandonPendingTransactions(_a0 interface{}, _a1 interface{}) *TxStore_AbandonPendingTransactions_Call { + return &TxStore_AbandonPendingTransactions_Call{Call: _e.mock.On("AbandonPendingTransactions", _a0, _a1)} } -func (_c *Storage_AbandonPendingTransactions_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *Storage_AbandonPendingTransactions_Call { +func (_c *TxStore_AbandonPendingTransactions_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *TxStore_AbandonPendingTransactions_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(common.Address)) }) return _c } -func (_c *Storage_AbandonPendingTransactions_Call) Return(_a0 error) *Storage_AbandonPendingTransactions_Call { +func (_c *TxStore_AbandonPendingTransactions_Call) Return(_a0 error) *TxStore_AbandonPendingTransactions_Call { _c.Call.Return(_a0) return _c } -func (_c *Storage_AbandonPendingTransactions_Call) RunAndReturn(run func(context.Context, common.Address) error) *Storage_AbandonPendingTransactions_Call { +func (_c *TxStore_AbandonPendingTransactions_Call) RunAndReturn(run func(context.Context, common.Address) error) *TxStore_AbandonPendingTransactions_Call { _c.Call.Return(run) return _c } -// AppendAttemptToTransaction provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Storage) AppendAttemptToTransaction(_a0 context.Context, _a1 uint64, _a2 *types.Attempt) error { - ret := _m.Called(_a0, _a1, _a2) +// AppendAttemptToTransaction provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *TxStore) AppendAttemptToTransaction(_a0 context.Context, _a1 uint64, _a2 common.Address, _a3 *types.Attempt) error { + ret := _m.Called(_a0, _a1, _a2, _a3) if len(ret) == 0 { panic("no return value specified for AppendAttemptToTransaction") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, *types.Attempt) error); ok { - r0 = rf(_a0, _a1, _a2) + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address, *types.Attempt) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) } else { r0 = ret.Error(0) } @@ -91,96 +90,40 @@ func (_m *Storage) AppendAttemptToTransaction(_a0 context.Context, _a1 uint64, _ return r0 } -// Storage_AppendAttemptToTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AppendAttemptToTransaction' -type Storage_AppendAttemptToTransaction_Call struct { +// TxStore_AppendAttemptToTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AppendAttemptToTransaction' +type TxStore_AppendAttemptToTransaction_Call struct { *mock.Call } // AppendAttemptToTransaction is a helper method to define mock.On call // - _a0 context.Context // - _a1 uint64 -// - _a2 *types.Attempt -func (_e *Storage_Expecter) AppendAttemptToTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_AppendAttemptToTransaction_Call { - return &Storage_AppendAttemptToTransaction_Call{Call: _e.mock.On("AppendAttemptToTransaction", _a0, _a1, _a2)} +// - _a2 common.Address +// - _a3 *types.Attempt +func (_e *TxStore_Expecter) AppendAttemptToTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *TxStore_AppendAttemptToTransaction_Call { + return &TxStore_AppendAttemptToTransaction_Call{Call: _e.mock.On("AppendAttemptToTransaction", _a0, _a1, _a2, _a3)} } -func (_c *Storage_AppendAttemptToTransaction_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 *types.Attempt)) *Storage_AppendAttemptToTransaction_Call { +func (_c *TxStore_AppendAttemptToTransaction_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address, _a3 *types.Attempt)) *TxStore_AppendAttemptToTransaction_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(uint64), args[2].(*types.Attempt)) + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address), args[3].(*types.Attempt)) }) return _c } -func (_c *Storage_AppendAttemptToTransaction_Call) Return(_a0 error) *Storage_AppendAttemptToTransaction_Call { +func (_c *TxStore_AppendAttemptToTransaction_Call) Return(_a0 error) *TxStore_AppendAttemptToTransaction_Call { _c.Call.Return(_a0) return _c } -func (_c *Storage_AppendAttemptToTransaction_Call) RunAndReturn(run func(context.Context, uint64, *types.Attempt) error) *Storage_AppendAttemptToTransaction_Call { +func (_c *TxStore_AppendAttemptToTransaction_Call) RunAndReturn(run func(context.Context, uint64, common.Address, *types.Attempt) error) *TxStore_AppendAttemptToTransaction_Call { _c.Call.Return(run) return _c } -// CountUnstartedTransactions provides a mock function with given fields: _a0, _a1 -func (_m *Storage) CountUnstartedTransactions(_a0 context.Context, _a1 common.Address) (int, error) { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for CountUnstartedTransactions") - } - - var r0 int - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, common.Address) (int, error)); ok { - return rf(_a0, _a1) - } - if rf, ok := ret.Get(0).(func(context.Context, common.Address) int); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Get(0).(int) - } - - if rf, ok := ret.Get(1).(func(context.Context, common.Address) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Storage_CountUnstartedTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountUnstartedTransactions' -type Storage_CountUnstartedTransactions_Call struct { - *mock.Call -} - -// CountUnstartedTransactions is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 common.Address -func (_e *Storage_Expecter) CountUnstartedTransactions(_a0 interface{}, _a1 interface{}) *Storage_CountUnstartedTransactions_Call { - return &Storage_CountUnstartedTransactions_Call{Call: _e.mock.On("CountUnstartedTransactions", _a0, _a1)} -} - -func (_c *Storage_CountUnstartedTransactions_Call) Run(run func(_a0 context.Context, _a1 common.Address)) *Storage_CountUnstartedTransactions_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(common.Address)) - }) - return _c -} - -func (_c *Storage_CountUnstartedTransactions_Call) Return(_a0 int, _a1 error) *Storage_CountUnstartedTransactions_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Storage_CountUnstartedTransactions_Call) RunAndReturn(run func(context.Context, common.Address) (int, error)) *Storage_CountUnstartedTransactions_Call { - _c.Call.Return(run) - return _c -} - -// CreateEmptyUnconfirmedTransaction provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 -func (_m *Storage) CreateEmptyUnconfirmedTransaction(_a0 context.Context, _a1 common.Address, _a2 *big.Int, _a3 uint64, _a4 uint64) (*types.Transaction, error) { - ret := _m.Called(_a0, _a1, _a2, _a3, _a4) +// CreateEmptyUnconfirmedTransaction provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *TxStore) CreateEmptyUnconfirmedTransaction(_a0 context.Context, _a1 common.Address, _a2 uint64, _a3 uint64) (*types.Transaction, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) if len(ret) == 0 { panic("no return value specified for CreateEmptyUnconfirmedTransaction") @@ -188,19 +131,19 @@ func (_m *Storage) CreateEmptyUnconfirmedTransaction(_a0 context.Context, _a1 co var r0 *types.Transaction var r1 error - if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error)); ok { - return rf(_a0, _a1, _a2, _a3, _a4) + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64, uint64) (*types.Transaction, error)); ok { + return rf(_a0, _a1, _a2, _a3) } - if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int, uint64, uint64) *types.Transaction); ok { - r0 = rf(_a0, _a1, _a2, _a3, _a4) + if rf, ok := ret.Get(0).(func(context.Context, common.Address, uint64, uint64) *types.Transaction); ok { + r0 = rf(_a0, _a1, _a2, _a3) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*types.Transaction) } } - if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int, uint64, uint64) error); ok { - r1 = rf(_a0, _a1, _a2, _a3, _a4) + if rf, ok := ret.Get(1).(func(context.Context, common.Address, uint64, uint64) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) } else { r1 = ret.Error(1) } @@ -208,40 +151,39 @@ func (_m *Storage) CreateEmptyUnconfirmedTransaction(_a0 context.Context, _a1 co return r0, r1 } -// Storage_CreateEmptyUnconfirmedTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateEmptyUnconfirmedTransaction' -type Storage_CreateEmptyUnconfirmedTransaction_Call struct { +// TxStore_CreateEmptyUnconfirmedTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateEmptyUnconfirmedTransaction' +type TxStore_CreateEmptyUnconfirmedTransaction_Call struct { *mock.Call } // CreateEmptyUnconfirmedTransaction is a helper method to define mock.On call // - _a0 context.Context // - _a1 common.Address -// - _a2 *big.Int +// - _a2 uint64 // - _a3 uint64 -// - _a4 uint64 -func (_e *Storage_Expecter) CreateEmptyUnconfirmedTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}, _a4 interface{}) *Storage_CreateEmptyUnconfirmedTransaction_Call { - return &Storage_CreateEmptyUnconfirmedTransaction_Call{Call: _e.mock.On("CreateEmptyUnconfirmedTransaction", _a0, _a1, _a2, _a3, _a4)} +func (_e *TxStore_Expecter) CreateEmptyUnconfirmedTransaction(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *TxStore_CreateEmptyUnconfirmedTransaction_Call { + return &TxStore_CreateEmptyUnconfirmedTransaction_Call{Call: _e.mock.On("CreateEmptyUnconfirmedTransaction", _a0, _a1, _a2, _a3)} } -func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 *big.Int, _a3 uint64, _a4 uint64)) *Storage_CreateEmptyUnconfirmedTransaction_Call { +func (_c *TxStore_CreateEmptyUnconfirmedTransaction_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 uint64, _a3 uint64)) *TxStore_CreateEmptyUnconfirmedTransaction_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(common.Address), args[2].(*big.Int), args[3].(uint64), args[4].(uint64)) + run(args[0].(context.Context), args[1].(common.Address), args[2].(uint64), args[3].(uint64)) }) return _c } -func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *Storage_CreateEmptyUnconfirmedTransaction_Call { +func (_c *TxStore_CreateEmptyUnconfirmedTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *TxStore_CreateEmptyUnconfirmedTransaction_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Storage_CreateEmptyUnconfirmedTransaction_Call) RunAndReturn(run func(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error)) *Storage_CreateEmptyUnconfirmedTransaction_Call { +func (_c *TxStore_CreateEmptyUnconfirmedTransaction_Call) RunAndReturn(run func(context.Context, common.Address, uint64, uint64) (*types.Transaction, error)) *TxStore_CreateEmptyUnconfirmedTransaction_Call { _c.Call.Return(run) return _c } // CreateTransaction provides a mock function with given fields: _a0, _a1 -func (_m *Storage) CreateTransaction(_a0 context.Context, _a1 *types.TxRequest) (*types.Transaction, error) { +func (_m *TxStore) CreateTransaction(_a0 context.Context, _a1 *types.TxRequest) (*types.Transaction, error) { ret := _m.Called(_a0, _a1) if len(ret) == 0 { @@ -270,46 +212,46 @@ func (_m *Storage) CreateTransaction(_a0 context.Context, _a1 *types.TxRequest) return r0, r1 } -// Storage_CreateTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTransaction' -type Storage_CreateTransaction_Call struct { +// TxStore_CreateTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTransaction' +type TxStore_CreateTransaction_Call struct { *mock.Call } // CreateTransaction is a helper method to define mock.On call // - _a0 context.Context // - _a1 *types.TxRequest -func (_e *Storage_Expecter) CreateTransaction(_a0 interface{}, _a1 interface{}) *Storage_CreateTransaction_Call { - return &Storage_CreateTransaction_Call{Call: _e.mock.On("CreateTransaction", _a0, _a1)} +func (_e *TxStore_Expecter) CreateTransaction(_a0 interface{}, _a1 interface{}) *TxStore_CreateTransaction_Call { + return &TxStore_CreateTransaction_Call{Call: _e.mock.On("CreateTransaction", _a0, _a1)} } -func (_c *Storage_CreateTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.TxRequest)) *Storage_CreateTransaction_Call { +func (_c *TxStore_CreateTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.TxRequest)) *TxStore_CreateTransaction_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(*types.TxRequest)) }) return _c } -func (_c *Storage_CreateTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *Storage_CreateTransaction_Call { +func (_c *TxStore_CreateTransaction_Call) Return(_a0 *types.Transaction, _a1 error) *TxStore_CreateTransaction_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Storage_CreateTransaction_Call) RunAndReturn(run func(context.Context, *types.TxRequest) (*types.Transaction, error)) *Storage_CreateTransaction_Call { +func (_c *TxStore_CreateTransaction_Call) RunAndReturn(run func(context.Context, *types.TxRequest) (*types.Transaction, error)) *TxStore_CreateTransaction_Call { _c.Call.Return(run) return _c } -// DeleteAttemptForUnconfirmedTx provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Storage) DeleteAttemptForUnconfirmedTx(_a0 context.Context, _a1 uint64, _a2 *types.Attempt) error { - ret := _m.Called(_a0, _a1, _a2) +// DeleteAttemptForUnconfirmedTx provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *TxStore) DeleteAttemptForUnconfirmedTx(_a0 context.Context, _a1 uint64, _a2 *types.Attempt, _a3 common.Address) error { + ret := _m.Called(_a0, _a1, _a2, _a3) if len(ret) == 0 { panic("no return value specified for DeleteAttemptForUnconfirmedTx") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, *types.Attempt) error); ok { - r0 = rf(_a0, _a1, _a2) + if rf, ok := ret.Get(0).(func(context.Context, uint64, *types.Attempt, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) } else { r0 = ret.Error(0) } @@ -317,8 +259,8 @@ func (_m *Storage) DeleteAttemptForUnconfirmedTx(_a0 context.Context, _a1 uint64 return r0 } -// Storage_DeleteAttemptForUnconfirmedTx_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteAttemptForUnconfirmedTx' -type Storage_DeleteAttemptForUnconfirmedTx_Call struct { +// TxStore_DeleteAttemptForUnconfirmedTx_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteAttemptForUnconfirmedTx' +type TxStore_DeleteAttemptForUnconfirmedTx_Call struct { *mock.Call } @@ -326,29 +268,30 @@ type Storage_DeleteAttemptForUnconfirmedTx_Call struct { // - _a0 context.Context // - _a1 uint64 // - _a2 *types.Attempt -func (_e *Storage_Expecter) DeleteAttemptForUnconfirmedTx(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_DeleteAttemptForUnconfirmedTx_Call { - return &Storage_DeleteAttemptForUnconfirmedTx_Call{Call: _e.mock.On("DeleteAttemptForUnconfirmedTx", _a0, _a1, _a2)} +// - _a3 common.Address +func (_e *TxStore_Expecter) DeleteAttemptForUnconfirmedTx(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *TxStore_DeleteAttemptForUnconfirmedTx_Call { + return &TxStore_DeleteAttemptForUnconfirmedTx_Call{Call: _e.mock.On("DeleteAttemptForUnconfirmedTx", _a0, _a1, _a2, _a3)} } -func (_c *Storage_DeleteAttemptForUnconfirmedTx_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 *types.Attempt)) *Storage_DeleteAttemptForUnconfirmedTx_Call { +func (_c *TxStore_DeleteAttemptForUnconfirmedTx_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 *types.Attempt, _a3 common.Address)) *TxStore_DeleteAttemptForUnconfirmedTx_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(uint64), args[2].(*types.Attempt)) + run(args[0].(context.Context), args[1].(uint64), args[2].(*types.Attempt), args[3].(common.Address)) }) return _c } -func (_c *Storage_DeleteAttemptForUnconfirmedTx_Call) Return(_a0 error) *Storage_DeleteAttemptForUnconfirmedTx_Call { +func (_c *TxStore_DeleteAttemptForUnconfirmedTx_Call) Return(_a0 error) *TxStore_DeleteAttemptForUnconfirmedTx_Call { _c.Call.Return(_a0) return _c } -func (_c *Storage_DeleteAttemptForUnconfirmedTx_Call) RunAndReturn(run func(context.Context, uint64, *types.Attempt) error) *Storage_DeleteAttemptForUnconfirmedTx_Call { +func (_c *TxStore_DeleteAttemptForUnconfirmedTx_Call) RunAndReturn(run func(context.Context, uint64, *types.Attempt, common.Address) error) *TxStore_DeleteAttemptForUnconfirmedTx_Call { _c.Call.Return(run) return _c } // FetchUnconfirmedTransactionAtNonceWithCount provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Storage) FetchUnconfirmedTransactionAtNonceWithCount(_a0 context.Context, _a1 uint64, _a2 common.Address) (*types.Transaction, int, error) { +func (_m *TxStore) FetchUnconfirmedTransactionAtNonceWithCount(_a0 context.Context, _a1 uint64, _a2 common.Address) (*types.Transaction, int, error) { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { @@ -384,8 +327,8 @@ func (_m *Storage) FetchUnconfirmedTransactionAtNonceWithCount(_a0 context.Conte return r0, r1, r2 } -// Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchUnconfirmedTransactionAtNonceWithCount' -type Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call struct { +// TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchUnconfirmedTransactionAtNonceWithCount' +type TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call struct { *mock.Call } @@ -393,29 +336,29 @@ type Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call struct { // - _a0 context.Context // - _a1 uint64 // - _a2 common.Address -func (_e *Storage_Expecter) FetchUnconfirmedTransactionAtNonceWithCount(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { - return &Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call{Call: _e.mock.On("FetchUnconfirmedTransactionAtNonceWithCount", _a0, _a1, _a2)} +func (_e *TxStore_Expecter) FetchUnconfirmedTransactionAtNonceWithCount(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { + return &TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call{Call: _e.mock.On("FetchUnconfirmedTransactionAtNonceWithCount", _a0, _a1, _a2)} } -func (_c *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { +func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) }) return _c } -func (_c *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call) Return(_a0 *types.Transaction, _a1 int, _a2 error) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { +func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) Return(_a0 *types.Transaction, _a1 int, _a2 error) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { _c.Call.Return(_a0, _a1, _a2) return _c } -func (_c *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call) RunAndReturn(run func(context.Context, uint64, common.Address) (*types.Transaction, int, error)) *Storage_FetchUnconfirmedTransactionAtNonceWithCount_Call { +func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) RunAndReturn(run func(context.Context, uint64, common.Address) (*types.Transaction, int, error)) *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call { _c.Call.Return(run) return _c } // MarkTransactionsConfirmed provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Storage) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]uint64, []uint64, error) { +func (_m *TxStore) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]uint64, []uint64, error) { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { @@ -453,8 +396,8 @@ func (_m *Storage) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a return r0, r1, r2 } -// Storage_MarkTransactionsConfirmed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTransactionsConfirmed' -type Storage_MarkTransactionsConfirmed_Call struct { +// TxStore_MarkTransactionsConfirmed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTransactionsConfirmed' +type TxStore_MarkTransactionsConfirmed_Call struct { *mock.Call } @@ -462,38 +405,38 @@ type Storage_MarkTransactionsConfirmed_Call struct { // - _a0 context.Context // - _a1 uint64 // - _a2 common.Address -func (_e *Storage_Expecter) MarkTransactionsConfirmed(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_MarkTransactionsConfirmed_Call { - return &Storage_MarkTransactionsConfirmed_Call{Call: _e.mock.On("MarkTransactionsConfirmed", _a0, _a1, _a2)} +func (_e *TxStore_Expecter) MarkTransactionsConfirmed(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkTransactionsConfirmed_Call { + return &TxStore_MarkTransactionsConfirmed_Call{Call: _e.mock.On("MarkTransactionsConfirmed", _a0, _a1, _a2)} } -func (_c *Storage_MarkTransactionsConfirmed_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *Storage_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkTransactionsConfirmed_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_MarkTransactionsConfirmed_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) }) return _c } -func (_c *Storage_MarkTransactionsConfirmed_Call) Return(_a0 []uint64, _a1 []uint64, _a2 error) *Storage_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkTransactionsConfirmed_Call) Return(_a0 []uint64, _a1 []uint64, _a2 error) *TxStore_MarkTransactionsConfirmed_Call { _c.Call.Return(_a0, _a1, _a2) return _c } -func (_c *Storage_MarkTransactionsConfirmed_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]uint64, []uint64, error)) *Storage_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkTransactionsConfirmed_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]uint64, []uint64, error)) *TxStore_MarkTransactionsConfirmed_Call { _c.Call.Return(run) return _c } -// MarkTxFatal provides a mock function with given fields: _a0, _a1 -func (_m *Storage) MarkTxFatal(_a0 context.Context, _a1 *types.Transaction) error { - ret := _m.Called(_a0, _a1) +// MarkTxFatal provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) MarkTxFatal(_a0 context.Context, _a1 *types.Transaction, _a2 common.Address) error { + ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { panic("no return value specified for MarkTxFatal") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Error(0) } @@ -501,46 +444,47 @@ func (_m *Storage) MarkTxFatal(_a0 context.Context, _a1 *types.Transaction) erro return r0 } -// Storage_MarkTxFatal_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTxFatal' -type Storage_MarkTxFatal_Call struct { +// TxStore_MarkTxFatal_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTxFatal' +type TxStore_MarkTxFatal_Call struct { *mock.Call } // MarkTxFatal is a helper method to define mock.On call // - _a0 context.Context // - _a1 *types.Transaction -func (_e *Storage_Expecter) MarkTxFatal(_a0 interface{}, _a1 interface{}) *Storage_MarkTxFatal_Call { - return &Storage_MarkTxFatal_Call{Call: _e.mock.On("MarkTxFatal", _a0, _a1)} +// - _a2 common.Address +func (_e *TxStore_Expecter) MarkTxFatal(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkTxFatal_Call { + return &TxStore_MarkTxFatal_Call{Call: _e.mock.On("MarkTxFatal", _a0, _a1, _a2)} } -func (_c *Storage_MarkTxFatal_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Storage_MarkTxFatal_Call { +func (_c *TxStore_MarkTxFatal_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction, _a2 common.Address)) *TxStore_MarkTxFatal_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Transaction)) + run(args[0].(context.Context), args[1].(*types.Transaction), args[2].(common.Address)) }) return _c } -func (_c *Storage_MarkTxFatal_Call) Return(_a0 error) *Storage_MarkTxFatal_Call { +func (_c *TxStore_MarkTxFatal_Call) Return(_a0 error) *TxStore_MarkTxFatal_Call { _c.Call.Return(_a0) return _c } -func (_c *Storage_MarkTxFatal_Call) RunAndReturn(run func(context.Context, *types.Transaction) error) *Storage_MarkTxFatal_Call { +func (_c *TxStore_MarkTxFatal_Call) RunAndReturn(run func(context.Context, *types.Transaction, common.Address) error) *TxStore_MarkTxFatal_Call { _c.Call.Return(run) return _c } -// MarkUnconfirmedTransactionPurgeable provides a mock function with given fields: _a0, _a1 -func (_m *Storage) MarkUnconfirmedTransactionPurgeable(_a0 context.Context, _a1 uint64) error { - ret := _m.Called(_a0, _a1) +// MarkUnconfirmedTransactionPurgeable provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) MarkUnconfirmedTransactionPurgeable(_a0 context.Context, _a1 uint64, _a2 common.Address) error { + ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { panic("no return value specified for MarkUnconfirmedTransactionPurgeable") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Error(0) } @@ -548,46 +492,47 @@ func (_m *Storage) MarkUnconfirmedTransactionPurgeable(_a0 context.Context, _a1 return r0 } -// Storage_MarkUnconfirmedTransactionPurgeable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkUnconfirmedTransactionPurgeable' -type Storage_MarkUnconfirmedTransactionPurgeable_Call struct { +// TxStore_MarkUnconfirmedTransactionPurgeable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkUnconfirmedTransactionPurgeable' +type TxStore_MarkUnconfirmedTransactionPurgeable_Call struct { *mock.Call } // MarkUnconfirmedTransactionPurgeable is a helper method to define mock.On call // - _a0 context.Context // - _a1 uint64 -func (_e *Storage_Expecter) MarkUnconfirmedTransactionPurgeable(_a0 interface{}, _a1 interface{}) *Storage_MarkUnconfirmedTransactionPurgeable_Call { - return &Storage_MarkUnconfirmedTransactionPurgeable_Call{Call: _e.mock.On("MarkUnconfirmedTransactionPurgeable", _a0, _a1)} +// - _a2 common.Address +func (_e *TxStore_Expecter) MarkUnconfirmedTransactionPurgeable(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { + return &TxStore_MarkUnconfirmedTransactionPurgeable_Call{Call: _e.mock.On("MarkUnconfirmedTransactionPurgeable", _a0, _a1, _a2)} } -func (_c *Storage_MarkUnconfirmedTransactionPurgeable_Call) Run(run func(_a0 context.Context, _a1 uint64)) *Storage_MarkUnconfirmedTransactionPurgeable_Call { +func (_c *TxStore_MarkUnconfirmedTransactionPurgeable_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(uint64)) + run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) }) return _c } -func (_c *Storage_MarkUnconfirmedTransactionPurgeable_Call) Return(_a0 error) *Storage_MarkUnconfirmedTransactionPurgeable_Call { +func (_c *TxStore_MarkUnconfirmedTransactionPurgeable_Call) Return(_a0 error) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { _c.Call.Return(_a0) return _c } -func (_c *Storage_MarkUnconfirmedTransactionPurgeable_Call) RunAndReturn(run func(context.Context, uint64) error) *Storage_MarkUnconfirmedTransactionPurgeable_Call { +func (_c *TxStore_MarkUnconfirmedTransactionPurgeable_Call) RunAndReturn(run func(context.Context, uint64, common.Address) error) *TxStore_MarkUnconfirmedTransactionPurgeable_Call { _c.Call.Return(run) return _c } -// UpdateTransactionBroadcast provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *Storage) UpdateTransactionBroadcast(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash) error { - ret := _m.Called(_a0, _a1, _a2, _a3) +// UpdateTransactionBroadcast provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *TxStore) UpdateTransactionBroadcast(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash, _a4 common.Address) error { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) if len(ret) == 0 { panic("no return value specified for UpdateTransactionBroadcast") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, uint64, common.Hash) error); ok { - r0 = rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(0).(func(context.Context, uint64, uint64, common.Hash, common.Address) error); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) } else { r0 = ret.Error(0) } @@ -595,8 +540,8 @@ func (_m *Storage) UpdateTransactionBroadcast(_a0 context.Context, _a1 uint64, _ return r0 } -// Storage_UpdateTransactionBroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTransactionBroadcast' -type Storage_UpdateTransactionBroadcast_Call struct { +// TxStore_UpdateTransactionBroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTransactionBroadcast' +type TxStore_UpdateTransactionBroadcast_Call struct { *mock.Call } @@ -605,29 +550,30 @@ type Storage_UpdateTransactionBroadcast_Call struct { // - _a1 uint64 // - _a2 uint64 // - _a3 common.Hash -func (_e *Storage_Expecter) UpdateTransactionBroadcast(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *Storage_UpdateTransactionBroadcast_Call { - return &Storage_UpdateTransactionBroadcast_Call{Call: _e.mock.On("UpdateTransactionBroadcast", _a0, _a1, _a2, _a3)} +// - _a4 common.Address +func (_e *TxStore_Expecter) UpdateTransactionBroadcast(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}, _a4 interface{}) *TxStore_UpdateTransactionBroadcast_Call { + return &TxStore_UpdateTransactionBroadcast_Call{Call: _e.mock.On("UpdateTransactionBroadcast", _a0, _a1, _a2, _a3, _a4)} } -func (_c *Storage_UpdateTransactionBroadcast_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash)) *Storage_UpdateTransactionBroadcast_Call { +func (_c *TxStore_UpdateTransactionBroadcast_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 uint64, _a3 common.Hash, _a4 common.Address)) *TxStore_UpdateTransactionBroadcast_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(uint64), args[2].(uint64), args[3].(common.Hash)) + run(args[0].(context.Context), args[1].(uint64), args[2].(uint64), args[3].(common.Hash), args[4].(common.Address)) }) return _c } -func (_c *Storage_UpdateTransactionBroadcast_Call) Return(_a0 error) *Storage_UpdateTransactionBroadcast_Call { +func (_c *TxStore_UpdateTransactionBroadcast_Call) Return(_a0 error) *TxStore_UpdateTransactionBroadcast_Call { _c.Call.Return(_a0) return _c } -func (_c *Storage_UpdateTransactionBroadcast_Call) RunAndReturn(run func(context.Context, uint64, uint64, common.Hash) error) *Storage_UpdateTransactionBroadcast_Call { +func (_c *TxStore_UpdateTransactionBroadcast_Call) RunAndReturn(run func(context.Context, uint64, uint64, common.Hash, common.Address) error) *TxStore_UpdateTransactionBroadcast_Call { _c.Call.Return(run) return _c } // UpdateUnstartedTransactionWithNonce provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Storage) UpdateUnstartedTransactionWithNonce(_a0 context.Context, _a1 common.Address, _a2 uint64) (*types.Transaction, error) { +func (_m *TxStore) UpdateUnstartedTransactionWithNonce(_a0 context.Context, _a1 common.Address, _a2 uint64) (*types.Transaction, error) { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { @@ -656,8 +602,8 @@ func (_m *Storage) UpdateUnstartedTransactionWithNonce(_a0 context.Context, _a1 return r0, r1 } -// Storage_UpdateUnstartedTransactionWithNonce_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateUnstartedTransactionWithNonce' -type Storage_UpdateUnstartedTransactionWithNonce_Call struct { +// TxStore_UpdateUnstartedTransactionWithNonce_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateUnstartedTransactionWithNonce' +type TxStore_UpdateUnstartedTransactionWithNonce_Call struct { *mock.Call } @@ -665,34 +611,34 @@ type Storage_UpdateUnstartedTransactionWithNonce_Call struct { // - _a0 context.Context // - _a1 common.Address // - _a2 uint64 -func (_e *Storage_Expecter) UpdateUnstartedTransactionWithNonce(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Storage_UpdateUnstartedTransactionWithNonce_Call { - return &Storage_UpdateUnstartedTransactionWithNonce_Call{Call: _e.mock.On("UpdateUnstartedTransactionWithNonce", _a0, _a1, _a2)} +func (_e *TxStore_Expecter) UpdateUnstartedTransactionWithNonce(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_UpdateUnstartedTransactionWithNonce_Call { + return &TxStore_UpdateUnstartedTransactionWithNonce_Call{Call: _e.mock.On("UpdateUnstartedTransactionWithNonce", _a0, _a1, _a2)} } -func (_c *Storage_UpdateUnstartedTransactionWithNonce_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 uint64)) *Storage_UpdateUnstartedTransactionWithNonce_Call { +func (_c *TxStore_UpdateUnstartedTransactionWithNonce_Call) Run(run func(_a0 context.Context, _a1 common.Address, _a2 uint64)) *TxStore_UpdateUnstartedTransactionWithNonce_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(common.Address), args[2].(uint64)) }) return _c } -func (_c *Storage_UpdateUnstartedTransactionWithNonce_Call) Return(_a0 *types.Transaction, _a1 error) *Storage_UpdateUnstartedTransactionWithNonce_Call { +func (_c *TxStore_UpdateUnstartedTransactionWithNonce_Call) Return(_a0 *types.Transaction, _a1 error) *TxStore_UpdateUnstartedTransactionWithNonce_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Storage_UpdateUnstartedTransactionWithNonce_Call) RunAndReturn(run func(context.Context, common.Address, uint64) (*types.Transaction, error)) *Storage_UpdateUnstartedTransactionWithNonce_Call { +func (_c *TxStore_UpdateUnstartedTransactionWithNonce_Call) RunAndReturn(run func(context.Context, common.Address, uint64) (*types.Transaction, error)) *TxStore_UpdateUnstartedTransactionWithNonce_Call { _c.Call.Return(run) return _c } -// NewStorage creates a new instance of Storage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// NewTxStore creates a new instance of TxStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewStorage(t interface { +func NewTxStore(t interface { mock.TestingT Cleanup(func()) -}) *Storage { - mock := &Storage{} +}) *TxStore { + mock := &TxStore{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 554d7160457..a836fd3ef4c 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -113,7 +113,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Close() (merr error) { } func (o *Orchestrator[BLOCK_HASH, HEAD]) Trigger(addr common.Address) { - if err := o.txm.Trigger(); err != nil { + if err := o.txm.Trigger(addr); err != nil { o.lggr.Error(err) } } @@ -132,7 +132,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) RegisterResumeCallback(fn txmgr.ResumeC func (o *Orchestrator[BLOCK_HASH, HEAD]) Reset(addr common.Address, abandon bool) error { ok := o.IfStarted(func() { - if err := o.txm.Abandon(); err != nil { + if err := o.txm.Abandon(addr); err != nil { o.lggr.Error(err) } }) @@ -146,7 +146,6 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) OnNewLongestChain(ctx context.Context, } func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { - // TODO: Idempotency var wrappedTx *txmtypes.Transaction wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(context.TODO(), request.IdempotencyKey) if err != nil { @@ -210,7 +209,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, if err != nil { return } - o.txm.Trigger() + o.txm.Trigger(request.FromAddress) } sequence := evmtypes.Nonce(wrappedTx.Nonce) @@ -317,6 +316,6 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) SendNativeToken(ctx context.Context, ch } // Trigger the Txm to check for new transaction - err = o.txm.Trigger() + err = o.txm.Trigger(from) return tx, err } diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 9b88a5ccbde..ecc22322ff3 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -1,7 +1,6 @@ package storage import ( - "context" "fmt" "math/big" "sort" @@ -23,6 +22,8 @@ type InMemoryStore struct { sync.RWMutex lggr logger.Logger txIDCount uint64 + address common.Address + chainID *big.Int UnstartedTransactions []*types.Transaction UnconfirmedTransactions map[uint64]*types.Transaction @@ -32,16 +33,18 @@ type InMemoryStore struct { Transactions map[uint64]*types.Transaction } -func NewInMemoryStore(lggr logger.Logger) *InMemoryStore { +func NewInMemoryStore(lggr logger.Logger, address common.Address, chainID *big.Int) *InMemoryStore { return &InMemoryStore{ - lggr: logger.Named(lggr, "InMemoryStore"), + lggr: logger.Named(lggr, "InMemoryStore."+address.String()), + address: address, + chainID: chainID, UnconfirmedTransactions: make(map[uint64]*types.Transaction), ConfirmedTransactions: make(map[uint64]*types.Transaction), Transactions: make(map[uint64]*types.Transaction), } } -func (m *InMemoryStore) AbandonPendingTransactions(context.Context, common.Address) error { +func (m *InMemoryStore) AbandonPendingTransactions() { m.Lock() defer m.Unlock() @@ -56,11 +59,9 @@ func (m *InMemoryStore) AbandonPendingTransactions(context.Context, common.Addre m.FatalTransactions = append(m.FatalTransactions, tx) } m.UnconfirmedTransactions = make(map[uint64]*types.Transaction) - - return nil } -func (m *InMemoryStore) AppendAttemptToTransaction(_ context.Context, txNonce uint64, attempt *types.Attempt) error { +func (m *InMemoryStore) AppendAttemptToTransaction(txNonce uint64, attempt *types.Attempt) error { m.Lock() defer m.Unlock() @@ -81,26 +82,26 @@ func (m *InMemoryStore) AppendAttemptToTransaction(_ context.Context, txNonce ui return nil } -func (m *InMemoryStore) CountUnstartedTransactions(context.Context, common.Address) (int, error) { +func (m *InMemoryStore) CountUnstartedTransactions() int { m.RLock() defer m.RUnlock() - return len(m.UnstartedTransactions), nil + return len(m.UnstartedTransactions) } -func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(ctx context.Context, fromAddress common.Address, chainID *big.Int, nonce uint64, limit uint64) (*types.Transaction, error) { +func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit uint64) (*types.Transaction, error) { m.Lock() defer m.Unlock() m.txIDCount++ emptyTx := &types.Transaction{ ID: m.txIDCount, - ChainID: chainID, + ChainID: m.chainID, Nonce: nonce, - FromAddress: fromAddress, + FromAddress: m.address, ToAddress: common.Address{}, Value: big.NewInt(0), - SpecifiedGasLimit: limit, + SpecifiedGasLimit: gasLimit, CreatedAt: time.Now(), State: types.TxUnconfirmed, } @@ -115,7 +116,7 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(ctx context.Context, f return emptyTx.DeepCopy(), nil } -func (m *InMemoryStore) CreateTransaction(_ context.Context, txRequest *types.TxRequest) (*types.Transaction, error) { +func (m *InMemoryStore) CreateTransaction(txRequest *types.TxRequest) *types.Transaction { m.Lock() defer m.Unlock() @@ -124,8 +125,8 @@ func (m *InMemoryStore) CreateTransaction(_ context.Context, txRequest *types.Tx tx := &types.Transaction{ ID: m.txIDCount, IdempotencyKey: txRequest.IdempotencyKey, - ChainID: txRequest.ChainID, - FromAddress: txRequest.FromAddress, + ChainID: m.chainID, + FromAddress: m.address, ToAddress: txRequest.ToAddress, Value: txRequest.Value, Data: txRequest.Data, @@ -148,10 +149,10 @@ func (m *InMemoryStore) CreateTransaction(_ context.Context, txRequest *types.Tx copy := tx.DeepCopy() m.Transactions[copy.ID] = copy m.UnstartedTransactions = append(m.UnstartedTransactions, copy) - return tx, nil + return tx } -func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(_ context.Context, latestNonce uint64, _ common.Address) (txCopy *types.Transaction, unconfirmedCount int, err error) { +func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(latestNonce uint64) (txCopy *types.Transaction, unconfirmedCount int) { m.RLock() defer m.RUnlock() @@ -163,7 +164,7 @@ func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(_ context.Co return } -func (m *InMemoryStore) MarkTransactionsConfirmed(_ context.Context, latestNonce uint64, _ common.Address) ([]uint64, []uint64, error) { +func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, []uint64) { m.Lock() defer m.Unlock() @@ -194,10 +195,10 @@ func (m *InMemoryStore) MarkTransactionsConfirmed(_ context.Context, latestNonce } sort.Slice(confirmedTransactionIDs, func(i, j int) bool { return confirmedTransactionIDs[i] < confirmedTransactionIDs[j] }) sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) - return confirmedTransactionIDs, unconfirmedTransactionIDs, nil + return confirmedTransactionIDs, unconfirmedTransactionIDs } -func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(_ context.Context, nonce uint64) error { +func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(nonce uint64) error { m.Lock() defer m.Unlock() @@ -211,7 +212,7 @@ func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(_ context.Context, n return nil } -func (m *InMemoryStore) UpdateTransactionBroadcast(_ context.Context, txID uint64, txNonce uint64, attemptHash common.Hash) error { +func (m *InMemoryStore) UpdateTransactionBroadcast(txID uint64, txNonce uint64, attemptHash common.Hash) error { m.Lock() defer m.Unlock() @@ -232,7 +233,7 @@ func (m *InMemoryStore) UpdateTransactionBroadcast(_ context.Context, txID uint6 return nil } -func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(_ context.Context, _ common.Address, nonce uint64) (*types.Transaction, error) { +func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*types.Transaction, error) { m.Lock() defer m.Unlock() @@ -281,7 +282,7 @@ func (m *InMemoryStore) pruneConfirmedTransactions() []uint64 { } // Error Handler -func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(_ context.Context, transactionNonce uint64, attempt *types.Attempt) error { +func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(transactionNonce uint64, attempt *types.Attempt) error { m.Lock() defer m.Unlock() @@ -300,22 +301,22 @@ func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(_ context.Context, transac return fmt.Errorf("attempt with hash: %v for txID: %v was not found", attempt.Hash, attempt.TxID) } -func (m *InMemoryStore) MarkTxFatal(context.Context, *types.Transaction) error { +func (m *InMemoryStore) MarkTxFatal(*types.Transaction) error { return fmt.Errorf("not implemented") } // Orchestrator -func (m *InMemoryStore) FindTxWithIdempotencyKey(_ context.Context, idempotencyKey *string) (*types.Transaction, error) { +func (m *InMemoryStore) FindTxWithIdempotencyKey(idempotencyKey *string) *types.Transaction { m.Lock() defer m.Unlock() if idempotencyKey != nil { for _, tx := range m.Transactions { if tx.IdempotencyKey != nil && tx.IdempotencyKey == idempotencyKey { - return tx.DeepCopy(), nil + return tx.DeepCopy() } } } - return nil, nil + return nil } diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go new file mode 100644 index 00000000000..824b533c6a9 --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -0,0 +1,124 @@ +package storage + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +const StoreNotFoundForAddress string = "InMemoryStore for address: %v not found" + +type InMemoryStoreManager struct { + InMemoryStoreMap map[common.Address]*InMemoryStore +} + +func NewInMemoryStoreManager(lggr logger.Logger, addresses []common.Address, chainID *big.Int) *InMemoryStoreManager { + inMemoryStoreMap := make(map[common.Address]*InMemoryStore) + for _, address := range addresses { + inMemoryStoreMap[address] = NewInMemoryStore(lggr, address, chainID) + } + return &InMemoryStoreManager{InMemoryStoreMap: inMemoryStoreMap} +} + +func (m *InMemoryStoreManager) AbandonPendingTransactions(_ context.Context, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + store.AbandonPendingTransactions() + return nil + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) AppendAttemptToTransaction(_ context.Context, txNonce uint64, fromAddress common.Address, attempt *types.Attempt) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + store.AppendAttemptToTransaction(txNonce, attempt) + return nil + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) CountUnstartedTransactions(fromAddress common.Address) (int, error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.CountUnstartedTransactions(), nil + } + return 0, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) CreateEmptyUnconfirmedTransaction(_ context.Context, fromAddress common.Address, nonce uint64, gasLimit uint64) (*types.Transaction, error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.CreateEmptyUnconfirmedTransaction(nonce, gasLimit) + } + return nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) CreateTransaction(_ context.Context, txRequest *types.TxRequest) (*types.Transaction, error) { + if store, exists := m.InMemoryStoreMap[txRequest.FromAddress]; exists { + return store.CreateTransaction(txRequest), nil + } + return nil, fmt.Errorf(StoreNotFoundForAddress, txRequest.FromAddress) +} + +func (m *InMemoryStoreManager) FetchUnconfirmedTransactionAtNonceWithCount(_ context.Context, nonce uint64, fromAddress common.Address) (tx *types.Transaction, count int, err error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + tx, count = store.FetchUnconfirmedTransactionAtNonceWithCount(nonce) + return + } + return nil, 0, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) MarkTransactionsConfirmed(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxIDs []uint64, unconfirmedTxIDs []uint64, err error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + confirmedTxIDs, unconfirmedTxIDs = store.MarkTransactionsConfirmed(nonce) + return + } + return nil, nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) MarkUnconfirmedTransactionPurgeable(_ context.Context, nonce uint64, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + store.MarkUnconfirmedTransactionPurgeable(nonce) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) UpdateTransactionBroadcast(_ context.Context, txID uint64, nonce uint64, attemptHash common.Hash, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.UpdateTransactionBroadcast(txID, nonce, attemptHash) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) UpdateUnstartedTransactionWithNonce(_ context.Context, fromAddress common.Address, nonce uint64) (*types.Transaction, error) { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.UpdateUnstartedTransactionWithNonce(nonce) + } + return nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) DeleteAttemptForUnconfirmedTx(_ context.Context, nonce uint64, attempt *types.Attempt, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.DeleteAttemptForUnconfirmedTx(nonce, attempt) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) MarkTxFatal(_ context.Context, tx *types.Transaction, fromAddress common.Address) error { + if store, exists := m.InMemoryStoreMap[fromAddress]; exists { + return store.MarkTxFatal(tx) + } + return fmt.Errorf(StoreNotFoundForAddress, fromAddress) +} + +func (m *InMemoryStoreManager) FindTxWithIdempotencyKey(_ context.Context, idempotencyKey *string) (*types.Transaction, error) { + for _, store := range m.InMemoryStoreMap { + tx := store.FindTxWithIdempotencyKey(idempotencyKey) + if tx != nil { + return tx, nil + } + } + return nil, fmt.Errorf("key not found") +} diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 95d01c39663..46cc94290a8 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -6,9 +6,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/common" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/stretchr/testify/assert" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" @@ -19,19 +17,19 @@ func TestAbandonPendingTransactions(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) t.Run("abandons unstarted and unconfirmed transactions", func(t *testing.T) { // Unstarted - tx1 := insertUnstartedTransaction(m, fromAddress) - tx2 := insertUnstartedTransaction(m, fromAddress) + tx1 := insertUnstartedTransaction(m) + tx2 := insertUnstartedTransaction(m) // Unconfirmed - tx3, err := insertUnconfirmedTransaction(m, fromAddress, 3) + tx3, err := insertUnconfirmedTransaction(m, 3) assert.NoError(t, err) - tx4, err := insertUnconfirmedTransaction(m, fromAddress, 4) + tx4, err := insertUnconfirmedTransaction(m, 4) assert.NoError(t, err) - assert.NoError(t, m.AbandonPendingTransactions(tests.Context(t), fromAddress)) + m.AbandonPendingTransactions() assert.Equal(t, types.TxFatalError, tx1.State) assert.Equal(t, types.TxFatalError, tx2.State) @@ -41,16 +39,16 @@ func TestAbandonPendingTransactions(t *testing.T) { t.Run("skips all types apart from unstarted and unconfirmed transactions", func(t *testing.T) { // Fatal - tx1 := insertFataTransaction(m, fromAddress) - tx2 := insertFataTransaction(m, fromAddress) + tx1 := insertFataTransaction(m) + tx2 := insertFataTransaction(m) // Confirmed - tx3, err := insertConfirmedTransaction(m, fromAddress, 3) + tx3, err := insertConfirmedTransaction(m, 3) assert.NoError(t, err) - tx4, err := insertConfirmedTransaction(m, fromAddress, 4) + tx4, err := insertConfirmedTransaction(m, 4) assert.NoError(t, err) - assert.NoError(t, m.AbandonPendingTransactions(tests.Context(t), fromAddress)) + m.AbandonPendingTransactions() assert.Equal(t, types.TxFatalError, tx1.State) assert.Equal(t, types.TxFatalError, tx2.State) @@ -65,11 +63,11 @@ func TestAppendAttemptToTransaction(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - _, err := insertUnconfirmedTransaction(m, fromAddress, 0) // txID = 1 + _, err := insertUnconfirmedTransaction(m, 0) // txID = 1 assert.NoError(t, err) - _, err = insertConfirmedTransaction(m, fromAddress, 2) // txID = 1 + _, err = insertConfirmedTransaction(m, 2) // txID = 1 assert.NoError(t, err) t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { @@ -77,7 +75,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { newAttempt := &types.Attempt{ TxID: 1, } - assert.Error(t, m.AppendAttemptToTransaction(tests.Context(t), nonce, newAttempt)) + assert.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) }) t.Run("fails if unconfirmed transaction was found but has doesn't match the txID", func(t *testing.T) { @@ -85,7 +83,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { newAttempt := &types.Attempt{ TxID: 2, } - assert.Error(t, m.AppendAttemptToTransaction(tests.Context(t), nonce, newAttempt)) + assert.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) }) t.Run("appends attempt to transaction", func(t *testing.T) { @@ -93,7 +91,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { newAttempt := &types.Attempt{ TxID: 1, } - assert.NoError(t, m.AppendAttemptToTransaction(tests.Context(t), nonce, newAttempt)) + assert.NoError(t, m.AppendAttemptToTransaction(nonce, newAttempt)) }) } @@ -102,13 +100,12 @@ func TestCountUnstartedTransactions(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) - n, _ := m.CountUnstartedTransactions(tests.Context(t), fromAddress) - assert.Equal(t, 0, n) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - insertUnstartedTransaction(m, fromAddress) - n, _ = m.CountUnstartedTransactions(tests.Context(t), fromAddress) - assert.Equal(t, 1, n) + assert.Equal(t, 0, m.CountUnstartedTransactions()) + + insertUnstartedTransaction(m) + assert.Equal(t, 1, m.CountUnstartedTransactions()) } @@ -116,16 +113,16 @@ func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) - insertUnconfirmedTransaction(m, fromAddress, 0) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + insertUnconfirmedTransaction(m, 0) t.Run("fails if unconfirmed transaction with the same nonce exists", func(t *testing.T) { - _, err := m.CreateEmptyUnconfirmedTransaction(tests.Context(t), fromAddress, testutils.FixtureChainID, 0, 0) + _, err := m.CreateEmptyUnconfirmedTransaction(0, 0) assert.Error(t, err) }) t.Run("creates a new empty unconfirmed transaction", func(t *testing.T) { - tx, err := m.CreateEmptyUnconfirmedTransaction(tests.Context(t), fromAddress, testutils.FixtureChainID, 1, 0) + tx, err := m.CreateEmptyUnconfirmedTransaction(1, 0) assert.NoError(t, err) assert.Equal(t, types.TxUnconfirmed, tx.State) }) @@ -138,39 +135,33 @@ func TestCreateTransaction(t *testing.T) { fromAddress := testutils.NewAddress() t.Run("creates new transactions", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) now := time.Now() txR1 := &types.TxRequest{} txR2 := &types.TxRequest{} - tx1, err := m.CreateTransaction(tests.Context(t), txR1) - assert.NoError(t, err) + tx1 := m.CreateTransaction(txR1) assert.Equal(t, uint64(1), tx1.ID) - assert.Less(t, now, tx1.CreatedAt) + assert.LessOrEqual(t, now, tx1.CreatedAt) - tx2, err := m.CreateTransaction(tests.Context(t), txR2) - assert.NoError(t, err) + tx2 := m.CreateTransaction(txR2) assert.Equal(t, uint64(2), tx2.ID) - assert.Less(t, now, tx2.CreatedAt) + assert.LessOrEqual(t, now, tx2.CreatedAt) - count, _ := m.CountUnstartedTransactions(tests.Context(t), fromAddress) - assert.Equal(t, count, 2) + assert.Equal(t, 2, m.CountUnstartedTransactions()) }) t.Run("prunes oldest unstarted transactions if limit is reached", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) overshot := 5 for i := 1; i < maxQueuedTransactions+overshot; i++ { r := &types.TxRequest{} - tx, err := m.CreateTransaction(tests.Context(t), r) - assert.NoError(t, err) + tx := m.CreateTransaction(r) assert.Equal(t, uint64(i), tx.ID) } // total shouldn't exceed maxQueuedTransactions - total, err := m.CountUnstartedTransactions(tests.Context(t), fromAddress) - assert.NoError(t, err) - assert.Equal(t, maxQueuedTransactions, total) + assert.Equal(t, maxQueuedTransactions, m.CountUnstartedTransactions()) // earliest tx ID should be the same amount of the number of transactions that we dropped - tx, err := m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, 0) + tx, err := m.UpdateUnstartedTransactionWithNonce(0) assert.NoError(t, err) assert.Equal(t, uint64(overshot), tx.ID) }) @@ -181,15 +172,15 @@ func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - tx, count, _ := m.FetchUnconfirmedTransactionAtNonceWithCount(tests.Context(t), 0, fromAddress) + tx, count := m.FetchUnconfirmedTransactionAtNonceWithCount(0) assert.Nil(t, tx) assert.Equal(t, 0, count) var nonce uint64 = 0 - insertUnconfirmedTransaction(m, fromAddress, nonce) - tx, count, _ = m.FetchUnconfirmedTransactionAtNonceWithCount(tests.Context(t), nonce, fromAddress) + insertUnconfirmedTransaction(m, nonce) + tx, count = m.FetchUnconfirmedTransactionAtNonceWithCount(0) assert.Equal(t, tx.Nonce, nonce) assert.Equal(t, 1, count) @@ -201,22 +192,21 @@ func TestMarkTransactionsConfirmed(t *testing.T) { fromAddress := testutils.NewAddress() t.Run("returns 0 if there are no transactions", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) - un, cn, err := m.MarkTransactionsConfirmed(tests.Context(t), 100, fromAddress) - assert.NoError(t, err) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + un, cn := m.MarkTransactionsConfirmed(100) assert.Equal(t, len(un), 0) assert.Equal(t, len(cn), 0) }) t.Run("confirms transaction with nonce lower than the latest", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) - ctx1, err := insertUnconfirmedTransaction(m, fromAddress, 0) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + ctx1, err := insertUnconfirmedTransaction(m, 0) assert.NoError(t, err) - ctx2, err := insertUnconfirmedTransaction(m, fromAddress, 1) + ctx2, err := insertUnconfirmedTransaction(m, 1) assert.NoError(t, err) - ctxs, utxs, err := m.MarkTransactionsConfirmed(tests.Context(t), 1, fromAddress) + ctxs, utxs := m.MarkTransactionsConfirmed(1) assert.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) @@ -225,14 +215,14 @@ func TestMarkTransactionsConfirmed(t *testing.T) { }) t.Run("unconfirms transaction with nonce equal to or higher than the latest", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) - ctx1, err := insertConfirmedTransaction(m, fromAddress, 0) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + ctx1, err := insertConfirmedTransaction(m, 0) assert.NoError(t, err) - ctx2, err := insertConfirmedTransaction(m, fromAddress, 1) + ctx2, err := insertConfirmedTransaction(m, 1) assert.NoError(t, err) - ctxs, utxs, err := m.MarkTransactionsConfirmed(tests.Context(t), 1, fromAddress) + ctxs, utxs := m.MarkTransactionsConfirmed(1) assert.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) @@ -240,14 +230,13 @@ func TestMarkTransactionsConfirmed(t *testing.T) { assert.Equal(t, 0, len(ctxs)) }) t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) for i := 0; i < maxQueuedTransactions; i++ { - _, err := insertConfirmedTransaction(m, fromAddress, uint64(i)) + _, err := insertConfirmedTransaction(m, uint64(i)) assert.NoError(t, err) } assert.Equal(t, maxQueuedTransactions, len(m.ConfirmedTransactions)) - _, _, err := m.MarkTransactionsConfirmed(tests.Context(t), maxQueuedTransactions, fromAddress) - assert.NoError(t, err) + m.MarkTransactionsConfirmed(maxQueuedTransactions) assert.Equal(t, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset), len(m.ConfirmedTransactions)) }) } @@ -256,15 +245,15 @@ func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) // fails if tx was not found - err := m.MarkUnconfirmedTransactionPurgeable(tests.Context(t), 0) + err := m.MarkUnconfirmedTransactionPurgeable(0) assert.Error(t, err) - tx, err := insertUnconfirmedTransaction(m, fromAddress, 0) + tx, err := insertUnconfirmedTransaction(m, 0) assert.NoError(t, err) - err = m.MarkUnconfirmedTransactionPurgeable(tests.Context(t), 0) + err = m.MarkUnconfirmedTransactionPurgeable(0) assert.NoError(t, err) assert.Equal(t, true, tx.IsPurgeable) } @@ -275,32 +264,32 @@ func TestUpdateTransactionBroadcast(t *testing.T) { fromAddress := testutils.NewAddress() hash := testutils.NewHash() t.Run("fails if unconfirmed transaction was not found", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) var nonce uint64 = 0 - assert.Error(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) }) t.Run("fails if attempt was not found for a given transaction", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) var nonce uint64 = 0 - tx, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + tx, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) - assert.Error(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) // Attempt with different hash attempt := &types.Attempt{TxID: tx.ID, Hash: testutils.NewHash()} tx.Attempts = append(tx.Attempts, attempt) - assert.Error(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) }) t.Run("updates transaction's and attempt's broadcast times", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) var nonce uint64 = 0 - tx, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + tx, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) attempt := &types.Attempt{TxID: tx.ID, Hash: hash} tx.Attempts = append(tx.Attempts, attempt) - assert.NoError(t, m.UpdateTransactionBroadcast(tests.Context(t), 0, nonce, hash)) + assert.NoError(t, m.UpdateTransactionBroadcast(0, nonce, hash)) assert.False(t, tx.LastBroadcastAt.IsZero()) assert.False(t, attempt.BroadcastAt.IsZero()) }) @@ -311,29 +300,29 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { fromAddress := testutils.NewAddress() t.Run("returns nil if there are no unstarted transactions", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) - tx, err := m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, 0) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + tx, err := m.UpdateUnstartedTransactionWithNonce(0) assert.NoError(t, err) assert.Nil(t, tx) }) t.Run("fails if there is already another unstarted transaction with the same nonce", func(t *testing.T) { var nonce uint64 = 0 - m := NewInMemoryStore(logger.Test(t)) - insertUnstartedTransaction(m, fromAddress) - _, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + insertUnstartedTransaction(m) + _, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) - _, err = m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, nonce) + _, err = m.UpdateUnstartedTransactionWithNonce(nonce) assert.Error(t, err) }) t.Run("updates unstarted transaction to unconfirmed and assigns a nonce", func(t *testing.T) { var nonce uint64 = 0 - m := NewInMemoryStore(logger.Test(t)) - insertUnstartedTransaction(m, fromAddress) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + insertUnstartedTransaction(m) - tx, err := m.UpdateUnstartedTransactionWithNonce(tests.Context(t), fromAddress, nonce) + tx, err := m.UpdateUnstartedTransactionWithNonce(nonce) assert.NoError(t, err) assert.Equal(t, nonce, tx.Nonce) assert.Equal(t, types.TxUnconfirmed, tx.State) @@ -345,20 +334,20 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { fromAddress := testutils.NewAddress() t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) tx := &types.Transaction{Nonce: 0} attempt := &types.Attempt{TxID: 0} - err := m.DeleteAttemptForUnconfirmedTx(tests.Context(t), tx.Nonce, attempt) + err := m.DeleteAttemptForUnconfirmedTx(tx.Nonce, attempt) assert.Error(t, err) }) t.Run("fails if corresponding unconfirmed attempt for txID was not found", func(t *testing.T) { - m := NewInMemoryStore(logger.Test(t)) - _, err := insertUnconfirmedTransaction(m, fromAddress, 0) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + _, err := insertUnconfirmedTransaction(m, 0) assert.NoError(t, err) attempt := &types.Attempt{TxID: 2, Hash: testutils.NewHash()} - err = m.DeleteAttemptForUnconfirmedTx(tests.Context(t), 0, attempt) + err = m.DeleteAttemptForUnconfirmedTx(0, attempt) assert.Error(t, err) }) @@ -366,13 +355,13 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { t.Run("deletes attempt of unconfirmed transaction", func(t *testing.T) { hash := testutils.NewHash() var nonce uint64 = 0 - m := NewInMemoryStore(logger.Test(t)) - tx, err := insertUnconfirmedTransaction(m, fromAddress, nonce) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + tx, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) attempt := &types.Attempt{TxID: 0, Hash: hash} tx.Attempts = append(tx.Attempts, attempt) - err = m.DeleteAttemptForUnconfirmedTx(tests.Context(t), nonce, attempt) + err = m.DeleteAttemptForUnconfirmedTx(nonce, attempt) assert.NoError(t, err) assert.Equal(t, 0, len(tx.Attempts)) @@ -382,10 +371,10 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { func TestPruneConfirmedTransactions(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t)) + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) total := 5 for i := 0; i < total; i++ { - _, err := insertConfirmedTransaction(m, fromAddress, uint64(i)) + _, err := insertConfirmedTransaction(m, uint64(i)) assert.NoError(t, err) } prunedTxIDs := m.pruneConfirmedTransactions() @@ -394,7 +383,7 @@ func TestPruneConfirmedTransactions(t *testing.T) { assert.Equal(t, total/pruneSubset, len(prunedTxIDs)) } -func insertUnstartedTransaction(m *InMemoryStore, fromAddress common.Address) *types.Transaction { +func insertUnstartedTransaction(m *InMemoryStore) *types.Transaction { m.Lock() defer m.Unlock() @@ -403,7 +392,7 @@ func insertUnstartedTransaction(m *InMemoryStore, fromAddress common.Address) *t ID: m.txIDCount, ChainID: testutils.FixtureChainID, Nonce: 0, - FromAddress: fromAddress, + FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), SpecifiedGasLimit: 0, @@ -415,7 +404,7 @@ func insertUnstartedTransaction(m *InMemoryStore, fromAddress common.Address) *t return tx } -func insertUnconfirmedTransaction(m *InMemoryStore, fromAddress common.Address, nonce uint64) (*types.Transaction, error) { +func insertUnconfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transaction, error) { m.Lock() defer m.Unlock() @@ -424,7 +413,7 @@ func insertUnconfirmedTransaction(m *InMemoryStore, fromAddress common.Address, ID: m.txIDCount, ChainID: testutils.FixtureChainID, Nonce: nonce, - FromAddress: fromAddress, + FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), SpecifiedGasLimit: 0, @@ -440,7 +429,7 @@ func insertUnconfirmedTransaction(m *InMemoryStore, fromAddress common.Address, return tx, nil } -func insertConfirmedTransaction(m *InMemoryStore, fromAddress common.Address, nonce uint64) (*types.Transaction, error) { +func insertConfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transaction, error) { m.Lock() defer m.Unlock() @@ -449,7 +438,7 @@ func insertConfirmedTransaction(m *InMemoryStore, fromAddress common.Address, no ID: m.txIDCount, ChainID: testutils.FixtureChainID, Nonce: nonce, - FromAddress: fromAddress, + FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), SpecifiedGasLimit: 0, @@ -465,7 +454,7 @@ func insertConfirmedTransaction(m *InMemoryStore, fromAddress common.Address, no return tx, nil } -func insertFataTransaction(m *InMemoryStore, fromAddress common.Address) *types.Transaction { +func insertFataTransaction(m *InMemoryStore) *types.Transaction { m.Lock() defer m.Unlock() @@ -474,7 +463,7 @@ func insertFataTransaction(m *InMemoryStore, fromAddress common.Address) *types. ID: m.txIDCount, ChainID: testutils.FixtureChainID, Nonce: 0, - FromAddress: fromAddress, + FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), SpecifiedGasLimit: 0, diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 1dff60ef3eb..aecb7c336cd 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -5,7 +5,6 @@ import ( "fmt" "math/big" "sync" - "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -34,18 +33,18 @@ type Client interface { type TxStore interface { AbandonPendingTransactions(context.Context, common.Address) error - AppendAttemptToTransaction(context.Context, uint64, *types.Attempt) error - CreateEmptyUnconfirmedTransaction(context.Context, common.Address, *big.Int, uint64, uint64) (*types.Transaction, error) + AppendAttemptToTransaction(context.Context, uint64, common.Address, *types.Attempt) error + CreateEmptyUnconfirmedTransaction(context.Context, common.Address, uint64, uint64) (*types.Transaction, error) CreateTransaction(context.Context, *types.TxRequest) (*types.Transaction, error) FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*types.Transaction, int, error) MarkTransactionsConfirmed(context.Context, uint64, common.Address) ([]uint64, []uint64, error) - MarkUnconfirmedTransactionPurgeable(context.Context, uint64) error - UpdateTransactionBroadcast(context.Context, uint64, uint64, common.Hash) error + MarkUnconfirmedTransactionPurgeable(context.Context, uint64, common.Address) error + UpdateTransactionBroadcast(context.Context, uint64, uint64, common.Hash, common.Address) error UpdateUnstartedTransactionWithNonce(context.Context, common.Address, uint64) (*types.Transaction, error) // ErrorHandler - DeleteAttemptForUnconfirmedTx(context.Context, uint64, *types.Attempt) error - MarkTxFatal(context.Context, *types.Transaction) error + DeleteAttemptForUnconfirmedTx(context.Context, uint64, *types.Attempt, common.Address) error + MarkTxFatal(context.Context, *types.Transaction, common.Address) error } type AttemptBuilder interface { @@ -54,11 +53,11 @@ type AttemptBuilder interface { } type ErrorHandler interface { - HandleError(tx *types.Transaction, message error, attemptBuilder AttemptBuilder, client Client, txStore TxStore) (err error) + HandleError(*types.Transaction, error, AttemptBuilder, Client, TxStore, func(common.Address, uint64), bool) (err error) } type StuckTxDetector interface { - DetectStuckTransactions(tx *types.Transaction) (bool, error) + DetectStuckTransaction(tx *types.Transaction) (bool, error) } type Config struct { @@ -70,57 +69,69 @@ type Config struct { type Txm struct { services.StateMachine - lggr logger.SugaredLogger - address common.Address - chainID *big.Int - client Client - attemptBuilder AttemptBuilder - errorHandler ErrorHandler - stuckTxDetector StuckTxDetector - txStore TxStore - config Config - nonce atomic.Uint64 - - triggerCh chan struct{} - broadcastStopCh services.StopChan - backfillStopCh services.StopChan - wg sync.WaitGroup + lggr logger.SugaredLogger + enabledAddresses []common.Address + chainID *big.Int + client Client + attemptBuilder AttemptBuilder + errorHandler ErrorHandler + stuckTxDetector StuckTxDetector + txStore TxStore + config Config + + nonceMapMu sync.Mutex + nonceMap map[common.Address]uint64 + + triggerCh map[common.Address]chan struct{} + stopCh services.StopChan + wg sync.WaitGroup } -func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, address common.Address) *Txm { +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, enabledAddresses []common.Address) *Txm { return &Txm{ - lggr: logger.Sugared(logger.Named(lggr, "Txm")), - address: address, - chainID: chainID, - client: client, - attemptBuilder: attemptBuilder, - txStore: txStore, - config: config, - triggerCh: make(chan struct{}), - broadcastStopCh: make(chan struct{}), - backfillStopCh: make(chan struct{}), + lggr: logger.Sugared(logger.Named(lggr, "Txm")), + enabledAddresses: enabledAddresses, + chainID: chainID, + client: client, + attemptBuilder: attemptBuilder, + txStore: txStore, + config: config, + nonceMap: make(map[common.Address]uint64), + triggerCh: make(map[common.Address]chan struct{}), } } func (t *Txm) Start(context.Context) error { return t.StartOnce("Txm", func() error { - pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) - if err != nil { - return err - } - t.nonce.Store(pendingNonce) - t.wg.Add(2) - go t.broadcastLoop() - go t.backfillLoop() + t.stopCh = make(chan struct{}) + for _, address := range t.enabledAddresses { + err := t.startAddress(address) + if err != nil { + return err + } + } return nil }) } +func (t *Txm) startAddress(address common.Address) error { + t.triggerCh[address] = make(chan struct{}, 1) + pendingNonce, err := t.client.PendingNonceAt(context.TODO(), address) + if err != nil { + return err + } + t.setNonce(address, pendingNonce) + + t.wg.Add(2) + go t.broadcastLoop(address) + go t.backfillLoop(address) + return nil +} + func (t *Txm) Close() error { return t.StopOnce("Txm", func() error { - close(t.broadcastStopCh) - close(t.backfillStopCh) + close(t.stopCh) t.wg.Wait() return nil }) @@ -134,17 +145,29 @@ func (t *Txm) CreateTransaction(ctx context.Context, txRequest *types.TxRequest) return } -func (t *Txm) Trigger() error { +func (t *Txm) Trigger(address common.Address) error { if !t.IfStarted(func() { - t.triggerCh <- struct{}{} + t.triggerCh[address] <- struct{}{} }) { return fmt.Errorf("Txm unstarted") } return nil } -func (t *Txm) Abandon() error { - return t.txStore.AbandonPendingTransactions(context.TODO(), t.address) +func (t *Txm) Abandon(address common.Address) error { + return t.txStore.AbandonPendingTransactions(context.TODO(), address) +} + +func (t *Txm) getNonce(address common.Address) uint64 { + t.nonceMapMu.Lock() + defer t.nonceMapMu.Unlock() + return t.nonceMap[address] +} + +func (t *Txm) setNonce(address common.Address, nonce uint64) { + t.nonceMapMu.Lock() + t.nonceMap[address] = nonce + defer t.nonceMapMu.Unlock() } func newBackoff() backoff.Backoff { @@ -155,16 +178,18 @@ func newBackoff() backoff.Backoff { } } -func (t *Txm) broadcastLoop() { +func (t *Txm) broadcastLoop(address common.Address) { defer t.wg.Done() broadcasterTicker := time.NewTicker(utils.WithJitter(broadcastInterval)) defer broadcasterTicker.Stop() + ctx, cancel := t.stopCh.NewCtx() + defer cancel() backoffT := newBackoff() var backOffCh <-chan time.Time for { start := time.Now() - bo, err := t.broadcastTransaction() + bo, err := t.broadcastTransaction(ctx, address) if err != nil { t.lggr.Errorf("Error during transaction broadcasting: %v", err) } else { @@ -177,9 +202,9 @@ func (t *Txm) broadcastLoop() { backOffCh = nil } select { - case <-t.broadcastStopCh: + case <-ctx.Done(): return - case <-t.triggerCh: + case <-t.triggerCh[address]: broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) case <-backOffCh: broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) @@ -189,18 +214,20 @@ func (t *Txm) broadcastLoop() { } } -func (t *Txm) backfillLoop() { +func (t *Txm) backfillLoop(address common.Address) { defer t.wg.Done() backfillTicker := time.NewTicker(utils.WithJitter(t.config.BlockTime)) + ctx, cancel := t.stopCh.NewCtx() + defer cancel() defer backfillTicker.Stop() for { select { - case <-t.backfillStopCh: + case <-ctx.Done(): return case <-backfillTicker.C: start := time.Now() - if err := t.backfillTransactions(); err != nil { + if err := t.backfillTransactions(ctx, address); err != nil { t.lggr.Errorf("Error during backfill: %v", err) } else { t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) @@ -209,9 +236,9 @@ func (t *Txm) backfillLoop() { } } -func (t *Txm) broadcastTransaction() (bool, error) { +func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) (bool, error) { for { - _, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), 0, t.address) + _, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, address) if err != nil { return false, err } @@ -225,58 +252,59 @@ func (t *Txm) broadcastTransaction() (bool, error) { t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) return true, nil } - pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + pendingNonce, err := t.client.PendingNonceAt(ctx, address) if err != nil { return false, err } - if t.nonce.Load() > pendingNonce { + nonce := t.getNonce(address) + if nonce > pendingNonce { t.lggr.Warnf("Reached transaction limit. LocalNonce: %d, PendingNonce %d, unconfirmedCount: %d", - t.nonce.Load(), pendingNonce, unconfirmedCount) + nonce, pendingNonce, unconfirmedCount) return true, nil } } - tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(context.TODO(), t.address, t.nonce.Load()) + tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(ctx, address, t.getNonce(address)) if err != nil { return false, err } if tx == nil { return false, nil } - tx.Nonce = t.nonce.Load() + tx.Nonce = t.getNonce(address) + t.setNonce(address, tx.Nonce+1) tx.State = types.TxUnconfirmed - t.nonce.Add(1) - if err := t.createAndSendAttempt(tx); err != nil { + if err := t.createAndSendAttempt(ctx, tx, address); err != nil { return true, err } } } -func (t *Txm) createAndSendAttempt(tx *types.Transaction) error { - attempt, err := t.attemptBuilder.NewAttempt(context.TODO(), t.lggr, tx, t.config.EIP1559) +func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, address common.Address) error { + attempt, err := t.attemptBuilder.NewAttempt(ctx, t.lggr, tx, t.config.EIP1559) if err != nil { return err } - if err = t.txStore.AppendAttemptToTransaction(context.TODO(), tx.Nonce, attempt); err != nil { + if err = t.txStore.AppendAttemptToTransaction(ctx, tx.Nonce, address, attempt); err != nil { return err } - return t.sendTransactionWithError(tx, attempt) + return t.sendTransactionWithError(ctx, tx, attempt, address) } -func (t *Txm) sendTransactionWithError(tx *types.Transaction, attempt *types.Attempt) (err error) { +func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transaction, attempt *types.Attempt, address common.Address) (err error) { start := time.Now() - txErr := t.client.SendTransaction(context.TODO(), attempt.SignedTransaction) + txErr := t.client.SendTransaction(ctx, attempt.SignedTransaction) tx.AttemptCount++ t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { - if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore); err != nil { + if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore, t.setNonce, false); err != nil { return } } else if txErr != nil { - pendingNonce, err := t.client.PendingNonceAt(context.TODO(), t.address) + pendingNonce, err := t.client.PendingNonceAt(ctx, address) if err != nil { return err } @@ -286,16 +314,16 @@ func (t *Txm) sendTransactionWithError(tx *types.Transaction, attempt *types.Att } } - return t.txStore.UpdateTransactionBroadcast(context.TODO(), attempt.TxID, tx.Nonce, attempt.Hash) + return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, tx.Nonce, attempt.Hash, address) } -func (t *Txm) backfillTransactions() error { - latestNonce, err := t.client.NonceAt(context.TODO(), t.address, nil) +func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) error { + latestNonce, err := t.client.NonceAt(ctx, address, nil) if err != nil { return err } - confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(context.TODO(), latestNonce, t.address) + confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(ctx, latestNonce, address) if err != nil { return err } @@ -303,29 +331,29 @@ func (t *Txm) backfillTransactions() error { t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) } - tx, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(context.TODO(), latestNonce, t.address) + tx, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, latestNonce, address) if err != nil { return err } if unconfirmedCount == 0 { - t.lggr.Debugf("All transactions confirmed for address: %v", t.address) + t.lggr.Debugf("All transactions confirmed for address: %v", address) return nil } if tx == nil || tx.Nonce != latestNonce { - t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, t.address) - return t.createAndSendEmptyTx(latestNonce) + t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) + return t.createAndSendEmptyTx(ctx, latestNonce, address) } else { if !tx.IsPurgeable && t.stuckTxDetector != nil { - isStuck, err := t.stuckTxDetector.DetectStuckTransactions(tx) + isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) if err != nil { return err } if isStuck { tx.IsPurgeable = true - t.txStore.MarkUnconfirmedTransactionPurgeable(context.TODO(), tx.Nonce) + t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: ", tx.ID) - return t.createAndSendAttempt(tx) + return t.createAndSendAttempt(ctx, tx, address) } } @@ -338,16 +366,16 @@ func (t *Txm) backfillTransactions() error { if time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero() { // TODO: add optional graceful bumping strategy t.lggr.Info("Rebroadcasting attempt for txID: ", tx.ID) - return t.createAndSendAttempt(tx) + return t.createAndSendAttempt(ctx, tx, address) } } return nil } -func (t *Txm) createAndSendEmptyTx(latestNonce uint64) error { - tx, err := t.txStore.CreateEmptyUnconfirmedTransaction(context.TODO(), t.address, t.chainID, latestNonce, t.config.EmptyTxLimitDefault) +func (t *Txm) createAndSendEmptyTx(ctx context.Context, latestNonce uint64, address common.Address) error { + tx, err := t.txStore.CreateEmptyUnconfirmedTransaction(ctx, address, latestNonce, t.config.EmptyTxLimitDefault) if err != nil { return err } - return t.createAndSendAttempt(tx) + return t.createAndSendAttempt(ctx, tx, address) } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 8630a57b08c..894aa33f798 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -25,23 +25,28 @@ func TestLifecycle(t *testing.T) { client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 10 * time.Millisecond} - address := testutils.NewAddress() + address1 := testutils.NewAddress() + address2 := testutils.NewAddress() + assert.NotEqual(t, address1, address2) + addresses := []common.Address{address1, address2} t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, address) - client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), errors.New("error")).Once() + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, addresses) + client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() assert.Error(t, txm.Start(tests.Context(t))) }) t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) - txStore := storage.NewInMemoryStore(lggr) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) + txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) var nonce uint64 = 0 // Start - client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() + client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() + client.On("PendingNonceAt", mock.Anything, address2).Return(nonce, nil).Once() // backfill loop (may or may not be executed multiple times) - client.On("NonceAt", mock.Anything, address, mock.Anything).Return(nonce, nil) + client.On("NonceAt", mock.Anything, address1, mock.Anything).Return(nonce, nil) + client.On("NonceAt", mock.Anything, address2, mock.Anything).Return(nonce, nil) servicetest.Run(t, txm) tests.AssertLogEventually(t, observedLogs, "Backfill time elapsed") @@ -52,41 +57,44 @@ func TestLifecycle(t *testing.T) { func TestTrigger(t *testing.T) { t.Parallel() + address := testutils.NewAddress() + addresses := []common.Address{address} t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { - txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, common.Address{}) - txm.Trigger() - assert.Error(t, txm.Trigger(), "Txm unstarted") + txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, addresses) + txm.Trigger(address) + assert.Error(t, txm.Trigger(address), "Txm unstarted") }) t.Run("executes Trigger", func(t *testing.T) { lggr := logger.Test(t) - address := testutils.NewAddress() - txStore := storage.NewInMemoryStore(lggr) + txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) - config := Config{BlockTime: 10 * time.Second} - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) + config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) var nonce uint64 = 0 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() servicetest.Run(t, txm) - assert.NoError(t, txm.Trigger()) + assert.NoError(t, txm.Trigger(address)) }) } func TestBroadcastTransaction(t *testing.T) { t.Parallel() + ctx := tests.Context(t) client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) config := Config{} address := testutils.NewAddress() + addresses := []common.Address{address} t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { - mTxStore := mocks.NewStorage(t) + mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, address) - bo, err := txm.broadcastTransaction() + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + bo, err := txm.broadcastTransaction(ctx, address) assert.Error(t, err) assert.False(t, bo) assert.Contains(t, err.Error(), "call failed") @@ -94,36 +102,36 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("throws a warning and returns if unconfirmed transactions exceed maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) - mTxStore := mocks.NewStorage(t) + mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions+1), nil).Once() - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, address) - txm.broadcastTransaction() + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm.broadcastTransaction(ctx, address) tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) - mTxStore := mocks.NewStorage(t) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, address) - txm.nonce.Store(1) + mTxStore := mocks.NewTxStore(t) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm.setNonce(address, 1) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions/3), nil).Twice() client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), nil).Once() // LocalNonce: 1, PendingNonce: 0 - txm.broadcastTransaction() + txm.broadcastTransaction(ctx, address) client.On("PendingNonceAt", mock.Anything, address).Return(uint64(1), nil).Once() // LocalNonce: 1, PendingNonce: 1 mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() - txm.broadcastTransaction() + txm.broadcastTransaction(ctx, address) tests.AssertLogCountEventually(t, observedLogs, "Reached transaction limit.", 1) }) t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { - mTxStore := mocks.NewStorage(t) + mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, address) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, addresses) mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() - bo, err := txm.broadcastTransaction() + bo, err := txm.broadcastTransaction(ctx, address) assert.False(t, bo) assert.Error(t, err) assert.Contains(t, err.Error(), "call failed") @@ -131,37 +139,39 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("returns if there are no unstarted transactions", func(t *testing.T) { lggr := logger.Test(t) - txStore := storage.NewInMemoryStore(lggr) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, address) - bo, err := txm.broadcastTransaction() + txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + bo, err := txm.broadcastTransaction(ctx, address) assert.NoError(t, err) assert.False(t, bo) - assert.Equal(t, uint64(0), txm.nonce.Load()) + assert.Equal(t, uint64(0), txm.getNonce(address)) }) } func TestBackfillTransactions(t *testing.T) { t.Parallel() + ctx := tests.Context(t) client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) - storage := mocks.NewStorage(t) + storage := mocks.NewTxStore(t) config := Config{} address := testutils.NewAddress() + addresses := []common.Address{address} t.Run("fails if latest nonce fetching fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() - err := txm.backfillTransactions() + err := txm.backfillTransactions(ctx, address) assert.Error(t, err) assert.Contains(t, err.Error(), "latest nonce fail") }) t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, address) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil) storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")) - err := txm.backfillTransactions() + err := txm.backfillTransactions(ctx, address) assert.Error(t, err) assert.Contains(t, err.Error(), "marking transactions confirmed failed") }) diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 1a7e2d2d615..4c503ba336e 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -10,6 +10,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" @@ -116,17 +117,19 @@ func NewTxmv2( if err != nil { return nil, err } - // TXMv2 only supports 1 address for now - address := addresses[0] - attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMaxKey(address), estimator, keyStore) - inMemoryStore := storage.NewInMemoryStore(lggr) + priceMaxMap := make(map[common.Address]*assets.Wei) + for _, address := range addresses { + priceMaxMap[address] = fCfg.PriceMaxKey(address) + } + attemptBuilder := txm.NewAttemptBuilder(chainID, priceMaxMap, estimator, keyStore) + inMemoryStore := storage.NewInMemoryStoreManager(lggr, addresses, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), BlockTime: blockTime, //TODO: create new config RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStore, config, address) + t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStore, config, addresses) return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStore, fwdMgr), nil } From 3375df3bc2ce88ff8268340561862dfaaa5075be Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 1 Nov 2024 18:00:57 +0200 Subject: [PATCH 10/73] Add backoff mechanism for broadcasting and backfilling --- core/chains/evm/txm/txm.go | 61 +++++++++++++++++---------------- core/chains/evm/txm/txm_test.go | 6 ++-- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index aecb7c336cd..63c85973e05 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -170,22 +170,20 @@ func (t *Txm) setNonce(address common.Address, nonce uint64) { defer t.nonceMapMu.Unlock() } -func newBackoff() backoff.Backoff { +func newBackoff(min time.Duration) backoff.Backoff { return backoff.Backoff{ - Min: 1 * time.Second, - Max: 30 * time.Second, + Min: min, + Max: 1 * time.Minute, Jitter: true, } } func (t *Txm) broadcastLoop(address common.Address) { defer t.wg.Done() - broadcasterTicker := time.NewTicker(utils.WithJitter(broadcastInterval)) - defer broadcasterTicker.Stop() ctx, cancel := t.stopCh.NewCtx() defer cancel() - backoffT := newBackoff() - var backOffCh <-chan time.Time + broadcastWithBackoff := newBackoff(1 * time.Second) + var broadcastCh <-chan time.Time for { start := time.Now() @@ -196,19 +194,17 @@ func (t *Txm) broadcastLoop(address common.Address) { t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) } if bo { - backOffCh = time.After(backoffT.Duration()) + broadcastCh = time.After(broadcastWithBackoff.Duration()) } else { - backoffT = newBackoff() - backOffCh = nil + broadcastWithBackoff.Reset() + broadcastCh = time.After(utils.WithJitter(broadcastInterval)) } select { case <-ctx.Done(): return case <-t.triggerCh[address]: - broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) - case <-backOffCh: - broadcasterTicker.Reset(utils.WithJitter(broadcastInterval)) - case <-broadcasterTicker.C: + continue + case <-broadcastCh: continue } } @@ -216,22 +212,29 @@ func (t *Txm) broadcastLoop(address common.Address) { func (t *Txm) backfillLoop(address common.Address) { defer t.wg.Done() - backfillTicker := time.NewTicker(utils.WithJitter(t.config.BlockTime)) ctx, cancel := t.stopCh.NewCtx() defer cancel() - defer backfillTicker.Stop() + backfillWithBackoff := newBackoff(t.config.BlockTime) + backfillCh := time.After(utils.WithJitter(t.config.BlockTime)) for { select { case <-ctx.Done(): return - case <-backfillTicker.C: + case <-backfillCh: start := time.Now() - if err := t.backfillTransactions(ctx, address); err != nil { + bo, err := t.backfillTransactions(ctx, address) + if err != nil { t.lggr.Errorf("Error during backfill: %v", err) } else { t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) } + if bo { + backfillCh = time.After(backfillWithBackoff.Duration()) + } else { + backfillWithBackoff.Reset() + backfillCh = time.After(utils.WithJitter(t.config.BlockTime)) + } } } } @@ -317,15 +320,15 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, tx.Nonce, attempt.Hash, address) } -func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) error { +func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) (bool, error) { latestNonce, err := t.client.NonceAt(ctx, address, nil) if err != nil { - return err + return false, err } confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(ctx, latestNonce, address) if err != nil { - return err + return false, err } if len(confirmedTransactionIDs) > 0 || len(unconfirmedTransactionIDs) > 0 { t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) @@ -333,32 +336,32 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) tx, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, latestNonce, address) if err != nil { - return err + return false, err // TODO: add backoff to optimize requests } if unconfirmedCount == 0 { t.lggr.Debugf("All transactions confirmed for address: %v", address) - return nil + return true, err } if tx == nil || tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) - return t.createAndSendEmptyTx(ctx, latestNonce, address) + return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { if !tx.IsPurgeable && t.stuckTxDetector != nil { isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) if err != nil { - return err + return false, err } if isStuck { tx.IsPurgeable = true t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: ", tx.ID) - return t.createAndSendAttempt(ctx, tx, address) + return false, t.createAndSendAttempt(ctx, tx, address) } } if tx.AttemptCount >= maxAllowedAttempts { - return fmt.Errorf("reached max allowed attempts for txID: %d. TXM won't broadcast any more attempts."+ + return true, fmt.Errorf("reached max allowed attempts for txID: %d. TXM won't broadcast any more attempts."+ "If this error persists, it means the transaction won't be confirmed and the TXM needs to be restarted."+ "Look for any error messages from previous attempts that may indicate why this happened, i.e. wallet is out of funds. Tx: %v", tx.ID, tx) } @@ -366,10 +369,10 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero() { // TODO: add optional graceful bumping strategy t.lggr.Info("Rebroadcasting attempt for txID: ", tx.ID) - return t.createAndSendAttempt(ctx, tx, address) + return false, t.createAndSendAttempt(ctx, tx, address) } } - return nil + return false, nil } func (t *Txm) createAndSendEmptyTx(ctx context.Context, latestNonce uint64, address common.Address) error { diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 894aa33f798..d9d28b32617 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -162,8 +162,9 @@ func TestBackfillTransactions(t *testing.T) { t.Run("fails if latest nonce fetching fails", func(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() - err := txm.backfillTransactions(ctx, address) + bo, err := txm.backfillTransactions(ctx, address) assert.Error(t, err) + assert.False(t, bo) assert.Contains(t, err.Error(), "latest nonce fail") }) @@ -171,8 +172,9 @@ func TestBackfillTransactions(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil) storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")) - err := txm.backfillTransactions(ctx, address) + bo, err := txm.backfillTransactions(ctx, address) assert.Error(t, err) + assert.False(t, bo) assert.Contains(t, err.Error(), "marking transactions confirmed failed") }) } From 50fd3005af9c18ead75e97b42f562136bd0b43b1 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 1 Nov 2024 23:27:45 +0200 Subject: [PATCH 11/73] Minor fix --- core/chains/evm/txm/orchestrator.go | 2 +- core/chains/evm/txmgr/builder.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index a836fd3ef4c..2a456c6aa5f 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -147,7 +147,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) OnNewLongestChain(ctx context.Context, func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { var wrappedTx *txmtypes.Transaction - wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(context.TODO(), request.IdempotencyKey) + wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(ctx, request.IdempotencyKey) if err != nil { return } diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 4c503ba336e..15bfba13ba1 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -122,15 +122,15 @@ func NewTxmv2( priceMaxMap[address] = fCfg.PriceMaxKey(address) } attemptBuilder := txm.NewAttemptBuilder(chainID, priceMaxMap, estimator, keyStore) - inMemoryStore := storage.NewInMemoryStoreManager(lggr, addresses, chainID) + inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, addresses, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), BlockTime: blockTime, //TODO: create new config RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStore, config, addresses) - return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStore, fwdMgr), nil + t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, addresses) + return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr), nil } // NewEvmResender creates a new concrete EvmResender From f004bc42740fbb2d9a20be30d2908df46837025c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Sat, 2 Nov 2024 00:59:06 +0200 Subject: [PATCH 12/73] Add check to dummy keystore --- core/chains/evm/txm/dummy_keystore.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 800f9aeff70..7299a9e1803 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -3,6 +3,7 @@ package txm import ( "context" "crypto/ecdsa" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -28,5 +29,8 @@ func (k *DummyKeystore) Add(privateKeyString string, address common.Address) err } func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - return types.SignTx(tx, types.LatestSignerForChainID(chainID), k.privateKeyMap[fromAddress]) + if key, exists := k.privateKeyMap[fromAddress]; exists { + return types.SignTx(tx, types.LatestSignerForChainID(chainID), key) + } + return nil, fmt.Errorf("private key for address: %v not found", fromAddress) } From 2e97d646d8a3afc876b36cdce391c8a8fc030cf4 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Sat, 2 Nov 2024 02:24:56 +0200 Subject: [PATCH 13/73] Fix inmemory store logging per address --- core/chains/evm/txm/storage/inmemory_store.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index ecc22322ff3..4c62bdeb6f2 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -35,7 +35,7 @@ type InMemoryStore struct { func NewInMemoryStore(lggr logger.Logger, address common.Address, chainID *big.Int) *InMemoryStore { return &InMemoryStore{ - lggr: logger.Named(lggr, "InMemoryStore."+address.String()), + lggr: logger.Named(lggr, "InMemoryStore"), address: address, chainID: chainID, UnconfirmedTransactions: make(map[uint64]*types.Transaction), @@ -140,8 +140,8 @@ func (m *InMemoryStore) CreateTransaction(txRequest *types.TxRequest) *types.Tra } if len(m.UnstartedTransactions) == maxQueuedTransactions { - m.lggr.Warnf("Unstarted transactions queue reached max limit of: %d. Dropping oldest transaction: %v.", - maxQueuedTransactions, m.UnstartedTransactions[0]) + m.lggr.Warnf("Unstarted transactions queue for address: %v reached max limit of: %d. Dropping oldest transaction: %v.", + m.address, maxQueuedTransactions, m.UnstartedTransactions[0]) delete(m.Transactions, m.UnstartedTransactions[0].ID) m.UnstartedTransactions = m.UnstartedTransactions[1:maxQueuedTransactions] } @@ -191,7 +191,8 @@ func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, if len(m.ConfirmedTransactions) >= maxQueuedTransactions { prunedTxIDs := m.pruneConfirmedTransactions() - m.lggr.Debugf("Confirmed transactions map reached max limit of: %d. Pruned 1/3 of the oldest confirmed transactions. TxIDs: %v", maxQueuedTransactions, prunedTxIDs) + m.lggr.Debugf("Confirmed transactions map for address: %v reached max limit of: %d. Pruned 1/3 of the oldest confirmed transactions. TxIDs: %v", + m.address, maxQueuedTransactions, prunedTxIDs) } sort.Slice(confirmedTransactionIDs, func(i, j int) bool { return confirmedTransactionIDs[i] < confirmedTransactionIDs[j] }) sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) @@ -238,7 +239,7 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type defer m.Unlock() if len(m.UnstartedTransactions) == 0 { - m.lggr.Debug("Unstarted transaction queue is empty") + m.lggr.Debug("Unstarted transactions queue is empty for address: %v", m.address) return nil, nil } From 80c70e1aec57e87db72c966073331a213247810c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 4 Nov 2024 17:34:08 +0200 Subject: [PATCH 14/73] Remove unnecessary const --- core/chains/evm/txm/txm.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 63c85973e05..b188b478674 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -19,10 +19,9 @@ import ( ) const ( - broadcastInterval time.Duration = 30 * time.Second - inFlightTransactionRecheckInterval time.Duration = 1 * time.Second - maxInFlightTransactions uint64 = 16 - maxAllowedAttempts uint16 = 10 + broadcastInterval time.Duration = 30 * time.Second + maxInFlightTransactions uint64 = 16 + maxAllowedAttempts uint16 = 10 ) type Client interface { From f97eeaa8f0a7775a25e3d0930127ac92ca38b95b Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 5 Nov 2024 14:31:07 +0200 Subject: [PATCH 15/73] Stuck tx detector alpha --- core/chains/evm/txm/storage/inmemory_store.go | 2 +- core/chains/evm/txm/stuck_tx_detector.go | 103 ++++++++++++++++++ core/chains/evm/txm/txm.go | 5 +- core/chains/evm/txm/txm_test.go | 22 ++-- core/chains/evm/txm/types/transaction.go | 3 + core/chains/evm/txmgr/builder.go | 17 ++- 6 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 core/chains/evm/txm/stuck_tx_detector.go diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 4c62bdeb6f2..2e92b330a54 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -239,7 +239,7 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type defer m.Unlock() if len(m.UnstartedTransactions) == 0 { - m.lggr.Debug("Unstarted transactions queue is empty for address: %v", m.address) + m.lggr.Debugf("Unstarted transactions queue is empty for address: %v", m.address) return nil, nil } diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go new file mode 100644 index 00000000000..d171b6e415c --- /dev/null +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -0,0 +1,103 @@ +package txm + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type StuckTxDetectorConfig struct { + BlockTime time.Duration + StuckTxBlockThreshold uint16 + DetectionApiUrl string +} + +type stuckTxDetector struct { + lggr logger.Logger + chainType chaintype.ChainType + config StuckTxDetectorConfig +} + +func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, config StuckTxDetectorConfig) *stuckTxDetector { + return &stuckTxDetector{ + lggr: lggr, + chainType: chaintype, + config: config, + } +} + +func (s *stuckTxDetector) DetectStuckTransaction(tx *types.Transaction) (bool, error) { + switch s.chainType { + // TODO: add correct chaintype + case chaintype.ChainArbitrum: + result, err := s.apiBasedDetection(tx) + if result || err != nil { + return result, err + } + return s.timeBasedDetection(tx), nil + default: + return s.timeBasedDetection(tx), nil + } +} + +func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { + threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) + if time.Since(tx.LastBroadcastAt) > threshold && !tx.LastBroadcastAt.IsZero() { + s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the configured threshold: %v. Transaction is now considered stuck and will be purged.", + tx.ID, tx.LastBroadcastAt, threshold) + return true + } + return false +} + +type ApiResponse struct { + Status string `json:"status,omitempty"` + Hash common.Hash `json:"hash,omitempty"` +} + +const ( + ApiStatusPending = "PENDING" + ApiStatusIncluded = "INCLUDED" + ApiStatusFailed = "FAILED" + ApiStatusCancelled = "CANCELLED" + ApiStatusUnknown = "UNKNOWN" +) + +func (s *stuckTxDetector) apiBasedDetection(tx *types.Transaction) (bool, error) { + for _, attempt := range tx.Attempts { + resp, err := http.Get(s.config.DetectionApiUrl + attempt.Hash.String()) + if err != nil { + return false, fmt.Errorf("failed to get transaction status for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + var apiResponse ApiResponse + err = json.Unmarshal(body, &apiResponse) + if err != nil { + return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + switch apiResponse.Status { + case ApiStatusPending, ApiStatusIncluded: + return false, nil + case ApiStatusFailed, ApiStatusCancelled: + return true, nil + case ApiStatusUnknown: + continue + default: + continue + } + } + return false, nil +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index b188b478674..7144b556794 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -86,7 +86,7 @@ type Txm struct { wg sync.WaitGroup } -func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, enabledAddresses []common.Address) *Txm { +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, stuckTxDetector StuckTxDetector, config Config, enabledAddresses []common.Address) *Txm { return &Txm{ lggr: logger.Sugared(logger.Named(lggr, "Txm")), enabledAddresses: enabledAddresses, @@ -94,6 +94,7 @@ func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder client: client, attemptBuilder: attemptBuilder, txStore: txStore, + stuckTxDetector: stuckTxDetector, config: config, nonceMap: make(map[common.Address]uint64), triggerCh: make(map[common.Address]chan struct{}), @@ -354,7 +355,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if isStuck { tx.IsPurgeable = true t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) - t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: ", tx.ID) + t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: %d", tx.ID) return false, t.createAndSendAttempt(ctx, tx, address) } } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index d9d28b32617..e168cdf2afd 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -31,7 +31,7 @@ func TestLifecycle(t *testing.T) { addresses := []common.Address{address1, address2} t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, nil, config, addresses) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() assert.Error(t, txm.Start(tests.Context(t))) }) @@ -39,7 +39,7 @@ func TestLifecycle(t *testing.T) { t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, addresses) var nonce uint64 = 0 // Start client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() @@ -60,7 +60,7 @@ func TestTrigger(t *testing.T) { address := testutils.NewAddress() addresses := []common.Address{address} t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { - txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, addresses) + txm := NewTxm(logger.Test(t), nil, nil, nil, nil, nil, Config{}, addresses) txm.Trigger(address) assert.Error(t, txm.Trigger(address), "Txm unstarted") }) @@ -71,7 +71,7 @@ func TestTrigger(t *testing.T) { client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, addresses) var nonce uint64 = 0 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() @@ -93,7 +93,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, addresses) bo, err := txm.broadcastTransaction(ctx, address) assert.Error(t, err) assert.False(t, bo) @@ -104,7 +104,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions+1), nil).Once() - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, addresses) txm.broadcastTransaction(ctx, address) tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) @@ -112,7 +112,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, addresses) txm.setNonce(address, 1) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions/3), nil).Twice() @@ -129,7 +129,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, addresses) mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() bo, err := txm.broadcastTransaction(ctx, address) assert.False(t, bo) @@ -140,7 +140,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("returns if there are no unstarted transactions", func(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, addresses) bo, err := txm.broadcastTransaction(ctx, address) assert.NoError(t, err) assert.False(t, bo) @@ -160,7 +160,7 @@ func TestBackfillTransactions(t *testing.T) { addresses := []common.Address{address} t.Run("fails if latest nonce fetching fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, addresses) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() bo, err := txm.backfillTransactions(ctx, address) assert.Error(t, err) @@ -169,7 +169,7 @@ func TestBackfillTransactions(t *testing.T) { }) t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, addresses) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil) storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")) bo, err := txm.backfillTransactions(ctx, address) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index ff380206267..a2c05a0e771 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -159,6 +159,9 @@ type TxMeta struct { // CCIP MessageIDs []string `json:"MessageIDs,omitempty"` SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` + + // Dual Transmit + DualTransmit bool `json:"DualTransmit,omitempty"` } type QueueingTxStrategy struct { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 15bfba13ba1..7e6493b4aa0 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -97,8 +97,8 @@ func NewTxmv2( ds sqlutil.DataSource, chainConfig ChainConfig, fCfg FeeConfig, + txConfig config.Transactions, blockTime time.Duration, - fwdEnabled bool, client client.Client, lggr logger.Logger, logPoller logpoller.LogPoller, @@ -106,13 +106,24 @@ func NewTxmv2( estimator gas.EvmFeeEstimator, ) (TxManager, error) { var fwdMgr *forwarders.FwdMgr - if fwdEnabled { + if txConfig.ForwardersEnabled() { fwdMgr = forwarders.NewFwdMgr(ds, client, logPoller, lggr, chainConfig) } else { lggr.Info("ForwarderManager: Disabled") } chainID := client.ConfiguredChainID() + + var stuckTxDetector txm.StuckTxDetector + if txConfig.AutoPurge().Enabled() { + stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ + BlockTime: blockTime, + StuckTxBlockThreshold: uint16(*txConfig.AutoPurge().Threshold()), + DetectionApiUrl: txConfig.AutoPurge().DetectionApiUrl().Path, + } + stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) + } + addresses, err := keyStore.EnabledAddressesForChain(context.TODO(), chainID) if err != nil { return nil, err @@ -129,7 +140,7 @@ func NewTxmv2( RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, addresses) + t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, addresses) return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr), nil } From 7e7d697f9fa894f303f9e25c1981a1a0efea611f Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 5 Nov 2024 17:54:06 +0200 Subject: [PATCH 16/73] Update stuck tx detection --- core/chains/evm/config/chaintype/chaintype.go | 6 +++++- core/chains/evm/txm/stuck_tx_detector.go | 10 +++++----- core/chains/evm/txm/types/transaction.go | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/core/chains/evm/config/chaintype/chaintype.go b/core/chains/evm/config/chaintype/chaintype.go index b2eff02834b..bc2eace8ca3 100644 --- a/core/chains/evm/config/chaintype/chaintype.go +++ b/core/chains/evm/config/chaintype/chaintype.go @@ -23,6 +23,7 @@ const ( ChainZkEvm ChainType = "zkevm" ChainZkSync ChainType = "zksync" ChainZircuit ChainType = "zircuit" + ChainDualBroadcast ChainType = "dualBroadcast" ) // IsL2 returns true if this chain is a Layer 2 chain. Notably: @@ -39,7 +40,7 @@ func (c ChainType) IsL2() bool { func (c ChainType) IsValid() bool { switch c { - case "", ChainArbitrum, ChainAstar, ChainCelo, ChainGnosis, ChainHedera, ChainKroma, ChainMantle, ChainMetis, ChainOptimismBedrock, ChainScroll, ChainWeMix, ChainXLayer, ChainZkEvm, ChainZkSync, ChainZircuit: + case "", ChainArbitrum, ChainAstar, ChainCelo, ChainGnosis, ChainHedera, ChainKroma, ChainMantle, ChainMetis, ChainOptimismBedrock, ChainScroll, ChainWeMix, ChainXLayer, ChainZkEvm, ChainZkSync, ChainZircuit, ChainDualBroadcast: return true } return false @@ -77,6 +78,8 @@ func FromSlug(slug string) ChainType { return ChainZkSync case "zircuit": return ChainZircuit + case "dualBroadcast": + return ChainDualBroadcast default: return ChainType(slug) } @@ -144,4 +147,5 @@ var ErrInvalid = fmt.Errorf("must be one of %s or omitted", strings.Join([]strin string(ChainZkEvm), string(ChainZkSync), string(ChainZircuit), + string(ChainDualBroadcast), }, ", ")) diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go index d171b6e415c..486a978db7b 100644 --- a/core/chains/evm/txm/stuck_tx_detector.go +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -37,9 +37,9 @@ func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, confi func (s *stuckTxDetector) DetectStuckTransaction(tx *types.Transaction) (bool, error) { switch s.chainType { - // TODO: add correct chaintype - case chaintype.ChainArbitrum: - result, err := s.apiBasedDetection(tx) + // TODO: rename + case chaintype.ChainDualBroadcast: + result, err := s.dualBroadcastDetection(tx) if result || err != nil { return result, err } @@ -52,7 +52,7 @@ func (s *stuckTxDetector) DetectStuckTransaction(tx *types.Transaction) (bool, e func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) if time.Since(tx.LastBroadcastAt) > threshold && !tx.LastBroadcastAt.IsZero() { - s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the configured threshold: %v. Transaction is now considered stuck and will be purged.", + s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", tx.ID, tx.LastBroadcastAt, threshold) return true } @@ -72,7 +72,7 @@ const ( ApiStatusUnknown = "UNKNOWN" ) -func (s *stuckTxDetector) apiBasedDetection(tx *types.Transaction) (bool, error) { +func (s *stuckTxDetector) dualBroadcastDetection(tx *types.Transaction) (bool, error) { for _, attempt := range tx.Attempts { resp, err := http.Get(s.config.DetectionApiUrl + attempt.Hash.String()) if err != nil { diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index a2c05a0e771..7e1028dccb7 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -160,8 +160,8 @@ type TxMeta struct { MessageIDs []string `json:"MessageIDs,omitempty"` SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` - // Dual Transmit - DualTransmit bool `json:"DualTransmit,omitempty"` + // Dual Broadcast + DualBroadcast bool `json:"DualBroadcast,omitempty"` } type QueueingTxStrategy struct { From 9eeb6fe4abd9185d2b6ffb97c746ce207a3b9140 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 8 Nov 2024 16:52:52 +0200 Subject: [PATCH 17/73] Add stuck_tx_detection and dual broadcast client --- core/chains/evm/keystore/eth.go | 1 + core/chains/evm/keystore/mocks/eth.go | 60 +++++++++ .../evm/txm/clientwrappers/chain_client.go | 30 +++++ .../clientwrappers/dual_broadcast_client.go | 125 ++++++++++++++++++ .../evm/txm/clientwrappers/geth_client.go | 51 +++++++ core/chains/evm/txm/mocks/attempt_builder.go | 2 +- core/chains/evm/txm/mocks/client.go | 29 ++-- core/chains/evm/txm/mocks/tx_store.go | 2 +- core/chains/evm/txm/stuck_tx_detector.go | 4 +- core/chains/evm/txm/txm.go | 5 +- core/chains/evm/txm/types/transaction.go | 3 +- core/chains/evm/txmgr/builder.go | 9 +- 12 files changed, 299 insertions(+), 22 deletions(-) create mode 100644 core/chains/evm/txm/clientwrappers/chain_client.go create mode 100644 core/chains/evm/txm/clientwrappers/dual_broadcast_client.go create mode 100644 core/chains/evm/txm/clientwrappers/geth_client.go diff --git a/core/chains/evm/keystore/eth.go b/core/chains/evm/keystore/eth.go index ff71e0a4f18..9c0986d9c3d 100644 --- a/core/chains/evm/keystore/eth.go +++ b/core/chains/evm/keystore/eth.go @@ -13,5 +13,6 @@ type Eth interface { CheckEnabled(ctx context.Context, address common.Address, chainID *big.Int) error EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) SubscribeToKeyChanges(ctx context.Context) (ch chan struct{}, unsub func()) } diff --git a/core/chains/evm/keystore/mocks/eth.go b/core/chains/evm/keystore/mocks/eth.go index b481be1b5c8..bfc85fc672c 100644 --- a/core/chains/evm/keystore/mocks/eth.go +++ b/core/chains/evm/keystore/mocks/eth.go @@ -133,6 +133,66 @@ func (_c *Eth_EnabledAddressesForChain_Call) RunAndReturn(run func(context.Conte return _c } +// SignMessage provides a mock function with given fields: ctx, address, message +func (_m *Eth) SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) { + ret := _m.Called(ctx, address, message) + + if len(ret) == 0 { + panic("no return value specified for SignMessage") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) ([]byte, error)); ok { + return rf(ctx, address, message) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) []byte); ok { + r0 = rf(ctx, address, message) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, []byte) error); ok { + r1 = rf(ctx, address, message) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Eth_SignMessage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SignMessage' +type Eth_SignMessage_Call struct { + *mock.Call +} + +// SignMessage is a helper method to define mock.On call +// - ctx context.Context +// - address common.Address +// - message []byte +func (_e *Eth_Expecter) SignMessage(ctx interface{}, address interface{}, message interface{}) *Eth_SignMessage_Call { + return &Eth_SignMessage_Call{Call: _e.mock.On("SignMessage", ctx, address, message)} +} + +func (_c *Eth_SignMessage_Call) Run(run func(ctx context.Context, address common.Address, message []byte)) *Eth_SignMessage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].([]byte)) + }) + return _c +} + +func (_c *Eth_SignMessage_Call) Return(_a0 []byte, _a1 error) *Eth_SignMessage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Eth_SignMessage_Call) RunAndReturn(run func(context.Context, common.Address, []byte) ([]byte, error)) *Eth_SignMessage_Call { + _c.Call.Return(run) + return _c +} + // SignTx provides a mock function with given fields: ctx, fromAddress, tx, chainID func (_m *Eth) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { ret := _m.Called(ctx, fromAddress, tx, chainID) diff --git a/core/chains/evm/txm/clientwrappers/chain_client.go b/core/chains/evm/txm/clientwrappers/chain_client.go new file mode 100644 index 00000000000..27ebd34f882 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/chain_client.go @@ -0,0 +1,30 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type ChainClient struct { + c client.Client +} + +func NewChainClient(client client.Client) *ChainClient { + return &ChainClient{c: client} +} + +func (c *ChainClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return c.c.NonceAt(ctx, address, blockNumber) +} + +func (c *ChainClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + return c.c.PendingNonceAt(ctx, address) +} + +func (c *ChainClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return c.c.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go new file mode 100644 index 00000000000..7606245078d --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -0,0 +1,125 @@ +package clientwrappers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "net/url" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type DualBroadcastClientKeystore interface { + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) +} + +type DualBroadcastClient struct { + c client.Client + keystore DualBroadcastClientKeystore + customUrl *url.URL +} + +func NewDualBroadcastClient(c client.Client, keystore DualBroadcastClientKeystore, customUrl *url.URL) *DualBroadcastClient { + return &DualBroadcastClient{ + c: c, + keystore: keystore, + customUrl: customUrl, + } +} + +func (d *DualBroadcastClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return d.c.NonceAt(ctx, address, blockNumber) +} + +func (d *DualBroadcastClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":["%s","pending"]}`, address.String())) + response, err := d.signAndPostMessage(ctx, address, body, "") + if err != nil { + return 0, err + } + + nonce, err := hexutil.DecodeUint64(response) + if err != nil { + return 0, fmt.Errorf("failed to decode response %v into uint64: %w", response, err) + } + return nonce, nil +} + +func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + meta, err := tx.GetMeta() + if err != nil { + return err + } + if meta.DualBroadcast && !tx.IsPurgeable { + data, err := attempt.SignedTransaction.MarshalBinary() + if err != nil { + return err + } + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, string(data))) + if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, meta.DualBroadcastParams); err != nil { + return err + } + return nil + } else { + return d.c.SendTransaction(ctx, attempt.SignedTransaction) + } +} + +func (d *DualBroadcastClient) signAndPostMessage(ctx context.Context, address common.Address, body []byte, urlParams string) (result string, err error) { + bodyReader := bytes.NewReader(body) + postReq, err := http.NewRequestWithContext(ctx, http.MethodPost, d.customUrl.String()+"?"+urlParams, bodyReader) + if err != nil { + return + } + + hashedBody := crypto.Keccak256Hash(body).Hex() + signedMessage, err := d.keystore.SignMessage(ctx, address, []byte(hashedBody)) + if err != nil { + return + } + + postReq.Header.Add("X-Flashbots-signature", address.String()+":"+hexutil.Encode(signedMessage)) + postReq.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(postReq) + if err != nil { + return result, fmt.Errorf("request %v failed: %w", postReq, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("request %v failed with status: %d", postReq, resp.StatusCode) + } + + keyJson, err := io.ReadAll(resp.Body) + if err != nil { + return + } + var response postResponse + err = json.Unmarshal(keyJson, &response) + if err != nil { + return result, fmt.Errorf("failed to unmarshal response into struct: %w: %s", err, string(keyJson)) + } + if response.Error.Message != "" { + return result, errors.New(response.Error.Message) + } + return response.Result, nil +} + +type postResponse struct { + Result string `json:"result,omitempty"` + Error postError +} + +type postError struct { + Message string `json:"message,omitempty"` +} diff --git a/core/chains/evm/txm/clientwrappers/geth_client.go b/core/chains/evm/txm/clientwrappers/geth_client.go new file mode 100644 index 00000000000..d97e5cfae35 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/geth_client.go @@ -0,0 +1,51 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" +) + +type GethClient struct { + *ethclient.Client +} + +func NewGethClient(client *ethclient.Client) *GethClient { + return &GethClient{ + Client: client, + } +} + +func (g *GethClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { + return g.Client.Client().BatchCallContext(ctx, b) +} + +func (g *GethClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + return g.Client.Client().CallContext(ctx, result, method, args...) +} + +func (g *GethClient) CallContract(ctx context.Context, message ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + var hex hexutil.Bytes + err := g.CallContext(ctx, &hex, "eth_call", client.ToBackwardCompatibleCallArg(message), client.ToBackwardCompatibleBlockNumArg(blockNumber)) + return hex, err +} + +func (g *GethClient) HeadByNumber(ctx context.Context, number *big.Int) (*evmtypes.Head, error) { + hexNumber := client.ToBlockNumArg(number) + args := []interface{}{hexNumber, false} + head := new(evmtypes.Head) + err := g.CallContext(ctx, head, "eth_getBlockByNumber", args...) + return head, err +} + +func (g *GethClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return g.Client.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/mocks/attempt_builder.go b/core/chains/evm/txm/mocks/attempt_builder.go index ef02397cf6b..aad06745cf4 100644 --- a/core/chains/evm/txm/mocks/attempt_builder.go +++ b/core/chains/evm/txm/mocks/attempt_builder.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.46.3. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go index 334580587e5..03849ad7e82 100644 --- a/core/chains/evm/txm/mocks/client.go +++ b/core/chains/evm/txm/mocks/client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.46.3. DO NOT EDIT. package mocks @@ -10,7 +10,7 @@ import ( mock "github.com/stretchr/testify/mock" - types "github.com/ethereum/go-ethereum/core/types" + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) // Client is an autogenerated mock type for the Client type @@ -141,17 +141,17 @@ func (_c *Client_PendingNonceAt_Call) RunAndReturn(run func(context.Context, com return _c } -// SendTransaction provides a mock function with given fields: _a0, _a1 -func (_m *Client) SendTransaction(_a0 context.Context, _a1 *types.Transaction) error { - ret := _m.Called(_a0, _a1) +// SendTransaction provides a mock function with given fields: ctx, tx, attempt +func (_m *Client) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + ret := _m.Called(ctx, tx, attempt) if len(ret) == 0 { panic("no return value specified for SendTransaction") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction, *types.Attempt) error); ok { + r0 = rf(ctx, tx, attempt) } else { r0 = ret.Error(0) } @@ -165,15 +165,16 @@ type Client_SendTransaction_Call struct { } // SendTransaction is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *types.Transaction -func (_e *Client_Expecter) SendTransaction(_a0 interface{}, _a1 interface{}) *Client_SendTransaction_Call { - return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", _a0, _a1)} +// - ctx context.Context +// - tx *types.Transaction +// - attempt *types.Attempt +func (_e *Client_Expecter) SendTransaction(ctx interface{}, tx interface{}, attempt interface{}) *Client_SendTransaction_Call { + return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", ctx, tx, attempt)} } -func (_c *Client_SendTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Client_SendTransaction_Call { +func (_c *Client_SendTransaction_Call) Run(run func(ctx context.Context, tx *types.Transaction, attempt *types.Attempt)) *Client_SendTransaction_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Transaction)) + run(args[0].(context.Context), args[1].(*types.Transaction), args[2].(*types.Attempt)) }) return _c } @@ -183,7 +184,7 @@ func (_c *Client_SendTransaction_Call) Return(_a0 error) *Client_SendTransaction return _c } -func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction) error) *Client_SendTransaction_Call { +func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction, *types.Attempt) error) *Client_SendTransaction_Call { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txm/mocks/tx_store.go b/core/chains/evm/txm/mocks/tx_store.go index a475bd996ad..866e095b1a1 100644 --- a/core/chains/evm/txm/mocks/tx_store.go +++ b/core/chains/evm/txm/mocks/tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.46.3. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go index 486a978db7b..6718ce8de95 100644 --- a/core/chains/evm/txm/stuck_tx_detector.go +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -86,12 +86,14 @@ func (s *stuckTxDetector) dualBroadcastDetection(tx *types.Transaction) (bool, e var apiResponse ApiResponse err = json.Unmarshal(body, &apiResponse) if err != nil { - return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w: %s", tx.ID, attempt.Hash, err, string(body)) } switch apiResponse.Status { case ApiStatusPending, ApiStatusIncluded: return false, nil case ApiStatusFailed, ApiStatusCancelled: + s.lggr.Debugf("TxID: %v with attempHash: %v was marked as failed/cancelled by the RPC. Transaction is now considered stuck and will be purged.", + tx.ID, attempt.Hash) return true, nil case ApiStatusUnknown: continue diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 7144b556794..0feff9c9519 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -8,7 +8,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - evmtypes "github.com/ethereum/go-ethereum/core/types" "github.com/jpillora/backoff" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -27,7 +26,7 @@ const ( type Client interface { PendingNonceAt(context.Context, common.Address) (uint64, error) NonceAt(context.Context, common.Address, *big.Int) (uint64, error) - SendTransaction(context.Context, *evmtypes.Transaction) error + SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error } type TxStore interface { @@ -299,7 +298,7 @@ func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, a func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transaction, attempt *types.Attempt, address common.Address) (err error) { start := time.Now() - txErr := t.client.SendTransaction(ctx, attempt.SignedTransaction) + txErr := t.client.SendTransaction(ctx, tx, attempt) tx.AttemptCount++ t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 7e1028dccb7..6732f6ea516 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -161,7 +161,8 @@ type TxMeta struct { SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` // Dual Broadcast - DualBroadcast bool `json:"DualBroadcast,omitempty"` + DualBroadcast bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams string `json:"DualBroadcastParams,omitempty"` } type QueueingTxStrategy struct { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 7e6493b4aa0..070a61c8ede 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/clientwrappers" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) @@ -140,7 +141,13 @@ func NewTxmv2( RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, addresses) + var c txm.Client + if chainConfig.ChainType() == chaintype.ChainDualBroadcast { + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txConfig.AutoPurge().DetectionApiUrl()) + } else { + c = clientwrappers.NewChainClient(client) + } + t := txm.NewTxm(lggr, chainID, c, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, addresses) return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr), nil } From f9af9e15fa313a208aa64bc5c6cac88dba6e1536 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 12 Nov 2024 14:40:32 +0200 Subject: [PATCH 18/73] Add support for TXMv2 --- core/chains/evm/config/chain_scoped.go | 4 + core/chains/evm/config/chain_scoped_txmv2.go | 25 ++ core/chains/evm/config/config.go | 7 + core/chains/evm/config/toml/config.go | 27 ++- core/chains/evm/config/toml/defaults.go | 1 + .../evm/config/toml/defaults/fallback.toml | 3 + .../clientwrappers/dual_broadcast_client.go | 4 +- core/chains/evm/txm/dummy_keystore.go | 13 + core/chains/evm/txm/types/transaction.go | 2 +- core/chains/evm/txmgr/builder.go | 12 +- core/chains/legacyevm/evm_txm.go | 43 ++-- core/config/docs/chains-evm.toml | 8 + docs/CONFIG.md | 222 ++++++++++++++++++ 13 files changed, 346 insertions(+), 25 deletions(-) create mode 100644 core/chains/evm/config/chain_scoped_txmv2.go diff --git a/core/chains/evm/config/chain_scoped.go b/core/chains/evm/config/chain_scoped.go index de89272b5e2..3a7ff43d8a6 100644 --- a/core/chains/evm/config/chain_scoped.go +++ b/core/chains/evm/config/chain_scoped.go @@ -52,6 +52,10 @@ func (e *EVMConfig) BalanceMonitor() BalanceMonitor { return &balanceMonitorConfig{c: e.C.BalanceMonitor} } +func (e *EVMConfig) TxmV2() TxmV2 { + return &txmv2Config{c: e.C.TxmV2} +} + func (e *EVMConfig) Transactions() Transactions { return &transactionsConfig{c: e.C.Transactions} } diff --git a/core/chains/evm/config/chain_scoped_txmv2.go b/core/chains/evm/config/chain_scoped_txmv2.go new file mode 100644 index 00000000000..5422f636443 --- /dev/null +++ b/core/chains/evm/config/chain_scoped_txmv2.go @@ -0,0 +1,25 @@ +package config + +import ( + "net/url" + "time" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" +) + +type txmv2Config struct { + c toml.TxmV2 +} + +func (t *txmv2Config) Enabled() bool { + return *t.c.Enabled +} + +func (t *txmv2Config) BlockTime() *time.Duration { + d := t.c.BlockTime.Duration() + return &d +} + +func (t *txmv2Config) CustomUrl() *url.URL { + return t.c.CustomUrl.URL() +} diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index f2a571f94b0..fe049a173db 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -18,6 +18,7 @@ import ( type EVM interface { HeadTracker() HeadTracker BalanceMonitor() BalanceMonitor + TxmV2() TxmV2 Transactions() Transactions GasEstimator() GasEstimator OCR() OCR @@ -102,6 +103,12 @@ type ClientErrors interface { TooManyResults() string } +type TxmV2 interface { + Enabled() bool + BlockTime() *time.Duration + CustomUrl() *url.URL +} + type Transactions interface { ForwardersEnabled() bool ReaperInterval() time.Duration diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 0505449943e..5c762508ac0 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -300,8 +300,10 @@ func (c *EVMConfig) ValidateConfig() (err error) { is := c.ChainType.ChainType() if is != must { if must == "" { - err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), - Msg: "must not be set with this chain id"}) + if c.ChainType.ChainType() != chaintype.ChainDualBroadcast { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), + Msg: "must not be set with this chain id"}) + } } else { err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), Msg: fmt.Sprintf("only %q can be used with this chain id", must)}) @@ -387,6 +389,7 @@ type Chain struct { FinalizedBlockOffset *uint32 NoNewFinalizedHeadsThreshold *commonconfig.Duration + TxmV2 TxmV2 `toml:",omitempty"` Transactions Transactions `toml:",omitempty"` BalanceMonitor BalanceMonitor `toml:",omitempty"` GasEstimator GasEstimator `toml:",omitempty"` @@ -471,6 +474,26 @@ func (c *Chain) ValidateConfig() (err error) { return } +type TxmV2 struct { + Enabled *bool `toml:",omitempty"` + BlockTime *commonconfig.Duration `toml:",omitempty"` + CustomUrl *commonconfig.URL `toml:",omitempty"` +} + +func (t *TxmV2) setFrom(f *TxmV2) { + if v := f.Enabled; v != nil { + t.Enabled = f.Enabled + } + + if v := f.BlockTime; v != nil { + t.BlockTime = f.BlockTime + } + + if v := f.CustomUrl; v != nil { + t.CustomUrl = f.CustomUrl + } +} + type Transactions struct { ForwardersEnabled *bool MaxInFlight *uint32 diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 0885d94e6df..5ce014921f4 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -241,6 +241,7 @@ func (c *Chain) SetFrom(f *Chain) { c.NoNewFinalizedHeadsThreshold = v } + c.TxmV2.setFrom(&f.TxmV2) c.Transactions.setFrom(&f.Transactions) c.BalanceMonitor.setFrom(&f.BalanceMonitor) c.GasEstimator.setFrom(&f.GasEstimator) diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index ab349ee4688..44228968fc6 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -18,6 +18,9 @@ FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0' LogBroadcasterEnabled = true +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index 7606245078d..8ae2c68172a 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -60,12 +60,12 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra if err != nil { return err } - if meta.DualBroadcast && !tx.IsPurgeable { + if meta!= nil && meta.DualBroadcast && !tx.IsPurgeable { data, err := attempt.SignedTransaction.MarshalBinary() if err != nil { return err } - body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, string(data))) + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, hexutil.Encode(data))) if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, meta.DualBroadcastParams); err != nil { return err } diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 7299a9e1803..047c0b38736 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -34,3 +35,15 @@ func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx } return nil, fmt.Errorf("private key for address: %v not found", fromAddress) } + +func (k *DummyKeystore) SignMessage(ctx context.Context, address common.Address, data []byte) ([]byte, error) { + key, exists := k.privateKeyMap[address] + if !exists { + return nil, fmt.Errorf("private key for address: %v not found", address) + } + signature, err := crypto.Sign(accounts.TextHash(data), key) + if err != nil { + return nil, fmt.Errorf("failed to sign message for address: %v", address) + } + return signature, nil +} diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 6732f6ea516..ca146f8936e 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -78,7 +78,7 @@ func (t *Transaction) DeepCopy() *Transaction { } func (t *Transaction) GetMeta() (*TxMeta, error) { - if t.Meta != nil { + if t.Meta == nil { return nil, nil } var m TxMeta diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 070a61c8ede..d83a3ba21d2 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -94,12 +94,12 @@ func NewEvmTxm( return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError) } -func NewTxmv2( +func NewTxmV2( ds sqlutil.DataSource, chainConfig ChainConfig, fCfg FeeConfig, txConfig config.Transactions, - blockTime time.Duration, + txmV2Config config.TxmV2, client client.Client, lggr logger.Logger, logPoller logpoller.LogPoller, @@ -118,9 +118,9 @@ func NewTxmv2( var stuckTxDetector txm.StuckTxDetector if txConfig.AutoPurge().Enabled() { stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ - BlockTime: blockTime, + BlockTime: *txmV2Config.BlockTime(), StuckTxBlockThreshold: uint16(*txConfig.AutoPurge().Threshold()), - DetectionApiUrl: txConfig.AutoPurge().DetectionApiUrl().Path, + DetectionApiUrl: txConfig.AutoPurge().DetectionApiUrl().String(), } stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) } @@ -137,13 +137,13 @@ func NewTxmv2( inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, addresses, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), - BlockTime: blockTime, //TODO: create new config + BlockTime: *txmV2Config.BlockTime(), RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } var c txm.Client if chainConfig.ChainType() == chaintype.ChainDualBroadcast { - c = clientwrappers.NewDualBroadcastClient(client, keyStore, txConfig.AutoPurge().DetectionApiUrl()) + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomUrl()) } else { c = clientwrappers.NewChainClient(client) } diff --git a/core/chains/legacyevm/evm_txm.go b/core/chains/legacyevm/evm_txm.go index 3a96a9da937..1192462156e 100644 --- a/core/chains/legacyevm/evm_txm.go +++ b/core/chains/legacyevm/evm_txm.go @@ -55,20 +55,35 @@ func newEvmTxm( } if opts.GenTxManager == nil { - txm, err = txmgr.NewTxm( - ds, - cfg, - txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), - cfg.Transactions(), - cfg.NodePool().Errors(), - databaseConfig, - listenerConfig, - client, - lggr, - logPoller, - opts.KeyStore, - estimator, - headTracker) + if cfg.TxmV2().Enabled() { + txm, err = txmgr.NewTxmV2( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.TxmV2(), + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + ) + } else { + txm, err = txmgr.NewTxm( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.NodePool().Errors(), + databaseConfig, + listenerConfig, + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + headTracker) + } } else { txm = opts.GenTxManager(chainID) } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 62360cb02cb..1e50ff28585 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -129,6 +129,14 @@ ReaperThreshold = '168h' # Default # ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. ResendAfterThreshold = '1m' # Default +[EVM.TxmV2] +# Enabled enables TxmV2. +Enabled = false # Default +# BlockTime controls the frequency of the backfill loop of TxmV2. +BlockTime = '10s' # Example +# CustomUrl configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomUrl = 'https://example.api.io' # Example + [EVM.Transactions.AutoPurge] # Enabled enables or disables automatically purging transactions that have been idenitified as terminally stuck (will never be included on-chain). This feature is only expected to be used by ZK chains. Enabled = false # Default diff --git a/docs/CONFIG.md b/docs/CONFIG.md index cb7a2710d4e..a86c010471a 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1983,6 +1983,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2087,6 +2090,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2191,6 +2197,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2295,6 +2304,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2400,6 +2412,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '13m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2508,6 +2523,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2612,6 +2630,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2717,6 +2738,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2821,6 +2845,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '45s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2924,6 +2951,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3027,6 +3057,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3131,6 +3164,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3236,6 +3272,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3340,6 +3379,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3444,6 +3486,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '6m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3548,6 +3593,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3652,6 +3700,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3756,6 +3807,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3860,6 +3914,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3968,6 +4025,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4075,6 +4135,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4179,6 +4242,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4283,6 +4349,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4390,6 +4459,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4498,6 +4570,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4606,6 +4681,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4709,6 +4787,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4813,6 +4894,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4917,6 +5001,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5021,6 +5108,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5125,6 +5215,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5228,6 +5321,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5333,6 +5429,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '2h0m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5441,6 +5540,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5549,6 +5651,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5653,6 +5758,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5756,6 +5864,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5860,6 +5971,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5968,6 +6082,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6073,6 +6190,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6181,6 +6301,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6289,6 +6412,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6396,6 +6522,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6500,6 +6629,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6604,6 +6736,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6708,6 +6843,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '45m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6813,6 +6951,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6924,6 +7065,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7033,6 +7177,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7136,6 +7283,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7239,6 +7389,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7343,6 +7496,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7447,6 +7603,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7550,6 +7709,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '12m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7654,6 +7816,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7762,6 +7927,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '12m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7871,6 +8039,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7979,6 +8150,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8086,6 +8260,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8193,6 +8370,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8301,6 +8481,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8409,6 +8592,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8513,6 +8699,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8621,6 +8810,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8725,6 +8917,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9066,6 +9261,33 @@ ResendAfterThreshold = '1m' # Default ``` ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. +## EVM.TxmV2 +```toml +[EVM.TxmV2] +Enabled = false # Default +BlockTime = '10s' # Example +CustomUrl = 'https://example.api.io' # Example +``` + + +### Enabled +```toml +Enabled = false # Default +``` +Enabled enables TxmV2. + +### BlockTime +```toml +BlockTime = '10s' # Example +``` +BlockTime controls the frequency of the backfill loop of TxmV2. + +### CustomUrl +```toml +CustomUrl = 'https://example.api.io' # Example +``` +CustomUrl configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. + ## EVM.Transactions.AutoPurge ```toml [EVM.Transactions.AutoPurge] From 0bfe0112acb09d170083f2a535068e3334a8025c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 12 Nov 2024 15:33:26 +0200 Subject: [PATCH 19/73] Fix txm to work with enabled addresses from keystore --- .mockery.yaml | 1 + core/chains/evm/txm/attempt_builder.go | 24 ++--- core/chains/evm/txm/dummy_keystore.go | 7 ++ core/chains/evm/txm/mocks/attempt_builder.go | 2 +- core/chains/evm/txm/mocks/client.go | 2 +- core/chains/evm/txm/mocks/keystore.go | 98 +++++++++++++++++++ core/chains/evm/txm/mocks/tx_store.go | 2 +- core/chains/evm/txm/orchestrator.go | 50 +++++----- .../evm/txm/storage/inmemory_store_manager.go | 22 ++++- .../storage/inmemory_store_manager_test.go | 34 +++++++ core/chains/evm/txm/txm.go | 50 ++++++---- core/chains/evm/txm/txm_test.go | 40 ++++---- core/chains/evm/txm/types/transaction.go | 2 +- core/chains/evm/txmgr/builder.go | 17 +--- 14 files changed, 252 insertions(+), 99 deletions(-) create mode 100644 core/chains/evm/txm/mocks/keystore.go create mode 100644 core/chains/evm/txm/storage/inmemory_store_manager_test.go diff --git a/.mockery.yaml b/.mockery.yaml index 5e219c5d228..c4ee5dd5a33 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -98,6 +98,7 @@ packages: Client: TxStore: AttemptBuilder: + Keystore: github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr: interfaces: ChainConfig: diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index 4879b91ccdf..0350f6d09b1 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -15,28 +15,28 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) -type Keystore interface { +type AttemptBuilderKeystore interface { SignTx(ctx context.Context, fromAddress common.Address, tx *evmtypes.Transaction, chainID *big.Int) (*evmtypes.Transaction, error) } type attemptBuilder struct { - chainID *big.Int - priceMaxMap map[common.Address]*assets.Wei - estimator gas.EvmFeeEstimator - keystore Keystore + chainID *big.Int + priceMax *assets.Wei + estimator gas.EvmFeeEstimator + keystore AttemptBuilderKeystore } -func NewAttemptBuilder(chainID *big.Int, priceMaxMap map[common.Address]*assets.Wei, estimator gas.EvmFeeEstimator, keystore Keystore) *attemptBuilder { +func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore AttemptBuilderKeystore) *attemptBuilder { return &attemptBuilder{ - chainID: chainID, - priceMaxMap: priceMaxMap, - estimator: estimator, - keystore: keystore, + chainID: chainID, + priceMax: priceMax, + estimator: estimator, + keystore: keystore, } } func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { - fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMaxMap[tx.FromAddress], &tx.FromAddress, &tx.ToAddress) + fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx } func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { - bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMaxMap[tx.FromAddress], nil) + bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) if err != nil { return nil, err } diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 7299a9e1803..3456c06b1e3 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -34,3 +34,10 @@ func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx } return nil, fmt.Errorf("private key for address: %v not found", fromAddress) } + +func (k *DummyKeystore) EnabledAddressesForChain(_ context.Context, _ *big.Int) (addresses []common.Address, err error) { + for address := range k.privateKeyMap { + addresses = append(addresses, address) + } + return +} diff --git a/core/chains/evm/txm/mocks/attempt_builder.go b/core/chains/evm/txm/mocks/attempt_builder.go index ef02397cf6b..aad06745cf4 100644 --- a/core/chains/evm/txm/mocks/attempt_builder.go +++ b/core/chains/evm/txm/mocks/attempt_builder.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.46.3. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go index 334580587e5..533298625e5 100644 --- a/core/chains/evm/txm/mocks/client.go +++ b/core/chains/evm/txm/mocks/client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.46.3. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txm/mocks/keystore.go b/core/chains/evm/txm/mocks/keystore.go new file mode 100644 index 00000000000..c61ddb8db04 --- /dev/null +++ b/core/chains/evm/txm/mocks/keystore.go @@ -0,0 +1,98 @@ +// Code generated by mockery v2.46.3. DO NOT EDIT. + +package mocks + +import ( + context "context" + big "math/big" + + common "github.com/ethereum/go-ethereum/common" + + mock "github.com/stretchr/testify/mock" +) + +// Keystore is an autogenerated mock type for the Keystore type +type Keystore struct { + mock.Mock +} + +type Keystore_Expecter struct { + mock *mock.Mock +} + +func (_m *Keystore) EXPECT() *Keystore_Expecter { + return &Keystore_Expecter{mock: &_m.Mock} +} + +// EnabledAddressesForChain provides a mock function with given fields: ctx, chainID +func (_m *Keystore) EnabledAddressesForChain(ctx context.Context, chainID *big.Int) ([]common.Address, error) { + ret := _m.Called(ctx, chainID) + + if len(ret) == 0 { + panic("no return value specified for EnabledAddressesForChain") + } + + var r0 []common.Address + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *big.Int) ([]common.Address, error)); ok { + return rf(ctx, chainID) + } + if rf, ok := ret.Get(0).(func(context.Context, *big.Int) []common.Address); ok { + r0 = rf(ctx, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]common.Address) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *big.Int) error); ok { + r1 = rf(ctx, chainID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Keystore_EnabledAddressesForChain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnabledAddressesForChain' +type Keystore_EnabledAddressesForChain_Call struct { + *mock.Call +} + +// EnabledAddressesForChain is a helper method to define mock.On call +// - ctx context.Context +// - chainID *big.Int +func (_e *Keystore_Expecter) EnabledAddressesForChain(ctx interface{}, chainID interface{}) *Keystore_EnabledAddressesForChain_Call { + return &Keystore_EnabledAddressesForChain_Call{Call: _e.mock.On("EnabledAddressesForChain", ctx, chainID)} +} + +func (_c *Keystore_EnabledAddressesForChain_Call) Run(run func(ctx context.Context, chainID *big.Int)) *Keystore_EnabledAddressesForChain_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*big.Int)) + }) + return _c +} + +func (_c *Keystore_EnabledAddressesForChain_Call) Return(addresses []common.Address, err error) *Keystore_EnabledAddressesForChain_Call { + _c.Call.Return(addresses, err) + return _c +} + +func (_c *Keystore_EnabledAddressesForChain_Call) RunAndReturn(run func(context.Context, *big.Int) ([]common.Address, error)) *Keystore_EnabledAddressesForChain_Call { + _c.Call.Return(run) + return _c +} + +// NewKeystore creates a new instance of Keystore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewKeystore(t interface { + mock.TestingT + Cleanup(func()) +}) *Keystore { + mock := &Keystore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/chains/evm/txm/mocks/tx_store.go b/core/chains/evm/txm/mocks/tx_store.go index a475bd996ad..866e095b1a1 100644 --- a/core/chains/evm/txm/mocks/tx_store.go +++ b/core/chains/evm/txm/mocks/tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.46.3. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 2a456c6aa5f..a6836e3c20e 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -26,33 +26,16 @@ import ( evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) -// TODO: use this after the migration -//type TxmOrchestrator interface { -// services.Service -// Trigger(addr common.Address) -// CreateTransaction(ctx context.Context, txRequest *types.Transaction) (id int64, err error) -// GetForwarderForEOA(ctx context.Context, eoa common.Address) (forwarder common.Address, err error) -// GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID common.Address) (forwarder common.Address, err error) -// RegisterResumeCallback(fn ResumeCallback) -// SendNativeToken(ctx context.Context, chainID *big.Int, from, to common.Address, value *big.Int, gasLimit uint64) (tx *types.Transaction, err error) -// CountTransactionsByState(ctx context.Context, state types.TxState) (count int, err error) -// GetTransactionStatus(ctx context.Context, idempotencyKey string) (state commontypes.TransactionStatus, err error) -// //Reset(addr ADDR, abandon bool) error // Potentially will be replaced by Abandon -// -// // Testing methods(?) -// FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) -// FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []types.TxState, chainID *big.Int) (txs []*types.Transaction, err error) -// FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*types.Transaction, err error) -// FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []types.TxState, chainID *big.Int) (txes []*types.Transaction, err error) -// FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) -// FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) -//} - type OrchestratorTxStore interface { + Add(addresses ...common.Address) error FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*txmtypes.Transaction, int, error) FindTxWithIdempotencyKey(context.Context, *string) (*txmtypes.Transaction, error) } +type OrchestratorKeystore interface { + EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) +} + // Generics are necessary to keep TXMv2 backwards compatible type Orchestrator[ BLOCK_HASH types.Hashable, @@ -64,6 +47,7 @@ type Orchestrator[ txm *Txm txStore OrchestratorTxStore fwdMgr *forwarders.FwdMgr + keystore OrchestratorKeystore resumeCallback txmgr.ResumeCallback } @@ -73,19 +57,31 @@ func NewTxmOrchestrator[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]]( txm *Txm, txStore OrchestratorTxStore, fwdMgr *forwarders.FwdMgr, + keystore OrchestratorKeystore, ) *Orchestrator[BLOCK_HASH, HEAD] { return &Orchestrator[BLOCK_HASH, HEAD]{ - lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), - chainID: chainID, - txm: txm, - txStore: txStore, - fwdMgr: fwdMgr, + lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), + chainID: chainID, + txm: txm, + txStore: txStore, + keystore: keystore, + fwdMgr: fwdMgr, } } func (o *Orchestrator[BLOCK_HASH, HEAD]) Start(ctx context.Context) error { return o.StartOnce("Orchestrator", func() error { var ms services.MultiStart + addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) + if err != nil { + return err + } + for _, address := range addresses { + err := o.txStore.Add(address) + if err != nil { + return err + } + } if err := ms.Start(ctx, o.txm); err != nil { return fmt.Errorf("Orchestrator: Txm failed to start: %w", err) } diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index 824b533c6a9..e680ee79391 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -14,15 +14,17 @@ import ( const StoreNotFoundForAddress string = "InMemoryStore for address: %v not found" type InMemoryStoreManager struct { + lggr logger.Logger + chainID *big.Int InMemoryStoreMap map[common.Address]*InMemoryStore } -func NewInMemoryStoreManager(lggr logger.Logger, addresses []common.Address, chainID *big.Int) *InMemoryStoreManager { +func NewInMemoryStoreManager(lggr logger.Logger, chainID *big.Int) *InMemoryStoreManager { inMemoryStoreMap := make(map[common.Address]*InMemoryStore) - for _, address := range addresses { - inMemoryStoreMap[address] = NewInMemoryStore(lggr, address, chainID) - } - return &InMemoryStoreManager{InMemoryStoreMap: inMemoryStoreMap} + return &InMemoryStoreManager{ + lggr: lggr, + chainID: chainID, + InMemoryStoreMap: inMemoryStoreMap} } func (m *InMemoryStoreManager) AbandonPendingTransactions(_ context.Context, fromAddress common.Address) error { @@ -33,6 +35,16 @@ func (m *InMemoryStoreManager) AbandonPendingTransactions(_ context.Context, fro return fmt.Errorf(StoreNotFoundForAddress, fromAddress) } +func (m *InMemoryStoreManager) Add(addresses ...common.Address) error { + for _, address := range addresses { + if _, exists := m.InMemoryStoreMap[address]; exists { + return fmt.Errorf("address %v already exists in store manager", address) + } + m.InMemoryStoreMap[address] = NewInMemoryStore(m.lggr, address, m.chainID) + } + return nil +} + func (m *InMemoryStoreManager) AppendAttemptToTransaction(_ context.Context, txNonce uint64, fromAddress common.Address, attempt *types.Attempt) error { if store, exists := m.InMemoryStoreMap[fromAddress]; exists { store.AppendAttemptToTransaction(txNonce, attempt) diff --git a/core/chains/evm/txm/storage/inmemory_store_manager_test.go b/core/chains/evm/txm/storage/inmemory_store_manager_test.go new file mode 100644 index 00000000000..27210898217 --- /dev/null +++ b/core/chains/evm/txm/storage/inmemory_store_manager_test.go @@ -0,0 +1,34 @@ +package storage + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" +) + +func TestAdd(t *testing.T) { + t.Parallel() + + fromAddress := testutils.NewAddress() + m := NewInMemoryStoreManager(logger.Test(t), testutils.FixtureChainID) + // Adds a new address + err := m.Add(fromAddress) + assert.NoError(t, err) + assert.Equal(t, 1, len(m.InMemoryStoreMap)) + + // Fails if address exists + err = m.Add(fromAddress) + assert.Error(t, err) + + // Adds multiple addresses + fromAddress1 := testutils.NewAddress() + fromAddress2 := testutils.NewAddress() + addresses := []common.Address{fromAddress1, fromAddress2} + err = m.Add(addresses...) + assert.NoError(t, err) + assert.Equal(t, 3, len(m.InMemoryStoreMap)) +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index b188b478674..b90368c20c2 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -59,6 +59,10 @@ type StuckTxDetector interface { DetectStuckTransaction(tx *types.Transaction) (bool, error) } +type Keystore interface { + EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) +} + type Config struct { EIP1559 bool BlockTime time.Duration @@ -68,15 +72,15 @@ type Config struct { type Txm struct { services.StateMachine - lggr logger.SugaredLogger - enabledAddresses []common.Address - chainID *big.Int - client Client - attemptBuilder AttemptBuilder - errorHandler ErrorHandler - stuckTxDetector StuckTxDetector - txStore TxStore - config Config + lggr logger.SugaredLogger + chainID *big.Int + client Client + attemptBuilder AttemptBuilder + errorHandler ErrorHandler + stuckTxDetector StuckTxDetector + txStore TxStore + keystore Keystore + config Config nonceMapMu sync.Mutex nonceMap map[common.Address]uint64 @@ -86,25 +90,29 @@ type Txm struct { wg sync.WaitGroup } -func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, enabledAddresses []common.Address) *Txm { +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, keystore Keystore) *Txm { return &Txm{ - lggr: logger.Sugared(logger.Named(lggr, "Txm")), - enabledAddresses: enabledAddresses, - chainID: chainID, - client: client, - attemptBuilder: attemptBuilder, - txStore: txStore, - config: config, - nonceMap: make(map[common.Address]uint64), - triggerCh: make(map[common.Address]chan struct{}), + lggr: logger.Sugared(logger.Named(lggr, "Txm")), + keystore: keystore, + chainID: chainID, + client: client, + attemptBuilder: attemptBuilder, + txStore: txStore, + config: config, + nonceMap: make(map[common.Address]uint64), + triggerCh: make(map[common.Address]chan struct{}), } } -func (t *Txm) Start(context.Context) error { +func (t *Txm) Start(ctx context.Context) error { return t.StartOnce("Txm", func() error { t.stopCh = make(chan struct{}) - for _, address := range t.enabledAddresses { + addresses, err := t.keystore.EnabledAddressesForChain(ctx, t.chainID) + if err != nil { + return err + } + for _, address := range addresses { err := t.startAddress(address) if err != nil { return err diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index d9d28b32617..b3493eecf62 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -29,17 +29,20 @@ func TestLifecycle(t *testing.T) { address2 := testutils.NewAddress() assert.NotEqual(t, address1, address2) addresses := []common.Address{address1, address2} + keystore := mocks.NewKeystore(t) + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return(addresses, nil) t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() assert.Error(t, txm.Start(tests.Context(t))) }) t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) - txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + assert.NoError(t, txStore.Add(addresses...)) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) var nonce uint64 = 0 // Start client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() @@ -58,20 +61,22 @@ func TestTrigger(t *testing.T) { t.Parallel() address := testutils.NewAddress() - addresses := []common.Address{address} + keystore := mocks.NewKeystore(t) + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { - txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, addresses) + txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, keystore) txm.Trigger(address) assert.Error(t, txm.Trigger(address), "Txm unstarted") }) t.Run("executes Trigger", func(t *testing.T) { lggr := logger.Test(t) - txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + assert.NoError(t, txStore.Add(address)) client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) var nonce uint64 = 0 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() @@ -88,12 +93,12 @@ func TestBroadcastTransaction(t *testing.T) { ab := mocks.NewAttemptBuilder(t) config := Config{} address := testutils.NewAddress() - addresses := []common.Address{address} + keystore := mocks.NewKeystore(t) t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) assert.Error(t, err) assert.False(t, bo) @@ -104,7 +109,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions+1), nil).Once() - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) txm.broadcastTransaction(ctx, address) tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) @@ -112,7 +117,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) txm.setNonce(address, 1) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions/3), nil).Twice() @@ -129,7 +134,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() bo, err := txm.broadcastTransaction(ctx, address) assert.False(t, bo) @@ -139,8 +144,9 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("returns if there are no unstarted transactions", func(t *testing.T) { lggr := logger.Test(t) - txStore := storage.NewInMemoryStoreManager(lggr, addresses, testutils.FixtureChainID) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, addresses) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + assert.NoError(t, txStore.Add(address)) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) assert.NoError(t, err) assert.False(t, bo) @@ -157,10 +163,10 @@ func TestBackfillTransactions(t *testing.T) { storage := mocks.NewTxStore(t) config := Config{} address := testutils.NewAddress() - addresses := []common.Address{address} + keystore := mocks.NewKeystore(t) t.Run("fails if latest nonce fetching fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() bo, err := txm.backfillTransactions(ctx, address) assert.Error(t, err) @@ -169,7 +175,7 @@ func TestBackfillTransactions(t *testing.T) { }) t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, addresses) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil) storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")) bo, err := txm.backfillTransactions(ctx, address) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index ff380206267..0cb11ad8f02 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -78,7 +78,7 @@ func (t *Transaction) DeepCopy() *Transaction { } func (t *Transaction) GetMeta() (*TxMeta, error) { - if t.Meta != nil { + if t.Meta == nil { return nil, nil } var m TxMeta diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 15bfba13ba1..568929e3551 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -10,7 +10,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" @@ -113,24 +112,16 @@ func NewTxmv2( } chainID := client.ConfiguredChainID() - addresses, err := keyStore.EnabledAddressesForChain(context.TODO(), chainID) - if err != nil { - return nil, err - } - priceMaxMap := make(map[common.Address]*assets.Wei) - for _, address := range addresses { - priceMaxMap[address] = fCfg.PriceMaxKey(address) - } - attemptBuilder := txm.NewAttemptBuilder(chainID, priceMaxMap, estimator, keyStore) - inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, addresses, chainID) + attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMax(), estimator, keyStore) + inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), BlockTime: blockTime, //TODO: create new config RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, addresses) - return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr), nil + t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, keyStore) + return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore), nil } // NewEvmResender creates a new concrete EvmResender From 45beb8eb54964d01c8476fc661697e9eab948fa3 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 12 Nov 2024 16:23:51 +0200 Subject: [PATCH 20/73] Fix orchestrator's monitoring call --- core/chains/evm/txm/orchestrator.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index a6836e3c20e..252a2c7cec4 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -234,8 +234,20 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, } func (o *Orchestrator[BLOCK_HASH, HEAD]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (uint32, error) { - _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, common.Address{}) - return uint32(count), err + addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) + if err != nil { + return 0, err + } + total := 0 + for _, address := range addresses { + _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, address) + if err != nil { + return 0, err + } + total += count + } + + return uint32(total), err } func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { From 8786bc5802af50c52635a477b764a368184f1199 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 12 Nov 2024 17:31:08 +0200 Subject: [PATCH 21/73] Fix AttemptBuilder --- core/chains/evm/txm/attempt_builder.go | 20 ++++++++++---------- core/chains/evm/txm/orchestrator.go | 22 ++++++++++++++++------ core/chains/evm/txmgr/builder.go | 2 +- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index 0350f6d09b1..78a26929e8c 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -20,23 +20,23 @@ type AttemptBuilderKeystore interface { } type attemptBuilder struct { - chainID *big.Int - priceMax *assets.Wei - estimator gas.EvmFeeEstimator - keystore AttemptBuilderKeystore + chainID *big.Int + priceMax *assets.Wei + gas.EvmFeeEstimator + keystore AttemptBuilderKeystore } func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore AttemptBuilderKeystore) *attemptBuilder { return &attemptBuilder{ - chainID: chainID, - priceMax: priceMax, - estimator: estimator, - keystore: keystore, + chainID: chainID, + priceMax: priceMax, + EvmFeeEstimator: estimator, + keystore: keystore, } } func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { - fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) + fee, estimatedGasLimit, err := a.EvmFeeEstimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx } func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { - bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) + bumpedFee, bumpedFeeLimit, err := a.EvmFeeEstimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) if err != nil { return nil, err } diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 252a2c7cec4..8c4f41e2232 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -36,6 +36,10 @@ type OrchestratorKeystore interface { EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } +type OrchestratorAttemptBuilder interface { + services.Service +} + // Generics are necessary to keep TXMv2 backwards compatible type Orchestrator[ BLOCK_HASH types.Hashable, @@ -48,6 +52,7 @@ type Orchestrator[ txStore OrchestratorTxStore fwdMgr *forwarders.FwdMgr keystore OrchestratorKeystore + attemptBuilder OrchestratorAttemptBuilder resumeCallback txmgr.ResumeCallback } @@ -58,20 +63,25 @@ func NewTxmOrchestrator[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]]( txStore OrchestratorTxStore, fwdMgr *forwarders.FwdMgr, keystore OrchestratorKeystore, + attemptBuilder OrchestratorAttemptBuilder, ) *Orchestrator[BLOCK_HASH, HEAD] { return &Orchestrator[BLOCK_HASH, HEAD]{ - lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), - chainID: chainID, - txm: txm, - txStore: txStore, - keystore: keystore, - fwdMgr: fwdMgr, + lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), + chainID: chainID, + txm: txm, + txStore: txStore, + keystore: keystore, + attemptBuilder: attemptBuilder, + fwdMgr: fwdMgr, } } func (o *Orchestrator[BLOCK_HASH, HEAD]) Start(ctx context.Context) error { return o.StartOnce("Orchestrator", func() error { var ms services.MultiStart + if err := ms.Start(ctx, o.attemptBuilder); err != nil { + return fmt.Errorf("Orchestrator: AttemptBuilder failed to start: %w", err) + } addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) if err != nil { return err diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index a597b913e0d..9be0556bd30 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -134,7 +134,7 @@ func NewTxmV2( } c := clientwrappers.NewChainClient(client) t := txm.NewTxm(lggr, chainID, c, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, keyStore) - return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore), nil + return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil } // NewEvmResender creates a new concrete EvmResender From 331a45d36a710e8e6bbbcace548cd9a405896cc5 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 12 Nov 2024 17:40:31 +0200 Subject: [PATCH 22/73] Enable DualBroadcast client --- core/chains/evm/txmgr/builder.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 9be0556bd30..40cae9d1bc9 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -132,7 +132,12 @@ func NewTxmV2( RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - c := clientwrappers.NewChainClient(client) + var c txm.Client + if chainConfig.ChainType() == chaintype.ChainDualBroadcast { + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomUrl()) + } else { + c = clientwrappers.NewChainClient(client) + } t := txm.NewTxm(lggr, chainID, c, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, keyStore) return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil } From 8f51afe0a744e049993c75b31c1cca93217f0dfc Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 13:01:40 +0200 Subject: [PATCH 23/73] Enable dual broadcast --- common/txmgr/txmgr.go | 31 ++++++++++++++++ common/txmgr/types/tx.go | 4 ++ .../ocrimpls/contract_transmitter_test.go | 3 +- core/chains/evm/txmgr/builder.go | 6 ++- core/chains/evm/txmgr/evm_tx_store_test.go | 2 +- core/chains/evm/txmgr/txmgr_test.go | 3 +- core/chains/legacyevm/evm_txm.go | 37 +++++++++++-------- .../headreporter/prometheus_reporter_test.go | 3 +- core/services/vrf/delegate_test.go | 2 +- core/services/vrf/v2/integration_v2_test.go | 2 +- core/services/vrf/v2/listener_v2_test.go | 2 +- 11 files changed, 70 insertions(+), 25 deletions(-) diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index 28d505e5e05..1a088ac7cd8 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -65,6 +65,19 @@ type TxManager[ GetTransactionStatus(ctx context.Context, transactionID string) (state commontypes.TransactionStatus, err error) } +type TxmV2Wrapper[ + CHAIN_ID types.ID, + HEAD types.Head[BLOCK_HASH], + ADDR types.Hashable, + TX_HASH types.Hashable, + BLOCK_HASH types.Hashable, + SEQ types.Sequence, + FEE feetypes.Fee, +] interface { + services.Service + CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) +} + type reset struct { // f is the function to execute between stopping/starting the // Broadcaster and Confirmer @@ -112,6 +125,7 @@ type Txm[ fwdMgr txmgrtypes.ForwarderManager[ADDR] txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] newErrorClassifier NewErrorClassifier + txmv2wrapper TxmV2Wrapper[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RegisterResumeCallback(fn ResumeCallback) { @@ -146,6 +160,7 @@ func NewTxm[ tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], finalizer txmgrtypes.Finalizer[BLOCK_HASH, HEAD], newErrorClassifierFunc NewErrorClassifier, + txmv2wrapper TxmV2Wrapper[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { b := Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ logger: logger.Sugared(lggr), @@ -168,6 +183,7 @@ func NewTxm[ tracker: tracker, newErrorClassifier: newErrorClassifierFunc, finalizer: finalizer, + txmv2wrapper: txmv2wrapper, } if txCfg.ResendAfterThreshold() <= 0 { @@ -206,6 +222,12 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Start(ctx return fmt.Errorf("Txm: Finalizer failed to start: %w", err) } + if b.txmv2wrapper != nil { + if err := ms.Start(ctx, b.txmv2wrapper); err != nil { + return fmt.Errorf("Txm: Txmv2 failed to start: %w", err) + } + } + b.logger.Info("Txm starting runLoop") b.wg.Add(1) go b.runLoop() @@ -459,6 +481,12 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) runLoop() if err != nil && (!errors.Is(err, services.ErrAlreadyStopped) || !errors.Is(err, services.ErrCannotStopUnstarted)) { b.logger.Errorw(fmt.Sprintf("Failed to Close Finalizer: %v", err), "err", err) } + if b.txmv2wrapper != nil { + err = b.txmv2wrapper.Close() + if err != nil && (!errors.Is(err, services.ErrAlreadyStopped) || !errors.Is(err, services.ErrCannotStopUnstarted)) { + b.logger.Errorw(fmt.Sprintf("Failed to Close Finalizer: %v", err), "err", err) + } + } return case <-keysChanged: // This check prevents the weird edge-case where you can select @@ -512,6 +540,9 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Trigger(ad func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { // Check for existing Tx with IdempotencyKey. If found, return the Tx and do nothing // Skipping CreateTransaction to avoid double send + if b.txmv2wrapper != nil && txRequest.Meta != nil && txRequest.Meta.DualBroadcast { + return b.txmv2wrapper.CreateTransaction(ctx, txRequest) + } if txRequest.IdempotencyKey != nil { var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) diff --git a/common/txmgr/types/tx.go b/common/txmgr/types/tx.go index b65f7edf6e5..ea3fc55330c 100644 --- a/common/txmgr/types/tx.go +++ b/common/txmgr/types/tx.go @@ -159,6 +159,10 @@ type TxMeta[ADDR types.Hashable, TX_HASH types.Hashable] struct { MessageIDs []string `json:"MessageIDs,omitempty"` // SeqNumbers is used by CCIP for tx to committed sequence numbers correlation in logs SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` + + // Dual Broadcast + DualBroadcast bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams string `json:"DualBroadcastParams,omitempty"` } type TxAttempt[ diff --git a/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go b/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go index 86ad8bef809..18c7ce7d2a4 100644 --- a/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go +++ b/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go @@ -478,7 +478,8 @@ func makeTestEvmTxm( lp, keyStore, estimator, - ht) + ht, + nil) require.NoError(t, err, "can't create tx manager") _, unsub := broadcaster.Subscribe(txm) diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 40cae9d1bc9..91593c90ced 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -42,6 +42,7 @@ func NewTxm( keyStore keystore.Eth, estimator gas.EvmFeeEstimator, headTracker latestAndFinalizedBlockHeadTracker, + txmv2wrapper TxManager, ) (txm TxManager, err error, ) { @@ -69,7 +70,7 @@ func NewTxm( if txConfig.ResendAfterThreshold() > 0 { evmResender = NewEvmResender(lggr, txStore, txmClient, evmTracker, keyStore, txmgr.DefaultResenderPollInterval, chainConfig, txConfig) } - txm = NewEvmTxm(chainID, txmCfg, txConfig, keyStore, lggr, checker, fwdMgr, txAttemptBuilder, txStore, evmBroadcaster, evmConfirmer, evmResender, evmTracker, evmFinalizer) + txm = NewEvmTxm(chainID, txmCfg, txConfig, keyStore, lggr, checker, fwdMgr, txAttemptBuilder, txStore, evmBroadcaster, evmConfirmer, evmResender, evmTracker, evmFinalizer, txmv2wrapper) return txm, nil } @@ -89,8 +90,9 @@ func NewEvmTxm( resender *Resender, tracker *Tracker, finalizer Finalizer, + txmv2wrapper TxManager, ) *Txm { - return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError) + return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError, txmv2wrapper) } func NewTxmV2( diff --git a/core/chains/evm/txmgr/evm_tx_store_test.go b/core/chains/evm/txmgr/evm_tx_store_test.go index 9e1f135e0b2..1a89d46c9e9 100644 --- a/core/chains/evm/txmgr/evm_tx_store_test.go +++ b/core/chains/evm/txmgr/evm_tx_store_test.go @@ -1383,7 +1383,7 @@ func TestORM_UpdateTxUnstartedToInProgress(t *testing.T) { evmTxmCfg := txmgr.NewEvmTxmConfig(ccfg.EVM()) ec := evmtest.NewEthClientMockWithDefaultChain(t) txMgr := txmgr.NewEvmTxm(ec.ConfiguredChainID(), evmTxmCfg, ccfg.EVM().Transactions(), nil, logger.Test(t), nil, nil, - nil, txStore, nil, nil, nil, nil, nil) + nil, txStore, nil, nil, nil, nil, nil, nil) err := txMgr.XXXTestAbandon(fromAddress) // mark transaction as abandoned require.NoError(t, err) diff --git a/core/chains/evm/txmgr/txmgr_test.go b/core/chains/evm/txmgr/txmgr_test.go index 7052a694719..b07100158bc 100644 --- a/core/chains/evm/txmgr/txmgr_test.go +++ b/core/chains/evm/txmgr/txmgr_test.go @@ -86,7 +86,8 @@ func makeTestEvmTxm( lp, keyStore, estimator, - ht) + ht, + nil) } func TestTxm_SendNativeToken_DoesNotSendToZero(t *testing.T) { diff --git a/core/chains/legacyevm/evm_txm.go b/core/chains/legacyevm/evm_txm.go index 1192462156e..37e9e9f7223 100644 --- a/core/chains/legacyevm/evm_txm.go +++ b/core/chains/legacyevm/evm_txm.go @@ -6,6 +6,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" evmconfig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas/rollups" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" @@ -55,8 +56,9 @@ func newEvmTxm( } if opts.GenTxManager == nil { + var txmv2 txmgr.TxManager if cfg.TxmV2().Enabled() { - txm, err = txmgr.NewTxmV2( + txmv2, err = txmgr.NewTxmV2( ds, cfg, txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), @@ -68,22 +70,25 @@ func newEvmTxm( opts.KeyStore, estimator, ) - } else { - txm, err = txmgr.NewTxm( - ds, - cfg, - txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), - cfg.Transactions(), - cfg.NodePool().Errors(), - databaseConfig, - listenerConfig, - client, - lggr, - logPoller, - opts.KeyStore, - estimator, - headTracker) + if cfg.ChainType() != chaintype.ChainDualBroadcast { + return txmv2, estimator, err + } } + txm, err = txmgr.NewTxm( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.NodePool().Errors(), + databaseConfig, + listenerConfig, + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + headTracker, + txmv2) } else { txm = opts.GenTxManager(chainID) } diff --git a/core/services/headreporter/prometheus_reporter_test.go b/core/services/headreporter/prometheus_reporter_test.go index 9fd42baa15e..9d2a2bb18a6 100644 --- a/core/services/headreporter/prometheus_reporter_test.go +++ b/core/services/headreporter/prometheus_reporter_test.go @@ -135,7 +135,8 @@ func newLegacyChainContainer(t *testing.T, db *sqlx.DB) legacyevm.LegacyChainCon lp, keyStore, estimator, - ht) + ht, + nil) require.NoError(t, err) cfg := configtest.NewGeneralConfig(t, nil) diff --git a/core/services/vrf/delegate_test.go b/core/services/vrf/delegate_test.go index 7ef3febd021..03abaae29b6 100644 --- a/core/services/vrf/delegate_test.go +++ b/core/services/vrf/delegate_test.go @@ -82,7 +82,7 @@ func buildVrfUni(t *testing.T, db *sqlx.DB, cfg chainlink.GeneralConfig) vrfUniv btORM := bridges.NewORM(db) ks := keystore.NewInMemory(db, utils.FastScryptParams, lggr) _, dbConfig, evmConfig := txmgr.MakeTestConfigs(t) - txm, err := txmgr.NewTxm(db, evmConfig, evmConfig.GasEstimator(), evmConfig.Transactions(), nil, dbConfig, dbConfig.Listener(), ec, logger.TestLogger(t), nil, ks.Eth(), nil, nil) + txm, err := txmgr.NewTxm(db, evmConfig, evmConfig.GasEstimator(), evmConfig.Transactions(), nil, dbConfig, dbConfig.Listener(), ec, logger.TestLogger(t), nil, ks.Eth(), nil, nil, nil) orm := headtracker.NewORM(*testutils.FixtureChainID, db) require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), cltest.Head(51))) jrm := job.NewORM(db, prm, btORM, ks, lggr) diff --git a/core/services/vrf/v2/integration_v2_test.go b/core/services/vrf/v2/integration_v2_test.go index 4869cca0926..973affa8f08 100644 --- a/core/services/vrf/v2/integration_v2_test.go +++ b/core/services/vrf/v2/integration_v2_test.go @@ -141,7 +141,7 @@ func makeTestTxm(t *testing.T, txStore txmgr.TestEvmTxStore, keyStore keystore.M _, _, evmConfig := txmgr.MakeTestConfigs(t) txmConfig := txmgr.NewEvmTxmConfig(evmConfig) txm := txmgr.NewEvmTxm(ec.ConfiguredChainID(), txmConfig, evmConfig.Transactions(), keyStore.Eth(), logger.TestLogger(t), nil, nil, - nil, txStore, nil, nil, nil, nil, nil) + nil, txStore, nil, nil, nil, nil, nil, nil) return txm } diff --git a/core/services/vrf/v2/listener_v2_test.go b/core/services/vrf/v2/listener_v2_test.go index b7a8710c4f8..e2a2a703de8 100644 --- a/core/services/vrf/v2/listener_v2_test.go +++ b/core/services/vrf/v2/listener_v2_test.go @@ -40,7 +40,7 @@ func makeTestTxm(t *testing.T, txStore txmgr.TestEvmTxStore, keyStore keystore.M ec := evmtest.NewEthClientMockWithDefaultChain(t) txmConfig := txmgr.NewEvmTxmConfig(evmConfig) txm := txmgr.NewEvmTxm(ec.ConfiguredChainID(), txmConfig, evmConfig.Transactions(), keyStore.Eth(), logger.TestLogger(t), nil, nil, - nil, txStore, nil, nil, nil, nil, nil) + nil, txStore, nil, nil, nil, nil, nil, nil) return txm } From 6121fdfadcf0aaf509919fb31786407e63c84e27 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 13:11:38 +0200 Subject: [PATCH 24/73] AttemptBuilder fixes --- core/chains/evm/txm/attempt_builder.go | 20 +++++----- core/chains/evm/txm/orchestrator.go | 38 +++++++++++++++---- core/chains/evm/txm/storage/inmemory_store.go | 2 +- core/chains/evm/txm/txm.go | 2 +- core/chains/evm/txmgr/builder.go | 2 +- 5 files changed, 43 insertions(+), 21 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index 0350f6d09b1..78a26929e8c 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -20,23 +20,23 @@ type AttemptBuilderKeystore interface { } type attemptBuilder struct { - chainID *big.Int - priceMax *assets.Wei - estimator gas.EvmFeeEstimator - keystore AttemptBuilderKeystore + chainID *big.Int + priceMax *assets.Wei + gas.EvmFeeEstimator + keystore AttemptBuilderKeystore } func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore AttemptBuilderKeystore) *attemptBuilder { return &attemptBuilder{ - chainID: chainID, - priceMax: priceMax, - estimator: estimator, - keystore: keystore, + chainID: chainID, + priceMax: priceMax, + EvmFeeEstimator: estimator, + keystore: keystore, } } func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { - fee, estimatedGasLimit, err := a.estimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) + fee, estimatedGasLimit, err := a.EvmFeeEstimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx } func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { - bumpedFee, bumpedFeeLimit, err := a.estimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) + bumpedFee, bumpedFeeLimit, err := a.EvmFeeEstimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) if err != nil { return nil, err } diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index a6836e3c20e..8c4f41e2232 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -36,6 +36,10 @@ type OrchestratorKeystore interface { EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } +type OrchestratorAttemptBuilder interface { + services.Service +} + // Generics are necessary to keep TXMv2 backwards compatible type Orchestrator[ BLOCK_HASH types.Hashable, @@ -48,6 +52,7 @@ type Orchestrator[ txStore OrchestratorTxStore fwdMgr *forwarders.FwdMgr keystore OrchestratorKeystore + attemptBuilder OrchestratorAttemptBuilder resumeCallback txmgr.ResumeCallback } @@ -58,20 +63,25 @@ func NewTxmOrchestrator[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]]( txStore OrchestratorTxStore, fwdMgr *forwarders.FwdMgr, keystore OrchestratorKeystore, + attemptBuilder OrchestratorAttemptBuilder, ) *Orchestrator[BLOCK_HASH, HEAD] { return &Orchestrator[BLOCK_HASH, HEAD]{ - lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), - chainID: chainID, - txm: txm, - txStore: txStore, - keystore: keystore, - fwdMgr: fwdMgr, + lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), + chainID: chainID, + txm: txm, + txStore: txStore, + keystore: keystore, + attemptBuilder: attemptBuilder, + fwdMgr: fwdMgr, } } func (o *Orchestrator[BLOCK_HASH, HEAD]) Start(ctx context.Context) error { return o.StartOnce("Orchestrator", func() error { var ms services.MultiStart + if err := ms.Start(ctx, o.attemptBuilder); err != nil { + return fmt.Errorf("Orchestrator: AttemptBuilder failed to start: %w", err) + } addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) if err != nil { return err @@ -234,8 +244,20 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, } func (o *Orchestrator[BLOCK_HASH, HEAD]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (uint32, error) { - _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, common.Address{}) - return uint32(count), err + addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) + if err != nil { + return 0, err + } + total := 0 + for _, address := range addresses { + _, count, err := o.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, 0, address) + if err != nil { + return 0, err + } + total += count + } + + return uint32(total), err } func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 4c62bdeb6f2..2e92b330a54 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -239,7 +239,7 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type defer m.Unlock() if len(m.UnstartedTransactions) == 0 { - m.lggr.Debug("Unstarted transactions queue is empty for address: %v", m.address) + m.lggr.Debugf("Unstarted transactions queue is empty for address: %v", m.address) return nil, nil } diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index b90368c20c2..bdb8bee2234 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -362,7 +362,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if isStuck { tx.IsPurgeable = true t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) - t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: ", tx.ID) + t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: %d", tx.ID) return false, t.createAndSendAttempt(ctx, tx, address) } } diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 568929e3551..1323f11b7c5 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -121,7 +121,7 @@ func NewTxmv2( EmptyTxLimitDefault: fCfg.LimitDefault(), } t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, keyStore) - return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore), nil + return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil } // NewEvmResender creates a new concrete EvmResender From 9d7c5a0ebe952d802c4dcd065b8b30bfd490bed1 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 13:24:23 +0200 Subject: [PATCH 25/73] Add AttemptBuilder service exception --- core/chains/evm/txm/orchestrator.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 8c4f41e2232..9f4a2e0031c 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math/big" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" @@ -80,7 +81,10 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Start(ctx context.Context) error { return o.StartOnce("Orchestrator", func() error { var ms services.MultiStart if err := ms.Start(ctx, o.attemptBuilder); err != nil { - return fmt.Errorf("Orchestrator: AttemptBuilder failed to start: %w", err) + // TODO: hacky fix for DualBroadcast + if !strings.Contains(err.Error(), "already been started once") { + return fmt.Errorf("Orchestrator: AttemptBuilder failed to start: %w", err) + } } addresses, err := o.keystore.EnabledAddressesForChain(ctx, o.chainID) if err != nil { From 7831151b1f1994d18f7e7c0cfc81af4a6a61d84f Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 13:26:04 +0200 Subject: [PATCH 26/73] Add AttemptBuilder close --- core/chains/evm/txm/orchestrator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 8c4f41e2232..255c32cc1f5 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -111,6 +111,9 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Close() (merr error) { merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop ForwarderManager: %w", err)) } } + if err := o.attemptBuilder.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop AttemptBuilder: %w", err)) + } if err := o.txm.Close(); err != nil { merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop Txm: %w", err)) } From fc33337d402755eaa9f675bd9a6c93494ef07564 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 17:24:23 +0200 Subject: [PATCH 27/73] Minor updates --- core/chains/evm/txm/txm.go | 11 ++++--- core/chains/evm/txm/txm_test.go | 55 +++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index bdb8bee2234..49a03544868 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -20,7 +20,8 @@ import ( const ( broadcastInterval time.Duration = 30 * time.Second - maxInFlightTransactions uint64 = 16 + maxInFlightTransactions int = 16 + maxInFlightSubset int = 3 maxAllowedAttempts uint16 = 10 ) @@ -257,8 +258,8 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) // by checking the pending nonce so no more than maxInFlightTransactions/3 can get stuck simultaneously i.e. due // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. - if unconfirmedCount >= int(maxInFlightTransactions)/3 { - if unconfirmedCount > int(maxInFlightTransactions) { + if unconfirmedCount >= maxInFlightTransactions/maxInFlightSubset { + if unconfirmedCount > maxInFlightTransactions { t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) return true, nil } @@ -343,11 +344,11 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) tx, unconfirmedCount, err := t.txStore.FetchUnconfirmedTransactionAtNonceWithCount(ctx, latestNonce, address) if err != nil { - return false, err // TODO: add backoff to optimize requests + return false, err } if unconfirmedCount == 0 { t.lggr.Debugf("All transactions confirmed for address: %v", address) - return true, err + return false, err // TODO: add backoff to optimize requests } if tx == nil || tx.Nonce != latestNonce { diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index b3493eecf62..373ce01aed1 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -14,9 +14,12 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) func TestLifecycle(t *testing.T) { @@ -108,9 +111,11 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("throws a warning and returns if unconfirmed transactions exceed maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) - mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions+1), nil).Once() + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions+1, nil).Once() txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) - txm.broadcastTransaction(ctx, address) + bo, err := txm.broadcastTransaction(ctx, address) + assert.True(t, bo) + assert.NoError(t, err) tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) @@ -119,10 +124,12 @@ func TestBroadcastTransaction(t *testing.T) { mTxStore := mocks.NewTxStore(t) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) txm.setNonce(address, 1) - mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, int(maxInFlightTransactions/3), nil).Twice() + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions/3, nil).Twice() client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), nil).Once() // LocalNonce: 1, PendingNonce: 0 - txm.broadcastTransaction(ctx, address) + bo, err := txm.broadcastTransaction(ctx, address) + assert.True(t, bo) + assert.NoError(t, err) client.On("PendingNonceAt", mock.Anything, address).Return(uint64(1), nil).Once() // LocalNonce: 1, PendingNonce: 1 mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() @@ -152,6 +159,42 @@ func TestBroadcastTransaction(t *testing.T) { assert.False(t, bo) assert.Equal(t, uint64(0), txm.getNonce(address)) }) + + t.Run("picks a new tx and creates a new attempt then sends it and updates the broadcast time", func(t *testing.T) { + lggr := logger.Test(t) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + assert.NoError(t, txStore.Add(address)) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm.setNonce(address, 8) + IDK := "IDK" + txRequest := &types.TxRequest{ + IdempotencyKey: &IDK, + ChainID: testutils.FixtureChainID, + FromAddress: address, + ToAddress: testutils.NewAddress(), + SpecifiedGasLimit: 22000, + } + tx, err := txm.CreateTransaction(tests.Context(t), txRequest) + assert.NoError(t, err) + attempt := &types.Attempt{ + TxID: tx.ID, + Fee: gas.EvmFee{GasPrice: assets.NewWeiI(1)}, + GasLimit: 22000, + } + ab.On("NewAttempt", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(attempt, nil).Once() + client.On("SendTransaction", mock.Anything, mock.Anything).Return(nil).Once() + + bo, err := txm.broadcastTransaction(ctx, address) + assert.NoError(t, err) + assert.False(t, bo) + assert.Equal(t, uint64(9), txm.getNonce(address)) + tx, err = txStore.FindTxWithIdempotencyKey(tests.Context(t), &IDK) + assert.NoError(t, err) + assert.Equal(t, 1, len(tx.Attempts)) + var zeroTime time.Time + assert.Greater(t, tx.LastBroadcastAt, zeroTime) + assert.Greater(t, tx.Attempts[0].BroadcastAt, zeroTime) + }) } func TestBackfillTransactions(t *testing.T) { @@ -176,8 +219,8 @@ func TestBackfillTransactions(t *testing.T) { t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) - client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil) - storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")) + client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() + storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() bo, err := txm.backfillTransactions(ctx, address) assert.Error(t, err) assert.False(t, bo) From 87497a065ccbdd35db7a5855fb9d79ca571e33b2 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 17:35:33 +0200 Subject: [PATCH 28/73] Switch DualBroadcast params to pointers --- .../evm/txm/clientwrappers/dual_broadcast_client.go | 8 ++++++-- core/chains/evm/txm/types/transaction.go | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index 8ae2c68172a..20073127350 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -60,13 +60,17 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra if err != nil { return err } - if meta!= nil && meta.DualBroadcast && !tx.IsPurgeable { + if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { data, err := attempt.SignedTransaction.MarshalBinary() if err != nil { return err } + params := "" + if meta.DualBroadcastParams != nil { + params = *meta.DualBroadcastParams + } body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, hexutil.Encode(data))) - if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, meta.DualBroadcastParams); err != nil { + if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, params); err != nil { return err } return nil diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index ca146f8936e..078ea4e8947 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -161,8 +161,8 @@ type TxMeta struct { SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` // Dual Broadcast - DualBroadcast bool `json:"DualBroadcast,omitempty"` - DualBroadcastParams string `json:"DualBroadcastParams,omitempty"` + DualBroadcast *bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams *string `json:"DualBroadcastParams,omitempty"` } type QueueingTxStrategy struct { From 015f1bfe3c95d37da506565fc37c7736d7152568 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 13 Nov 2024 17:39:09 +0200 Subject: [PATCH 29/73] Update DualBroadcast types to pointers --- common/txmgr/txmgr.go | 4 ++-- common/txmgr/types/tx.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index 1a088ac7cd8..608d838d851 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -540,14 +540,14 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Trigger(ad func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { // Check for existing Tx with IdempotencyKey. If found, return the Tx and do nothing // Skipping CreateTransaction to avoid double send - if b.txmv2wrapper != nil && txRequest.Meta != nil && txRequest.Meta.DualBroadcast { + if b.txmv2wrapper != nil && txRequest.Meta != nil && txRequest.Meta.DualBroadcast != nil && *txRequest.Meta.DualBroadcast { return b.txmv2wrapper.CreateTransaction(ctx, txRequest) } if txRequest.IdempotencyKey != nil { var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) if err != nil { - return tx, fmt.Errorf("Failed to search for transaction with IdempotencyKey: %w", err) + return tx, fmt.Errorf("failed to search for transaction with IdempotencyKey: %w", err) } if existingTx != nil { b.logger.Infow("Found a Tx with IdempotencyKey. Returning existing Tx without creating a new one.", "IdempotencyKey", *txRequest.IdempotencyKey) diff --git a/common/txmgr/types/tx.go b/common/txmgr/types/tx.go index ea3fc55330c..f04047a36c1 100644 --- a/common/txmgr/types/tx.go +++ b/common/txmgr/types/tx.go @@ -161,8 +161,8 @@ type TxMeta[ADDR types.Hashable, TX_HASH types.Hashable] struct { SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` // Dual Broadcast - DualBroadcast bool `json:"DualBroadcast,omitempty"` - DualBroadcastParams string `json:"DualBroadcastParams,omitempty"` + DualBroadcast *bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams *string `json:"DualBroadcastParams,omitempty"` } type TxAttempt[ From fc11ca1fc27ba2b263766c62c2d78a56307c2532 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 15 Nov 2024 13:03:46 +0200 Subject: [PATCH 30/73] Make purgable attempts empty --- core/chains/evm/txm/attempt_builder.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index 78a26929e8c..ceae96f1b28 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -85,13 +85,17 @@ func (a *attemptBuilder) newCustomAttempt( func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transaction, gasPrice *assets.Wei, estimatedGasLimit uint64) (*types.Attempt, error) { var data []byte + var toAddress common.Address + value := big.NewInt(0) if !tx.IsPurgeable { data = tx.Data + toAddress = tx.ToAddress + value = tx.Value } legacyTx := evmtypes.LegacyTx{ Nonce: tx.Nonce, - To: &tx.ToAddress, - Value: tx.Value, + To: &toAddress, + Value: value, Gas: estimatedGasLimit, GasPrice: gasPrice.ToInt(), Data: data, @@ -115,13 +119,17 @@ func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transac func (a *attemptBuilder) newDynamicFeeAttempt(ctx context.Context, tx *types.Transaction, dynamicFee gas.DynamicFee, estimatedGasLimit uint64) (*types.Attempt, error) { var data []byte + var toAddress common.Address + value := big.NewInt(0) if !tx.IsPurgeable { data = tx.Data + toAddress = tx.ToAddress + value = tx.Value } dynamicTx := evmtypes.DynamicFeeTx{ Nonce: tx.Nonce, - To: &tx.ToAddress, - Value: tx.Value, + To: &toAddress, + Value: value, Gas: estimatedGasLimit, GasFeeCap: dynamicFee.GasFeeCap.ToInt(), GasTipCap: dynamicFee.GasTipCap.ToInt(), From d2dc53b0ee52233e963010d93b24c75e7c32c331 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 15 Nov 2024 17:15:25 +0200 Subject: [PATCH 31/73] Fix Idempotency in Store Manager --- core/chains/evm/txm/orchestrator.go | 16 +++++++++++++--- .../evm/txm/storage/inmemory_store_manager.go | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 255c32cc1f5..2af3110dc21 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -36,8 +36,12 @@ type OrchestratorKeystore interface { EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } -type OrchestratorAttemptBuilder interface { +type OrchestratorAttemptBuilder[ + BLOCK_HASH types.Hashable, + HEAD types.Head[BLOCK_HASH], +] interface { services.Service + OnNewLongestChain(ctx context.Context, head HEAD) } // Generics are necessary to keep TXMv2 backwards compatible @@ -52,7 +56,7 @@ type Orchestrator[ txStore OrchestratorTxStore fwdMgr *forwarders.FwdMgr keystore OrchestratorKeystore - attemptBuilder OrchestratorAttemptBuilder + attemptBuilder OrchestratorAttemptBuilder[BLOCK_HASH, HEAD] resumeCallback txmgr.ResumeCallback } @@ -63,7 +67,7 @@ func NewTxmOrchestrator[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]]( txStore OrchestratorTxStore, fwdMgr *forwarders.FwdMgr, keystore OrchestratorKeystore, - attemptBuilder OrchestratorAttemptBuilder, + attemptBuilder OrchestratorAttemptBuilder[BLOCK_HASH, HEAD], ) *Orchestrator[BLOCK_HASH, HEAD] { return &Orchestrator[BLOCK_HASH, HEAD]{ lggr: logger.Sugared(logger.Named(lggr, "Orchestrator")), @@ -152,6 +156,12 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Reset(addr common.Address, abandon bool } func (o *Orchestrator[BLOCK_HASH, HEAD]) OnNewLongestChain(ctx context.Context, head HEAD) { + ok := o.IfStarted(func() { + o.attemptBuilder.OnNewLongestChain(ctx, head) + }) + if !ok { + o.lggr.Debugw("Not started; ignoring head", "head", head, "state", o.State()) + } } func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index e680ee79391..7d91a2ebc8f 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -132,5 +132,5 @@ func (m *InMemoryStoreManager) FindTxWithIdempotencyKey(_ context.Context, idemp return tx, nil } } - return nil, fmt.Errorf("key not found") + return nil, nil } From 9abccc7cf73d54c2c652b632cbd485ed51332e42 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 11:42:30 +0200 Subject: [PATCH 32/73] Update trigger --- core/chains/evm/txm/orchestrator.go | 6 ++---- core/chains/evm/txm/txm.go | 7 +++---- core/chains/evm/txm/txm_test.go | 6 +++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 2af3110dc21..d709ce712a4 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -126,9 +126,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Close() (merr error) { } func (o *Orchestrator[BLOCK_HASH, HEAD]) Trigger(addr common.Address) { - if err := o.txm.Trigger(addr); err != nil { - o.lggr.Error(err) - } + o.txm.Trigger(addr) } func (o *Orchestrator[BLOCK_HASH, HEAD]) Name() string { @@ -347,6 +345,6 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) SendNativeToken(ctx context.Context, ch } // Trigger the Txm to check for new transaction - err = o.txm.Trigger(from) + o.txm.Trigger(from) return tx, err } diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 49a03544868..0674ab1c45c 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -153,13 +153,12 @@ func (t *Txm) CreateTransaction(ctx context.Context, txRequest *types.TxRequest) return } -func (t *Txm) Trigger(address common.Address) error { +func (t *Txm) Trigger(address common.Address) { if !t.IfStarted(func() { t.triggerCh[address] <- struct{}{} }) { - return fmt.Errorf("Txm unstarted") + t.lggr.Error("Txm unstarted") } - return nil } func (t *Txm) Abandon(address common.Address) error { @@ -348,7 +347,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) } if unconfirmedCount == 0 { t.lggr.Debugf("All transactions confirmed for address: %v", address) - return false, err // TODO: add backoff to optimize requests + return false, err // TODO: add backoff to optimize requests } if tx == nil || tx.Nonce != latestNonce { diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 373ce01aed1..ff6f877cc55 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -67,9 +67,10 @@ func TestTrigger(t *testing.T) { keystore := mocks.NewKeystore(t) keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { - txm := NewTxm(logger.Test(t), nil, nil, nil, nil, Config{}, keystore) + lggr, observedLogs := logger.TestObserved(t, zap.ErrorLevel) + txm := NewTxm(lggr, nil, nil, nil, nil, Config{}, keystore) txm.Trigger(address) - assert.Error(t, txm.Trigger(address), "Txm unstarted") + tests.AssertLogEventually(t, observedLogs, "Txm unstarted") }) t.Run("executes Trigger", func(t *testing.T) { @@ -84,7 +85,6 @@ func TestTrigger(t *testing.T) { // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() servicetest.Run(t, txm) - assert.NoError(t, txm.Trigger(address)) }) } From c2c1c4f3c71eac45de905d89416a85f51e2b8083 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 12:58:46 +0200 Subject: [PATCH 33/73] Fix lint --- core/chains/evm/txm/orchestrator.go | 25 ++++++++++------ core/chains/evm/txm/storage/inmemory_store.go | 7 +++-- .../evm/txm/storage/inmemory_store_manager.go | 5 ++-- .../storage/inmemory_store_manager_test.go | 7 +++-- .../evm/txm/storage/inmemory_store_test.go | 30 ++++++++----------- core/chains/evm/txm/txm.go | 5 +++- core/chains/evm/txm/txm_test.go | 8 ++--- core/chains/evm/txm/types/transaction.go | 4 +-- 8 files changed, 49 insertions(+), 42 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index d709ce712a4..37d17c7b344 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" "github.com/ethereum/go-ethereum/common" @@ -148,7 +149,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Reset(addr common.Address, abandon bool } }) if !ok { - return fmt.Errorf("Orchestrator not started yet") + return errors.New("Orchestrator not started yet") } return nil } @@ -198,9 +199,9 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, var meta *sqlutil.JSON if request.Meta != nil { - raw, err := json.Marshal(request.Meta) - if err != nil { - return tx, err + raw, mErr := json.Marshal(request.Meta) + if mErr != nil { + return tx, mErr } m := sqlutil.JSON(raw) meta = &m @@ -229,11 +230,14 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, o.txm.Trigger(request.FromAddress) } - sequence := evmtypes.Nonce(wrappedTx.Nonce) + if wrappedTx.ID > math.MaxInt64 { + return tx, fmt.Errorf("overflow for int64: %d", wrappedTx.ID) + } + tx = txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]{ + //nolint:gosec // disable G115 ID: int64(wrappedTx.ID), IdempotencyKey: wrappedTx.IdempotencyKey, - Sequence: &sequence, FromAddress: wrappedTx.FromAddress, ToAddress: wrappedTx.ToAddress, EncodedPayload: wrappedTx.Data, @@ -241,9 +245,9 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, FeeLimit: wrappedTx.SpecifiedGasLimit, CreatedAt: wrappedTx.CreatedAt, Meta: wrappedTx.Meta, - //Subject: wrappedTx.Subject, + // Subject: wrappedTx.Subject, - //TransmitChecker: wrappedTx.TransmitChecker, + // TransmitChecker: wrappedTx.TransmitChecker, ChainID: wrappedTx.ChainID, PipelineTaskRunID: wrappedTx.PipelineTaskRunID, @@ -268,7 +272,8 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CountTransactionsByState(ctx context.Co total += count } - return uint32(total), err + //nolint:gosec // disable G115 + return uint32(total), nil } func (o *Orchestrator[BLOCK_HASH, HEAD]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (time nullv4.Time, err error) { @@ -286,9 +291,11 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesByMetaFieldAndStates(ctx contex func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } + func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } + func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 2e92b330a54..8595a5c84c3 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "math/big" "sort" @@ -259,11 +260,11 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type // Shouldn't call lock because it's being called by a method that already has the lock func (m *InMemoryStore) pruneConfirmedTransactions() []uint64 { - var noncesToPrune []uint64 + noncesToPrune := make([]uint64, 0, len(m.ConfirmedTransactions)) for nonce := range m.ConfirmedTransactions { noncesToPrune = append(noncesToPrune, nonce) } - if len(noncesToPrune) <= 0 { + if len(noncesToPrune) == 0 { return nil } sort.Slice(noncesToPrune, func(i, j int) bool { return noncesToPrune[i] < noncesToPrune[j] }) @@ -303,7 +304,7 @@ func (m *InMemoryStore) DeleteAttemptForUnconfirmedTx(transactionNonce uint64, a } func (m *InMemoryStore) MarkTxFatal(*types.Transaction) error { - return fmt.Errorf("not implemented") + return errors.New("not implemented") } // Orchestrator diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index 7d91a2ebc8f..dfb777ab22b 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -47,8 +47,7 @@ func (m *InMemoryStoreManager) Add(addresses ...common.Address) error { func (m *InMemoryStoreManager) AppendAttemptToTransaction(_ context.Context, txNonce uint64, fromAddress common.Address, attempt *types.Attempt) error { if store, exists := m.InMemoryStoreMap[fromAddress]; exists { - store.AppendAttemptToTransaction(txNonce, attempt) - return nil + return store.AppendAttemptToTransaction(txNonce, attempt) } return fmt.Errorf(StoreNotFoundForAddress, fromAddress) } @@ -92,7 +91,7 @@ func (m *InMemoryStoreManager) MarkTransactionsConfirmed(_ context.Context, nonc func (m *InMemoryStoreManager) MarkUnconfirmedTransactionPurgeable(_ context.Context, nonce uint64, fromAddress common.Address) error { if store, exists := m.InMemoryStoreMap[fromAddress]; exists { - store.MarkUnconfirmedTransactionPurgeable(nonce) + return store.MarkUnconfirmedTransactionPurgeable(nonce) } return fmt.Errorf(StoreNotFoundForAddress, fromAddress) } diff --git a/core/chains/evm/txm/storage/inmemory_store_manager_test.go b/core/chains/evm/txm/storage/inmemory_store_manager_test.go index 27210898217..e10870a9942 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager_test.go @@ -4,9 +4,10 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/stretchr/testify/assert" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" ) @@ -18,7 +19,7 @@ func TestAdd(t *testing.T) { // Adds a new address err := m.Add(fromAddress) assert.NoError(t, err) - assert.Equal(t, 1, len(m.InMemoryStoreMap)) + assert.Len(t, m.InMemoryStoreMap, 1) // Fails if address exists err = m.Add(fromAddress) @@ -30,5 +31,5 @@ func TestAdd(t *testing.T) { addresses := []common.Address{fromAddress1, fromAddress2} err = m.Add(addresses...) assert.NoError(t, err) - assert.Equal(t, 3, len(m.InMemoryStoreMap)) + assert.Len(t, m.InMemoryStoreMap, 3) } diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 46cc94290a8..5bb011203ac 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -6,9 +6,10 @@ import ( "testing" "time" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/stretchr/testify/assert" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) @@ -54,9 +55,7 @@ func TestAbandonPendingTransactions(t *testing.T) { assert.Equal(t, types.TxFatalError, tx2.State) assert.Equal(t, types.TxConfirmed, tx3.State) assert.Equal(t, types.TxConfirmed, tx4.State) - }) - } func TestAppendAttemptToTransaction(t *testing.T) { @@ -79,7 +78,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { }) t.Run("fails if unconfirmed transaction was found but has doesn't match the txID", func(t *testing.T) { - var nonce uint64 = 0 + var nonce uint64 newAttempt := &types.Attempt{ TxID: 2, } @@ -87,12 +86,11 @@ func TestAppendAttemptToTransaction(t *testing.T) { }) t.Run("appends attempt to transaction", func(t *testing.T) { - var nonce uint64 = 0 + var nonce uint64 newAttempt := &types.Attempt{ TxID: 1, } assert.NoError(t, m.AppendAttemptToTransaction(nonce, newAttempt)) - }) } @@ -106,7 +104,6 @@ func TestCountUnstartedTransactions(t *testing.T) { insertUnstartedTransaction(m) assert.Equal(t, 1, m.CountUnstartedTransactions()) - } func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { @@ -114,7 +111,8 @@ func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - insertUnconfirmedTransaction(m, 0) + _, err := insertUnconfirmedTransaction(m, 0) + assert.NoError(t, err) t.Run("fails if unconfirmed transaction with the same nonce exists", func(t *testing.T) { _, err := m.CreateEmptyUnconfirmedTransaction(0, 0) @@ -126,7 +124,6 @@ func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { assert.NoError(t, err) assert.Equal(t, types.TxUnconfirmed, tx.State) }) - } func TestCreateTransaction(t *testing.T) { @@ -165,7 +162,6 @@ func TestCreateTransaction(t *testing.T) { assert.NoError(t, err) assert.Equal(t, uint64(overshot), tx.ID) }) - } func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { @@ -178,12 +174,12 @@ func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { assert.Nil(t, tx) assert.Equal(t, 0, count) - var nonce uint64 = 0 - insertUnconfirmedTransaction(m, nonce) + var nonce uint64 + _, err := insertUnconfirmedTransaction(m, nonce) + assert.NoError(t, err) tx, count = m.FetchUnconfirmedTransactionAtNonceWithCount(0) assert.Equal(t, tx.Nonce, nonce) assert.Equal(t, 1, count) - } func TestMarkTransactionsConfirmed(t *testing.T) { @@ -194,8 +190,8 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("returns 0 if there are no transactions", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) un, cn := m.MarkTransactionsConfirmed(100) - assert.Equal(t, len(un), 0) - assert.Equal(t, len(cn), 0) + assert.Empty(t, un) + assert.Empty(t, cn) }) t.Run("confirms transaction with nonce lower than the latest", func(t *testing.T) { @@ -211,7 +207,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) assert.Equal(t, ctxs[0], ctx1.ID) - assert.Equal(t, 0, len(utxs)) + assert.Empty(t, utxs) }) t.Run("unconfirms transaction with nonce equal to or higher than the latest", func(t *testing.T) { @@ -255,7 +251,7 @@ func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { assert.NoError(t, err) err = m.MarkUnconfirmedTransactionPurgeable(0) assert.NoError(t, err) - assert.Equal(t, true, tx.IsPurgeable) + assert.True(t, tx.IsPurgeable) } func TestUpdateTransactionBroadcast(t *testing.T) { diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 0674ab1c45c..926f75443ff 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -361,7 +361,10 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) } if isStuck { tx.IsPurgeable = true - t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) + err = t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) + if err != nil { + return false, err + } t.lggr.Infof("Marked tx as purgeable. Sending purge attempt for txID: %d", tx.ID) return false, t.createAndSendAttempt(ctx, tx, address) } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index ff6f877cc55..ae7d1d82975 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -57,7 +57,6 @@ func TestLifecycle(t *testing.T) { servicetest.Run(t, txm) tests.AssertLogEventually(t, observedLogs, "Backfill time elapsed") }) - } func TestTrigger(t *testing.T) { @@ -133,9 +132,10 @@ func TestBroadcastTransaction(t *testing.T) { client.On("PendingNonceAt", mock.Anything, address).Return(uint64(1), nil).Once() // LocalNonce: 1, PendingNonce: 1 mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() - txm.broadcastTransaction(ctx, address) + bo, err = txm.broadcastTransaction(ctx, address) + assert.False(t, bo) + assert.NoError(t, err) tests.AssertLogCountEventually(t, observedLogs, "Reached transaction limit.", 1) - }) t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { @@ -190,7 +190,7 @@ func TestBroadcastTransaction(t *testing.T) { assert.Equal(t, uint64(9), txm.getNonce(address)) tx, err = txStore.FindTxWithIdempotencyKey(tests.Context(t), &IDK) assert.NoError(t, err) - assert.Equal(t, 1, len(tx.Attempts)) + assert.Len(t, tx.Attempts, 1) var zeroTime time.Time assert.Greater(t, tx.LastBroadcastAt, zeroTime) assert.Greater(t, tx.Attempts[0].BroadcastAt, zeroTime) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 0cb11ad8f02..cf3a6dcc295 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -69,7 +69,7 @@ func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, erro func (t *Transaction) DeepCopy() *Transaction { copy := *t - var attemptsCopy []*Attempt + attemptsCopy := make([]*Attempt, 0, len(t.Attempts)) for _, attempt := range t.Attempts { attemptsCopy = append(attemptsCopy, attempt.DeepCopy()) } @@ -121,7 +121,7 @@ type TxRequest struct { Meta *sqlutil.JSON // TODO: *TxMeta after migration ForwarderAddress common.Address - //QueueingTxStrategy QueueingTxStrategy + // QueueingTxStrategy QueueingTxStrategy // Pipeline variables - if you aren't calling this from chain tx task within // the pipeline, you don't need these variables From a04b458dc6f455cc18be760ad48e2c2619ec202e Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 14:14:34 +0200 Subject: [PATCH 34/73] Add context to client --- .../evm/txm/clientwrappers/chain_client.go | 1 + core/chains/evm/txm/stuck_tx_detector.go | 22 ++++++++++++++----- core/chains/evm/txm/txm.go | 6 ++--- core/chains/evm/txmgr/builder.go | 4 ++-- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/core/chains/evm/txm/clientwrappers/chain_client.go b/core/chains/evm/txm/clientwrappers/chain_client.go index 27ebd34f882..7638cc53443 100644 --- a/core/chains/evm/txm/clientwrappers/chain_client.go +++ b/core/chains/evm/txm/clientwrappers/chain_client.go @@ -5,6 +5,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go index 6718ce8de95..aeeed7085b1 100644 --- a/core/chains/evm/txm/stuck_tx_detector.go +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -1,6 +1,7 @@ package txm import ( + "context" "encoding/json" "fmt" "io" @@ -17,7 +18,7 @@ import ( type StuckTxDetectorConfig struct { BlockTime time.Duration - StuckTxBlockThreshold uint16 + StuckTxBlockThreshold uint32 DetectionApiUrl string } @@ -35,11 +36,11 @@ func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, confi } } -func (s *stuckTxDetector) DetectStuckTransaction(tx *types.Transaction) (bool, error) { +func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) { switch s.chainType { // TODO: rename case chaintype.ChainDualBroadcast: - result, err := s.dualBroadcastDetection(tx) + result, err := s.dualBroadcastDetection(ctx, tx) if result || err != nil { return result, err } @@ -72,17 +73,26 @@ const ( ApiStatusUnknown = "UNKNOWN" ) -func (s *stuckTxDetector) dualBroadcastDetection(tx *types.Transaction) (bool, error) { +func (s *stuckTxDetector) dualBroadcastDetection(ctx context.Context, tx *types.Transaction) (bool, error) { for _, attempt := range tx.Attempts { - resp, err := http.Get(s.config.DetectionApiUrl + attempt.Hash.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.config.DetectionApiUrl+attempt.Hash.String(), nil) + if err != nil { + return false, fmt.Errorf("failed to make request for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + resp, err := http.DefaultClient.Do(req) if err != nil { return false, fmt.Errorf("failed to get transaction status for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) } - defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return false, fmt.Errorf("request %v failed with status: %d", req, resp.StatusCode) + } body, err := io.ReadAll(resp.Body) + resp.Body.Close() if err != nil { return false, err } + var apiResponse ApiResponse err = json.Unmarshal(body, &apiResponse) if err != nil { diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 8be28dcbf7d..9dfb2b6755b 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -56,7 +56,7 @@ type ErrorHandler interface { } type StuckTxDetector interface { - DetectStuckTransaction(tx *types.Transaction) (bool, error) + DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) } type Keystore interface { @@ -66,7 +66,7 @@ type Keystore interface { type Config struct { EIP1559 bool BlockTime time.Duration - RetryBlockThreshold uint16 + RetryBlockThreshold uint64 EmptyTxLimitDefault uint64 } @@ -355,7 +355,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { if !tx.IsPurgeable && t.stuckTxDetector != nil { - isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) + isStuck, err := t.stuckTxDetector.DetectStuckTransaction(ctx, tx) if err != nil { return false, err } diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 40cae9d1bc9..1dc771b8701 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -118,7 +118,7 @@ func NewTxmV2( if txConfig.AutoPurge().Enabled() { stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ BlockTime: *txmV2Config.BlockTime(), - StuckTxBlockThreshold: uint16(*txConfig.AutoPurge().Threshold()), + StuckTxBlockThreshold: *txConfig.AutoPurge().Threshold(), DetectionApiUrl: txConfig.AutoPurge().DetectionApiUrl().String(), } stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) @@ -129,7 +129,7 @@ func NewTxmV2( config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), BlockTime: *txmV2Config.BlockTime(), - RetryBlockThreshold: uint16(fCfg.BumpThreshold()), + RetryBlockThreshold: fCfg.BumpThreshold(), EmptyTxLimitDefault: fCfg.LimitDefault(), } var c txm.Client From 8044bef6dca083deeb7cd062b902dbee3ff3ce23 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 15:09:50 +0200 Subject: [PATCH 35/73] Fix more lint --- .../evm/txm/storage/inmemory_store_test.go | 24 +++++++++---------- core/chains/evm/txmgr/builder.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 5bb011203ac..d091628c8f7 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -223,7 +223,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) assert.Equal(t, utxs[0], ctx2.ID) - assert.Equal(t, 0, len(ctxs)) + assert.Empty(t, ctxs) }) t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) @@ -231,9 +231,9 @@ func TestMarkTransactionsConfirmed(t *testing.T) { _, err := insertConfirmedTransaction(m, uint64(i)) assert.NoError(t, err) } - assert.Equal(t, maxQueuedTransactions, len(m.ConfirmedTransactions)) + assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions) m.MarkTransactionsConfirmed(maxQueuedTransactions) - assert.Equal(t, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset), len(m.ConfirmedTransactions)) + assert.Len(t, m.ConfirmedTransactions, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset)) }) } @@ -261,13 +261,13 @@ func TestUpdateTransactionBroadcast(t *testing.T) { hash := testutils.NewHash() t.Run("fails if unconfirmed transaction was not found", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - var nonce uint64 = 0 + var nonce uint64 assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) }) t.Run("fails if attempt was not found for a given transaction", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - var nonce uint64 = 0 + var nonce uint64 tx, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) @@ -280,7 +280,7 @@ func TestUpdateTransactionBroadcast(t *testing.T) { t.Run("updates transaction's and attempt's broadcast times", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - var nonce uint64 = 0 + var nonce uint64 tx, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) attempt := &types.Attempt{TxID: tx.ID, Hash: hash} @@ -303,7 +303,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { }) t.Run("fails if there is already another unstarted transaction with the same nonce", func(t *testing.T) { - var nonce uint64 = 0 + var nonce uint64 m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) insertUnstartedTransaction(m) _, err := insertUnconfirmedTransaction(m, nonce) @@ -314,7 +314,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { }) t.Run("updates unstarted transaction to unconfirmed and assigns a nonce", func(t *testing.T) { - var nonce uint64 = 0 + var nonce uint64 m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) insertUnstartedTransaction(m) @@ -350,7 +350,7 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { t.Run("deletes attempt of unconfirmed transaction", func(t *testing.T) { hash := testutils.NewHash() - var nonce uint64 = 0 + var nonce uint64 m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) tx, err := insertUnconfirmedTransaction(m, nonce) assert.NoError(t, err) @@ -360,7 +360,7 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { err = m.DeleteAttemptForUnconfirmedTx(nonce, attempt) assert.NoError(t, err) - assert.Equal(t, 0, len(tx.Attempts)) + assert.Len(t, tx.Attempts, 0) }) } @@ -375,8 +375,8 @@ func TestPruneConfirmedTransactions(t *testing.T) { } prunedTxIDs := m.pruneConfirmedTransactions() left := total - total/pruneSubset - assert.Equal(t, left, len(m.ConfirmedTransactions)) - assert.Equal(t, total/pruneSubset, len(prunedTxIDs)) + assert.Len(t, m.ConfirmedTransactions, left) + assert.Len(t, prunedTxIDs, total/pruneSubset) } func insertUnstartedTransaction(m *InMemoryStore) *types.Transaction { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 7f3a3bf6147..28dbcfe64b1 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -122,7 +122,7 @@ func NewTxmv2( EmptyTxLimitDefault: fCfg.LimitDefault(), } t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, keyStore) - return txm.NewTxmOrchestrator[common.Hash, *evmtypes.Head](lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil + return txm.NewTxmOrchestrator(lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil } // NewEvmResender creates a new concrete EvmResender From 5032336dc16cd07ece198f730f13a76526c77fe7 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 16:00:16 +0200 Subject: [PATCH 36/73] Fix lint --- core/chains/evm/txm/storage/inmemory_store_test.go | 2 +- core/chains/evm/txm/txm_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index d091628c8f7..9e905e88c42 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -360,7 +360,7 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { err = m.DeleteAttemptForUnconfirmedTx(nonce, attempt) assert.NoError(t, err) - assert.Len(t, tx.Attempts, 0) + assert.Empty(t, tx.Attempts) }) } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index ae7d1d82975..3f5e6541629 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -46,7 +46,7 @@ func TestLifecycle(t *testing.T) { txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) assert.NoError(t, txStore.Add(addresses...)) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) - var nonce uint64 = 0 + var nonce uint64 // Start client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() client.On("PendingNonceAt", mock.Anything, address2).Return(nonce, nil).Once() @@ -80,7 +80,7 @@ func TestTrigger(t *testing.T) { ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) - var nonce uint64 = 0 + var nonce uint64 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() servicetest.Run(t, txm) From b16643dfc2a9b96460eff057ce98b0e1d72a0e55 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 16:08:02 +0200 Subject: [PATCH 37/73] Fix lint --- core/chains/evm/config/chain_scoped_txmv2.go | 4 ++-- core/chains/evm/config/config.go | 2 +- core/chains/evm/config/toml/config.go | 6 ++--- .../clientwrappers/dual_broadcast_client.go | 8 +++---- core/chains/evm/txm/stuck_tx_detector.go | 24 +++++++++---------- core/chains/evm/txmgr/builder.go | 4 ++-- core/config/docs/chains-evm.toml | 4 ++-- docs/CONFIG.md | 8 +++---- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/core/chains/evm/config/chain_scoped_txmv2.go b/core/chains/evm/config/chain_scoped_txmv2.go index 5422f636443..e50148cfae4 100644 --- a/core/chains/evm/config/chain_scoped_txmv2.go +++ b/core/chains/evm/config/chain_scoped_txmv2.go @@ -20,6 +20,6 @@ func (t *txmv2Config) BlockTime() *time.Duration { return &d } -func (t *txmv2Config) CustomUrl() *url.URL { - return t.c.CustomUrl.URL() +func (t *txmv2Config) CustomURL() *url.URL { + return t.c.CustomURL.URL() } diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index fe049a173db..be61bfe02ee 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -106,7 +106,7 @@ type ClientErrors interface { type TxmV2 interface { Enabled() bool BlockTime() *time.Duration - CustomUrl() *url.URL + CustomURL() *url.URL } type Transactions interface { diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 5c762508ac0..0f8b1eceee5 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -477,7 +477,7 @@ func (c *Chain) ValidateConfig() (err error) { type TxmV2 struct { Enabled *bool `toml:",omitempty"` BlockTime *commonconfig.Duration `toml:",omitempty"` - CustomUrl *commonconfig.URL `toml:",omitempty"` + CustomURL *commonconfig.URL `toml:",omitempty"` } func (t *TxmV2) setFrom(f *TxmV2) { @@ -489,8 +489,8 @@ func (t *TxmV2) setFrom(f *TxmV2) { t.BlockTime = f.BlockTime } - if v := f.CustomUrl; v != nil { - t.CustomUrl = f.CustomUrl + if v := f.CustomURL; v != nil { + t.CustomURL = f.CustomURL } } diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index 20073127350..aa3a2db10ee 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -26,14 +26,14 @@ type DualBroadcastClientKeystore interface { type DualBroadcastClient struct { c client.Client keystore DualBroadcastClientKeystore - customUrl *url.URL + customURL *url.URL } -func NewDualBroadcastClient(c client.Client, keystore DualBroadcastClientKeystore, customUrl *url.URL) *DualBroadcastClient { +func NewDualBroadcastClient(c client.Client, keystore DualBroadcastClientKeystore, customURL *url.URL) *DualBroadcastClient { return &DualBroadcastClient{ c: c, keystore: keystore, - customUrl: customUrl, + customURL: customURL, } } @@ -81,7 +81,7 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra func (d *DualBroadcastClient) signAndPostMessage(ctx context.Context, address common.Address, body []byte, urlParams string) (result string, err error) { bodyReader := bytes.NewReader(body) - postReq, err := http.NewRequestWithContext(ctx, http.MethodPost, d.customUrl.String()+"?"+urlParams, bodyReader) + postReq, err := http.NewRequestWithContext(ctx, http.MethodPost, d.customURL.String()+"?"+urlParams, bodyReader) if err != nil { return } diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go index aeeed7085b1..68d8caf0ed1 100644 --- a/core/chains/evm/txm/stuck_tx_detector.go +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -19,7 +19,7 @@ import ( type StuckTxDetectorConfig struct { BlockTime time.Duration StuckTxBlockThreshold uint32 - DetectionApiUrl string + DetectionURL string } type stuckTxDetector struct { @@ -60,22 +60,22 @@ func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { return false } -type ApiResponse struct { +type APIResponse struct { Status string `json:"status,omitempty"` Hash common.Hash `json:"hash,omitempty"` } const ( - ApiStatusPending = "PENDING" - ApiStatusIncluded = "INCLUDED" - ApiStatusFailed = "FAILED" - ApiStatusCancelled = "CANCELLED" - ApiStatusUnknown = "UNKNOWN" + APIStatusPending = "PENDING" + APIStatusIncluded = "INCLUDED" + APIStatusFailed = "FAILED" + APIStatusCancelled = "CANCELLED" + APIStatusUnknown = "UNKNOWN" ) func (s *stuckTxDetector) dualBroadcastDetection(ctx context.Context, tx *types.Transaction) (bool, error) { for _, attempt := range tx.Attempts { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.config.DetectionApiUrl+attempt.Hash.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.config.DetectionURL+attempt.Hash.String(), nil) if err != nil { return false, fmt.Errorf("failed to make request for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) } @@ -93,19 +93,19 @@ func (s *stuckTxDetector) dualBroadcastDetection(ctx context.Context, tx *types. return false, err } - var apiResponse ApiResponse + var apiResponse APIResponse err = json.Unmarshal(body, &apiResponse) if err != nil { return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w: %s", tx.ID, attempt.Hash, err, string(body)) } switch apiResponse.Status { - case ApiStatusPending, ApiStatusIncluded: + case APIStatusPending, APIStatusIncluded: return false, nil - case ApiStatusFailed, ApiStatusCancelled: + case APIStatusFailed, APIStatusCancelled: s.lggr.Debugf("TxID: %v with attempHash: %v was marked as failed/cancelled by the RPC. Transaction is now considered stuck and will be purged.", tx.ID, attempt.Hash) return true, nil - case ApiStatusUnknown: + case APIStatusUnknown: continue default: continue diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 5e63c9e6b25..0c224af4326 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -120,7 +120,7 @@ func NewTxmV2( stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ BlockTime: *txmV2Config.BlockTime(), StuckTxBlockThreshold: *txConfig.AutoPurge().Threshold(), - DetectionApiUrl: txConfig.AutoPurge().DetectionApiUrl().String(), + DetectionURL: txConfig.AutoPurge().DetectionApiUrl().String(), } stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) } @@ -135,7 +135,7 @@ func NewTxmV2( } var c txm.Client if chainConfig.ChainType() == chaintype.ChainDualBroadcast { - c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomUrl()) + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomURL()) } else { c = clientwrappers.NewChainClient(client) } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 1e50ff28585..cd99027e611 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -134,8 +134,8 @@ ResendAfterThreshold = '1m' # Default Enabled = false # Default # BlockTime controls the frequency of the backfill loop of TxmV2. BlockTime = '10s' # Example -# CustomUrl configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. -CustomUrl = 'https://example.api.io' # Example +# CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomURL = 'https://example.api.io' # Example [EVM.Transactions.AutoPurge] # Enabled enables or disables automatically purging transactions that have been idenitified as terminally stuck (will never be included on-chain). This feature is only expected to be used by ZK chains. diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 1020a2f3ce6..cf01ba73b32 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -9290,7 +9290,7 @@ ResendAfterThreshold controls how long to wait before re-broadcasting a transact [EVM.TxmV2] Enabled = false # Default BlockTime = '10s' # Example -CustomUrl = 'https://example.api.io' # Example +CustomURL = 'https://example.api.io' # Example ``` @@ -9306,11 +9306,11 @@ BlockTime = '10s' # Example ``` BlockTime controls the frequency of the backfill loop of TxmV2. -### CustomUrl +### CustomURL ```toml -CustomUrl = 'https://example.api.io' # Example +CustomURL = 'https://example.api.io' # Example ``` -CustomUrl configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. ## EVM.Transactions.AutoPurge ```toml From feb243a5724394bbd5459ed12bbfaf089dd4494f Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 17:13:08 +0200 Subject: [PATCH 38/73] Fix lint --- core/chains/evm/txm/orchestrator.go | 1 + core/chains/evm/txm/storage/inmemory_store.go | 6 +++--- core/chains/evm/txm/storage/inmemory_store_test.go | 3 +++ core/chains/evm/txm/txm.go | 2 ++ core/chains/evm/txm/txm_test.go | 7 ++++--- core/chains/evm/txm/types/transaction.go | 13 ++++++------- core/chains/evm/txmgr/builder.go | 1 + 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 37d17c7b344..182ddd3b1cd 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -296,6 +296,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByReceiptBlockNum( return } +//nolint:gosec // keep API backwards compatible func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 8595a5c84c3..857f3656ddd 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -147,9 +147,9 @@ func (m *InMemoryStore) CreateTransaction(txRequest *types.TxRequest) *types.Tra m.UnstartedTransactions = m.UnstartedTransactions[1:maxQueuedTransactions] } - copy := tx.DeepCopy() - m.Transactions[copy.ID] = copy - m.UnstartedTransactions = append(m.UnstartedTransactions, copy) + txCopy := tx.DeepCopy() + m.Transactions[txCopy.ID] = txCopy + m.UnstartedTransactions = append(m.UnstartedTransactions, txCopy) return tx } diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 9e905e88c42..58a203e1cc3 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -153,6 +153,7 @@ func TestCreateTransaction(t *testing.T) { for i := 1; i < maxQueuedTransactions+overshot; i++ { r := &types.TxRequest{} tx := m.CreateTransaction(r) + //nolint:gosec // this won't overflow assert.Equal(t, uint64(i), tx.ID) } // total shouldn't exceed maxQueuedTransactions @@ -160,6 +161,7 @@ func TestCreateTransaction(t *testing.T) { // earliest tx ID should be the same amount of the number of transactions that we dropped tx, err := m.UpdateUnstartedTransactionWithNonce(0) assert.NoError(t, err) + //nolint:gosec // this won't overflow assert.Equal(t, uint64(overshot), tx.ID) }) } @@ -228,6 +230,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) for i := 0; i < maxQueuedTransactions; i++ { + //nolint:gosec _, err := insertConfirmedTransaction(m, uint64(i)) assert.NoError(t, err) } diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 926f75443ff..e12abec162a 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -262,6 +262,7 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) return true, nil } + //nolint:gosec pendingNonce, err := t.client.PendingNonceAt(ctx, address) if err != nil { return false, err @@ -350,6 +351,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err // TODO: add backoff to optimize requests } + //nolint:gosec if tx == nil || tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 3f5e6541629..4b18d937a30 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "go.uber.org/zap" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -44,7 +45,7 @@ func TestLifecycle(t *testing.T) { t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) - assert.NoError(t, txStore.Add(addresses...)) + require.NoError(t, txStore.Add(addresses...)) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) var nonce uint64 // Start @@ -102,7 +103,7 @@ func TestBroadcastTransaction(t *testing.T) { mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) - assert.Error(t, err) + require.Error(t, err) assert.False(t, bo) assert.Contains(t, err.Error(), "call failed") }) @@ -128,7 +129,7 @@ func TestBroadcastTransaction(t *testing.T) { client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), nil).Once() // LocalNonce: 1, PendingNonce: 0 bo, err := txm.broadcastTransaction(ctx, address) assert.True(t, bo) - assert.NoError(t, err) + require.NoError(t, err) client.On("PendingNonceAt", mock.Anything, address).Return(uint64(1), nil).Once() // LocalNonce: 1, PendingNonce: 1 mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index cf3a6dcc295..b9a098c617f 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -68,13 +68,13 @@ func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, erro } func (t *Transaction) DeepCopy() *Transaction { - copy := *t + txCopy := *t attemptsCopy := make([]*Attempt, 0, len(t.Attempts)) for _, attempt := range t.Attempts { attemptsCopy = append(attemptsCopy, attempt.DeepCopy()) } - copy.Attempts = attemptsCopy - return © + txCopy.Attempts = attemptsCopy + return &txCopy } func (t *Transaction) GetMeta() (*TxMeta, error) { @@ -102,12 +102,11 @@ type Attempt struct { } func (a *Attempt) DeepCopy() *Attempt { - copy := *a + txCopy := *a if a.SignedTransaction != nil { - signedTransactionCopy := *a.SignedTransaction - copy.SignedTransaction = &signedTransactionCopy + txCopy.SignedTransaction = a.SignedTransaction.WithoutBlobTxSidecar() } - return © + return &txCopy } type TxRequest struct { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 28dbcfe64b1..035df0bc819 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -118,6 +118,7 @@ func NewTxmv2( config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), BlockTime: blockTime, //TODO: create new config + //nolint:gosec // we want to reuse the existing config until migrations RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } From 338f2daf5b39b4721f2f8f119438a2aafe0629ff Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 17:17:49 +0200 Subject: [PATCH 39/73] Fix DualBroadcast client --- .../chains/evm/txm/clientwrappers/dual_broadcast_client.go | 7 ++++--- core/chains/evm/txmgr/builder.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index aa3a2db10ee..054c37602f0 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -60,6 +60,7 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra if err != nil { return err } + //nolint:gosec if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { data, err := attempt.SignedTransaction.MarshalBinary() if err != nil { @@ -104,14 +105,14 @@ func (d *DualBroadcastClient) signAndPostMessage(ctx context.Context, address co return result, fmt.Errorf("request %v failed with status: %d", postReq, resp.StatusCode) } - keyJson, err := io.ReadAll(resp.Body) + keyJSON, err := io.ReadAll(resp.Body) if err != nil { return } var response postResponse - err = json.Unmarshal(keyJson, &response) + err = json.Unmarshal(keyJSON, &response) if err != nil { - return result, fmt.Errorf("failed to unmarshal response into struct: %w: %s", err, string(keyJson)) + return result, fmt.Errorf("failed to unmarshal response into struct: %w: %s", err, string(keyJSON)) } if response.Error.Message != "" { return result, errors.New(response.Error.Message) diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 4592585d492..8e1644f3a73 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -130,7 +130,7 @@ func NewTxmV2( config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), BlockTime: *txmV2Config.BlockTime(), - //nolint:gosec // we want to reuse the existing config until migrations + //nolint:gosec // reuse existing config until migration RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } From 8a4e25fb30c89317d2bfe35fd6f77e2a849938d8 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 18:07:44 +0200 Subject: [PATCH 40/73] More lint fixes --- core/chains/evm/txm/orchestrator.go | 2 +- core/chains/evm/txm/storage/inmemory_store_test.go | 3 ++- core/chains/evm/txm/txm.go | 3 +-- core/chains/evm/txm/txm_test.go | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 182ddd3b1cd..3316aa4fc58 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -296,7 +296,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByReceiptBlockNum( return } -//nolint:gosec // keep API backwards compatible +//nolint:nolintlint // keep API backwards compatible func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 58a203e1cc3..68c791cb400 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -230,7 +230,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) for i := 0; i < maxQueuedTransactions; i++ { - //nolint:gosec + //nolint:gosec // this won't overflow _, err := insertConfirmedTransaction(m, uint64(i)) assert.NoError(t, err) } @@ -373,6 +373,7 @@ func TestPruneConfirmedTransactions(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) total := 5 for i := 0; i < total; i++ { + //nolint:gosec // this won't overflow _, err := insertConfirmedTransaction(m, uint64(i)) assert.NoError(t, err) } diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index e12abec162a..b03912b4fcd 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -262,7 +262,7 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) return true, nil } - //nolint:gosec + //nolint:govet //linter nonsense pendingNonce, err := t.client.PendingNonceAt(ctx, address) if err != nil { return false, err @@ -351,7 +351,6 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err // TODO: add backoff to optimize requests } - //nolint:gosec if tx == nil || tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 4b18d937a30..c2454e51079 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -115,7 +115,7 @@ func TestBroadcastTransaction(t *testing.T) { txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) assert.True(t, bo) - assert.NoError(t, err) + require.NoError(t, err) tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) @@ -164,7 +164,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("picks a new tx and creates a new attempt then sends it and updates the broadcast time", func(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) - assert.NoError(t, txStore.Add(address)) + require.NoError(t, txStore.Add(address)) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) txm.setNonce(address, 8) IDK := "IDK" @@ -176,7 +176,7 @@ func TestBroadcastTransaction(t *testing.T) { SpecifiedGasLimit: 22000, } tx, err := txm.CreateTransaction(tests.Context(t), txRequest) - assert.NoError(t, err) + require.NoError(t, err) attempt := &types.Attempt{ TxID: tx.ID, Fee: gas.EvmFee{GasPrice: assets.NewWeiI(1)}, From 83d15abdc013501b6b2cbcf5ad71a7b69480146e Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 19 Nov 2024 18:09:43 +0200 Subject: [PATCH 41/73] More lint fixes --- core/chains/evm/txm/clientwrappers/dual_broadcast_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index 054c37602f0..1907e5dcd20 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -60,7 +60,7 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra if err != nil { return err } - //nolint:gosec + //nolint:revive if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { data, err := attempt.SignedTransaction.MarshalBinary() if err != nil { From ee67544de71ee1a946411687ad8a8c645eae7229 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 20 Nov 2024 13:28:45 +0200 Subject: [PATCH 42/73] Fix lint --- core/chains/evm/txm/orchestrator.go | 2 +- core/chains/evm/txm/txm.go | 8 ++++---- core/chains/evm/txm/txm_test.go | 28 ++++++++++++++-------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 3316aa4fc58..694cce3d14b 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -296,7 +296,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithMetaFieldByReceiptBlockNum( return } -//nolint:nolintlint // keep API backwards compatible +//nolint:revive // keep API backwards compatible func (o *Orchestrator[BLOCK_HASH, HEAD]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txs []*txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { return } diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index b03912b4fcd..76847a97d29 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -262,10 +262,9 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) return true, nil } - //nolint:govet //linter nonsense - pendingNonce, err := t.client.PendingNonceAt(ctx, address) - if err != nil { - return false, err + pendingNonce, e := t.client.PendingNonceAt(ctx, address) + if e != nil { + return false, e } nonce := t.getNonce(address) if nonce > pendingNonce { @@ -354,6 +353,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if tx == nil || tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) + //nolint:revive //linter nonsense } else { if !tx.IsPurgeable && t.stuckTxDetector != nil { isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index c2454e51079..9fded2e982d 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -39,7 +39,7 @@ func TestLifecycle(t *testing.T) { t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() - assert.Error(t, txm.Start(tests.Context(t))) + require.Error(t, txm.Start(tests.Context(t))) }) t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { @@ -76,7 +76,7 @@ func TestTrigger(t *testing.T) { t.Run("executes Trigger", func(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) - assert.NoError(t, txStore.Add(address)) + require.NoError(t, txStore.Add(address)) client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} @@ -105,7 +105,7 @@ func TestBroadcastTransaction(t *testing.T) { bo, err := txm.broadcastTransaction(ctx, address) require.Error(t, err) assert.False(t, bo) - assert.Contains(t, err.Error(), "call failed") + require.ErrorContains(t, err, "call failed") }) t.Run("throws a warning and returns if unconfirmed transactions exceed maxInFlightTransactions", func(t *testing.T) { @@ -135,7 +135,7 @@ func TestBroadcastTransaction(t *testing.T) { mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil).Once() bo, err = txm.broadcastTransaction(ctx, address) assert.False(t, bo) - assert.NoError(t, err) + require.NoError(t, err) tests.AssertLogCountEventually(t, observedLogs, "Reached transaction limit.", 1) }) @@ -146,17 +146,17 @@ func TestBroadcastTransaction(t *testing.T) { mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() bo, err := txm.broadcastTransaction(ctx, address) assert.False(t, bo) - assert.Error(t, err) - assert.Contains(t, err.Error(), "call failed") + require.Error(t, err) + require.ErrorContains(t, err, "call failed") }) t.Run("returns if there are no unstarted transactions", func(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) - assert.NoError(t, txStore.Add(address)) + require.NoError(t, txStore.Add(address)) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, bo) assert.Equal(t, uint64(0), txm.getNonce(address)) }) @@ -186,11 +186,11 @@ func TestBroadcastTransaction(t *testing.T) { client.On("SendTransaction", mock.Anything, mock.Anything).Return(nil).Once() bo, err := txm.broadcastTransaction(ctx, address) - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, bo) assert.Equal(t, uint64(9), txm.getNonce(address)) tx, err = txStore.FindTxWithIdempotencyKey(tests.Context(t), &IDK) - assert.NoError(t, err) + require.NoError(t, err) assert.Len(t, tx.Attempts, 1) var zeroTime time.Time assert.Greater(t, tx.LastBroadcastAt, zeroTime) @@ -213,9 +213,9 @@ func TestBackfillTransactions(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() bo, err := txm.backfillTransactions(ctx, address) - assert.Error(t, err) + require.Error(t, err) assert.False(t, bo) - assert.Contains(t, err.Error(), "latest nonce fail") + require.ErrorContains(t, err, "latest nonce fail") }) t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { @@ -223,8 +223,8 @@ func TestBackfillTransactions(t *testing.T) { client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() bo, err := txm.backfillTransactions(ctx, address) - assert.Error(t, err) + require.Error(t, err) assert.False(t, bo) - assert.Contains(t, err.Error(), "marking transactions confirmed failed") + require.ErrorContains(t, err, "marking transactions confirmed failed") }) } From c05c87535e2a01d73e4085f6c88b818ffa0c1e0c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 20 Nov 2024 13:31:35 +0200 Subject: [PATCH 43/73] Fix lint --- core/chains/evm/txm/clientwrappers/dual_broadcast_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index 1907e5dcd20..0bbd2530765 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -60,7 +60,7 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra if err != nil { return err } - //nolint:revive + //nolint:revive //linter nonsense if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { data, err := attempt.SignedTransaction.MarshalBinary() if err != nil { From 0f707a1c07d6e7130bce9543b4891e98400489ca Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 20 Nov 2024 14:13:40 +0200 Subject: [PATCH 44/73] Fix lint final --- core/chains/evm/txm/orchestrator.go | 1 - .../storage/inmemory_store_manager_test.go | 7 +- .../evm/txm/storage/inmemory_store_test.go | 81 ++++++++++--------- core/chains/evm/txm/txm.go | 7 +- core/chains/evm/txmgr/builder.go | 4 +- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 694cce3d14b..ac2e62f4592 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -235,7 +235,6 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, } tx = txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]{ - //nolint:gosec // disable G115 ID: int64(wrappedTx.ID), IdempotencyKey: wrappedTx.IdempotencyKey, FromAddress: wrappedTx.FromAddress, diff --git a/core/chains/evm/txm/storage/inmemory_store_manager_test.go b/core/chains/evm/txm/storage/inmemory_store_manager_test.go index e10870a9942..aff589fb9e1 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager_test.go @@ -5,6 +5,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -18,18 +19,18 @@ func TestAdd(t *testing.T) { m := NewInMemoryStoreManager(logger.Test(t), testutils.FixtureChainID) // Adds a new address err := m.Add(fromAddress) - assert.NoError(t, err) + require.NoError(t, err) assert.Len(t, m.InMemoryStoreMap, 1) // Fails if address exists err = m.Add(fromAddress) - assert.Error(t, err) + require.Error(t, err) // Adds multiple addresses fromAddress1 := testutils.NewAddress() fromAddress2 := testutils.NewAddress() addresses := []common.Address{fromAddress1, fromAddress2} err = m.Add(addresses...) - assert.NoError(t, err) + require.NoError(t, err) assert.Len(t, m.InMemoryStoreMap, 3) } diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 68c791cb400..62fc28cea64 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -26,9 +27,9 @@ func TestAbandonPendingTransactions(t *testing.T) { // Unconfirmed tx3, err := insertUnconfirmedTransaction(m, 3) - assert.NoError(t, err) + require.NoError(t, err) tx4, err := insertUnconfirmedTransaction(m, 4) - assert.NoError(t, err) + require.NoError(t, err) m.AbandonPendingTransactions() @@ -45,9 +46,9 @@ func TestAbandonPendingTransactions(t *testing.T) { // Confirmed tx3, err := insertConfirmedTransaction(m, 3) - assert.NoError(t, err) + require.NoError(t, err) tx4, err := insertConfirmedTransaction(m, 4) - assert.NoError(t, err) + require.NoError(t, err) m.AbandonPendingTransactions() @@ -65,16 +66,16 @@ func TestAppendAttemptToTransaction(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) _, err := insertUnconfirmedTransaction(m, 0) // txID = 1 - assert.NoError(t, err) + require.NoError(t, err) _, err = insertConfirmedTransaction(m, 2) // txID = 1 - assert.NoError(t, err) + require.NoError(t, err) t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { var nonce uint64 = 1 newAttempt := &types.Attempt{ TxID: 1, } - assert.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + require.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) }) t.Run("fails if unconfirmed transaction was found but has doesn't match the txID", func(t *testing.T) { @@ -82,7 +83,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { newAttempt := &types.Attempt{ TxID: 2, } - assert.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + require.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) }) t.Run("appends attempt to transaction", func(t *testing.T) { @@ -90,7 +91,7 @@ func TestAppendAttemptToTransaction(t *testing.T) { newAttempt := &types.Attempt{ TxID: 1, } - assert.NoError(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + require.NoError(t, m.AppendAttemptToTransaction(nonce, newAttempt)) }) } @@ -112,16 +113,16 @@ func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) _, err := insertUnconfirmedTransaction(m, 0) - assert.NoError(t, err) + require.NoError(t, err) t.Run("fails if unconfirmed transaction with the same nonce exists", func(t *testing.T) { _, err := m.CreateEmptyUnconfirmedTransaction(0, 0) - assert.Error(t, err) + require.Error(t, err) }) t.Run("creates a new empty unconfirmed transaction", func(t *testing.T) { tx, err := m.CreateEmptyUnconfirmedTransaction(1, 0) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, types.TxUnconfirmed, tx.State) }) } @@ -160,7 +161,7 @@ func TestCreateTransaction(t *testing.T) { assert.Equal(t, maxQueuedTransactions, m.CountUnstartedTransactions()) // earliest tx ID should be the same amount of the number of transactions that we dropped tx, err := m.UpdateUnstartedTransactionWithNonce(0) - assert.NoError(t, err) + require.NoError(t, err) //nolint:gosec // this won't overflow assert.Equal(t, uint64(overshot), tx.ID) }) @@ -178,7 +179,7 @@ func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { var nonce uint64 _, err := insertUnconfirmedTransaction(m, nonce) - assert.NoError(t, err) + require.NoError(t, err) tx, count = m.FetchUnconfirmedTransactionAtNonceWithCount(0) assert.Equal(t, tx.Nonce, nonce) assert.Equal(t, 1, count) @@ -199,13 +200,13 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("confirms transaction with nonce lower than the latest", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) ctx1, err := insertUnconfirmedTransaction(m, 0) - assert.NoError(t, err) + require.NoError(t, err) ctx2, err := insertUnconfirmedTransaction(m, 1) - assert.NoError(t, err) + require.NoError(t, err) ctxs, utxs := m.MarkTransactionsConfirmed(1) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) assert.Equal(t, ctxs[0], ctx1.ID) @@ -215,13 +216,13 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("unconfirms transaction with nonce equal to or higher than the latest", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) ctx1, err := insertConfirmedTransaction(m, 0) - assert.NoError(t, err) + require.NoError(t, err) ctx2, err := insertConfirmedTransaction(m, 1) - assert.NoError(t, err) + require.NoError(t, err) ctxs, utxs := m.MarkTransactionsConfirmed(1) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) assert.Equal(t, utxs[0], ctx2.ID) @@ -232,7 +233,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { for i := 0; i < maxQueuedTransactions; i++ { //nolint:gosec // this won't overflow _, err := insertConfirmedTransaction(m, uint64(i)) - assert.NoError(t, err) + require.NoError(t, err) } assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions) m.MarkTransactionsConfirmed(maxQueuedTransactions) @@ -248,12 +249,12 @@ func TestMarkUnconfirmedTransactionPurgeable(t *testing.T) { // fails if tx was not found err := m.MarkUnconfirmedTransactionPurgeable(0) - assert.Error(t, err) + require.Error(t, err) tx, err := insertUnconfirmedTransaction(m, 0) - assert.NoError(t, err) + require.NoError(t, err) err = m.MarkUnconfirmedTransactionPurgeable(0) - assert.NoError(t, err) + require.NoError(t, err) assert.True(t, tx.IsPurgeable) } @@ -265,30 +266,30 @@ func TestUpdateTransactionBroadcast(t *testing.T) { t.Run("fails if unconfirmed transaction was not found", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) var nonce uint64 - assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + require.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) }) t.Run("fails if attempt was not found for a given transaction", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) var nonce uint64 tx, err := insertUnconfirmedTransaction(m, nonce) - assert.NoError(t, err) - assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + require.NoError(t, err) + require.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) // Attempt with different hash attempt := &types.Attempt{TxID: tx.ID, Hash: testutils.NewHash()} tx.Attempts = append(tx.Attempts, attempt) - assert.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + require.Error(t, m.UpdateTransactionBroadcast(0, nonce, hash)) }) t.Run("updates transaction's and attempt's broadcast times", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) var nonce uint64 tx, err := insertUnconfirmedTransaction(m, nonce) - assert.NoError(t, err) + require.NoError(t, err) attempt := &types.Attempt{TxID: tx.ID, Hash: hash} tx.Attempts = append(tx.Attempts, attempt) - assert.NoError(t, m.UpdateTransactionBroadcast(0, nonce, hash)) + require.NoError(t, m.UpdateTransactionBroadcast(0, nonce, hash)) assert.False(t, tx.LastBroadcastAt.IsZero()) assert.False(t, attempt.BroadcastAt.IsZero()) }) @@ -301,7 +302,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { t.Run("returns nil if there are no unstarted transactions", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) tx, err := m.UpdateUnstartedTransactionWithNonce(0) - assert.NoError(t, err) + require.NoError(t, err) assert.Nil(t, tx) }) @@ -310,10 +311,10 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) insertUnstartedTransaction(m) _, err := insertUnconfirmedTransaction(m, nonce) - assert.NoError(t, err) + require.NoError(t, err) _, err = m.UpdateUnstartedTransactionWithNonce(nonce) - assert.Error(t, err) + require.Error(t, err) }) t.Run("updates unstarted transaction to unconfirmed and assigns a nonce", func(t *testing.T) { @@ -322,7 +323,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { insertUnstartedTransaction(m) tx, err := m.UpdateUnstartedTransactionWithNonce(nonce) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, nonce, tx.Nonce) assert.Equal(t, types.TxUnconfirmed, tx.State) }) @@ -337,18 +338,18 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { tx := &types.Transaction{Nonce: 0} attempt := &types.Attempt{TxID: 0} err := m.DeleteAttemptForUnconfirmedTx(tx.Nonce, attempt) - assert.Error(t, err) + require.Error(t, err) }) t.Run("fails if corresponding unconfirmed attempt for txID was not found", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) _, err := insertUnconfirmedTransaction(m, 0) - assert.NoError(t, err) + require.NoError(t, err) attempt := &types.Attempt{TxID: 2, Hash: testutils.NewHash()} err = m.DeleteAttemptForUnconfirmedTx(0, attempt) - assert.Error(t, err) + require.Error(t, err) }) t.Run("deletes attempt of unconfirmed transaction", func(t *testing.T) { @@ -356,12 +357,12 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { var nonce uint64 m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) tx, err := insertUnconfirmedTransaction(m, nonce) - assert.NoError(t, err) + require.NoError(t, err) attempt := &types.Attempt{TxID: 0, Hash: hash} tx.Attempts = append(tx.Attempts, attempt) err = m.DeleteAttemptForUnconfirmedTx(nonce, attempt) - assert.NoError(t, err) + require.NoError(t, err) assert.Empty(t, tx.Attempts) }) @@ -375,7 +376,7 @@ func TestPruneConfirmedTransactions(t *testing.T) { for i := 0; i < total; i++ { //nolint:gosec // this won't overflow _, err := insertConfirmedTransaction(m, uint64(i)) - assert.NoError(t, err) + require.NoError(t, err) } prunedTxIDs := m.pruneConfirmedTransactions() left := total - total/pruneSubset diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 76847a97d29..5bb7e6e0b75 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -177,9 +177,9 @@ func (t *Txm) setNonce(address common.Address, nonce uint64) { defer t.nonceMapMu.Unlock() } -func newBackoff(min time.Duration) backoff.Backoff { +func newBackoff(minDuration time.Duration) backoff.Backoff { return backoff.Backoff{ - Min: min, + Min: minDuration, Max: 1 * time.Minute, Jitter: true, } @@ -353,8 +353,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if tx == nil || tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) - //nolint:revive //linter nonsense - } else { + } else { //nolint:revive //linter nonsense if !tx.IsPurgeable && t.stuckTxDetector != nil { isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) if err != nil { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 035df0bc819..85d2a894ddc 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -116,8 +116,8 @@ func NewTxmv2( attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMax(), estimator, keyStore) inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID) config := txm.Config{ - EIP1559: fCfg.EIP1559DynamicFees(), - BlockTime: blockTime, //TODO: create new config + EIP1559: fCfg.EIP1559DynamicFees(), + BlockTime: blockTime, //TODO: create new config //nolint:gosec // we want to reuse the existing config until migrations RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), From 291bbc0c28df3b013a0d2932880f4afa0e25141c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 20 Nov 2024 17:30:48 +0200 Subject: [PATCH 45/73] Fix races --- core/chains/evm/txm/orchestrator.go | 1 + core/chains/evm/txm/txm.go | 15 ++++++++++----- core/chains/evm/txm/txm_test.go | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index ac2e62f4592..694cce3d14b 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -235,6 +235,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, } tx = txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]{ + //nolint:gosec // disable G115 ID: int64(wrappedTx.ID), IdempotencyKey: wrappedTx.IdempotencyKey, FromAddress: wrappedTx.FromAddress, diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 5bb7e6e0b75..8d7eba419b1 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -124,7 +124,8 @@ func (t *Txm) Start(ctx context.Context) error { } func (t *Txm) startAddress(address common.Address) error { - t.triggerCh[address] = make(chan struct{}, 1) + triggerCh := make(chan struct{}, 1) + t.triggerCh[address] = triggerCh pendingNonce, err := t.client.PendingNonceAt(context.TODO(), address) if err != nil { return err @@ -132,7 +133,7 @@ func (t *Txm) startAddress(address common.Address) error { t.setNonce(address, pendingNonce) t.wg.Add(2) - go t.broadcastLoop(address) + go t.broadcastLoop(address, triggerCh) go t.backfillLoop(address) return nil } @@ -155,7 +156,11 @@ func (t *Txm) CreateTransaction(ctx context.Context, txRequest *types.TxRequest) func (t *Txm) Trigger(address common.Address) { if !t.IfStarted(func() { - t.triggerCh[address] <- struct{}{} + triggerCh, exists := t.triggerCh[address] + if !exists { + return + } + triggerCh <- struct{}{} }) { t.lggr.Error("Txm unstarted") } @@ -185,7 +190,7 @@ func newBackoff(minDuration time.Duration) backoff.Backoff { } } -func (t *Txm) broadcastLoop(address common.Address) { +func (t *Txm) broadcastLoop(address common.Address, triggerCh chan struct{}) { defer t.wg.Done() ctx, cancel := t.stopCh.NewCtx() defer cancel() @@ -209,7 +214,7 @@ func (t *Txm) broadcastLoop(address common.Address) { select { case <-ctx.Done(): return - case <-t.triggerCh[address]: + case <-triggerCh: continue case <-broadcastCh: continue diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 9fded2e982d..8314bddb811 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -65,7 +65,6 @@ func TestTrigger(t *testing.T) { address := testutils.NewAddress() keystore := mocks.NewKeystore(t) - keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.ErrorLevel) txm := NewTxm(lggr, nil, nil, nil, nil, Config{}, keystore) @@ -80,6 +79,7 @@ func TestTrigger(t *testing.T) { client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) var nonce uint64 // Start From d45090b8beb40b255d7cbed6f196e885f588639e Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 25 Nov 2024 11:37:54 +0200 Subject: [PATCH 46/73] Update logs --- core/chains/evm/txm/txm.go | 4 ++-- core/chains/evm/txm/types/transaction.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 8d7eba419b1..0fafe8df006 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -201,7 +201,7 @@ func (t *Txm) broadcastLoop(address common.Address, triggerCh chan struct{}) { start := time.Now() bo, err := t.broadcastTransaction(ctx, address) if err != nil { - t.lggr.Errorf("Error during transaction broadcasting: %v", err) + t.lggr.Errorf("Error during transaction broadcasting: %w", err) } else { t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) } @@ -237,7 +237,7 @@ func (t *Txm) backfillLoop(address common.Address) { start := time.Now() bo, err := t.backfillTransactions(ctx, address) if err != nil { - t.lggr.Errorf("Error during backfill: %v", err) + t.lggr.Errorf("Error during backfill: %w", err) } else { t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) } diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index b9a098c617f..94ddabce81b 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -58,6 +58,13 @@ type Transaction struct { CallbackCompleted bool } +// func (t *Transaction) String() string { +// return fmt.Sprintf(`{"ID":%d, "IdempotencyKey":%v, "ChainID":%v, "Nonce":%d, "FromAddress":%v, "ToAddress":%v, "Value":%v, `+ +// `"Data":%v, "SpecifiedGasLimit":%d, "CreatedAt":%v, "LastBroadcastAt":%v, "State":%v, "IsPurgeable":%v, "AttemptCount":%d, `+ +// `"Meta":%v, "Subject":%v, "PipelineTaskRunID":%v, "MinConfirmations":%v, "SignalCallback":%v, "CallbackCompleted":%v`, +// t.ID, *t.IdempotencyKey, t.ChainID, t.Nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.LastBroadcastAt, +// t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject, t.PipelineTaskRunID, t.MinConfirmations, t.SignalCallback, t.CallbackCompleted) +// } func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, error) { for _, a := range t.Attempts { if a.Hash == attemptHash { @@ -109,6 +116,11 @@ func (a *Attempt) DeepCopy() *Attempt { return &txCopy } +func (a *Attempt) String() string { + return fmt.Sprintf(`{"ID":%d, "TxID":%d, "Hash":%v, "Fee":%v, "GasLimit":%d, "Type":%v, "CreatedAt":%v, "BroadcastAt":%v}`, + a.ID, a.TxID, a.Hash, a.Fee, a.GasLimit, a.Type, a.CreatedAt, a.BroadcastAt) +} + type TxRequest struct { IdempotencyKey *string ChainID *big.Int From 43ed971f93787c28a3f904fa0af761ed8175f881 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 25 Nov 2024 11:52:52 +0200 Subject: [PATCH 47/73] Fix lint --- core/chains/evm/txm/txm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 0fafe8df006..4153f8072f1 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -201,7 +201,7 @@ func (t *Txm) broadcastLoop(address common.Address, triggerCh chan struct{}) { start := time.Now() bo, err := t.broadcastTransaction(ctx, address) if err != nil { - t.lggr.Errorf("Error during transaction broadcasting: %w", err) + t.lggr.Errorw("Error during transaction broadcasting", "err", err) } else { t.lggr.Debug("Transaction broadcasting time elapsed: ", time.Since(start)) } @@ -237,7 +237,7 @@ func (t *Txm) backfillLoop(address common.Address) { start := time.Now() bo, err := t.backfillTransactions(ctx, address) if err != nil { - t.lggr.Errorf("Error during backfill: %w", err) + t.lggr.Errorw("Error during backfill", "err", err) } else { t.lggr.Debug("Backfill time elapsed: ", time.Since(start)) } From 21ff676ffb12f3d87e6c1f2ecbe6f07be1519690 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 25 Nov 2024 15:15:06 +0200 Subject: [PATCH 48/73] Start documentation --- core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md diff --git a/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md new file mode 100644 index 00000000000..b7c350a2a17 --- /dev/null +++ b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md @@ -0,0 +1,8 @@ + +# Transaction Manager V2 + +## Configs +- `EIP1559`: enables EIP-1559 mode. This means the transaction manager will create and broadcast Dynamic attempts. Set this to false to broadcast Legacy transactions. +- `BlockTime`: controls the interval of the backfill loop. This dictates how frequently the transaction manager will check for confirmed transactions, rebroadcast stuck ones, and fill any nonce gaps. Transactions are getting confirmed only during new blocks so it's best if you set this to a value close to the block time. At least one RPC call is made during each BlockTime interval so the absolute minimum should be 2s. A small jitter is applied so the timeout won't be exactly the same each time. +- `RetryBlockThreshold`: is the number of blocks to wait for a transaction stuck in the mempool before automatically rebroadcasting it with a new attempt. +- `EmptyTxLimitDefault`: sets default gas limit for empty transactions. Empty transactions are created in case there is a nonce gap or another stuck transaction in the mempool to fill a given nonce. These are empty transactions and they don't have any data or value. \ No newline at end of file From 887e199dfe3c4cc1711bf0e353af514e20626952 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 27 Nov 2024 14:55:20 +0200 Subject: [PATCH 49/73] Update DummyKeystore Add method --- core/chains/evm/txm/dummy_keystore.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 3456c06b1e3..1161f34058b 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -19,11 +19,19 @@ func NewKeystore() *DummyKeystore { return &DummyKeystore{privateKeyMap: make(map[common.Address]*ecdsa.PrivateKey)} } -func (k *DummyKeystore) Add(privateKeyString string, address common.Address) error { +func (k *DummyKeystore) Add(privateKeyString string) error { privateKey, err := crypto.HexToECDSA(privateKeyString) if err != nil { return err } + + publicKey := privateKey.Public() + publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) + if !ok { + return fmt.Errorf("error casting public key: %v to ECDSA", publicKey) + } + + address := crypto.PubkeyToAddress(*publicKeyECDSA) k.privateKeyMap[address] = privateKey return nil } From cecb0e1ada32235adbddcbab87163b4123e4ec8c Mon Sep 17 00:00:00 2001 From: Dimitris Grigoriou Date: Wed, 27 Nov 2024 15:26:50 +0200 Subject: [PATCH 50/73] Txmv2 stuck tx detection (#15436) * Stuck tx detector alpha * Update stuck tx detection * Add stuck_tx_detection and dual broadcast client * Add support for TXMv2 * Fix orchestrator's monitoring call * Fix AttemptBuilder * Enable DualBroadcast client * Switch DualBroadcast params to pointers * Add context to client * Fix lint * Fix DualBroadcast client * More lint fixes * Fix lint --- core/chains/evm/config/chain_scoped.go | 4 + core/chains/evm/config/chain_scoped_txmv2.go | 25 ++ core/chains/evm/config/chaintype/chaintype.go | 6 +- core/chains/evm/config/config.go | 7 + core/chains/evm/config/toml/config.go | 27 ++- core/chains/evm/config/toml/defaults.go | 1 + .../evm/config/toml/defaults/fallback.toml | 3 + core/chains/evm/keystore/eth.go | 1 + core/chains/evm/keystore/mocks/eth.go | 60 +++++ .../evm/txm/clientwrappers/chain_client.go | 31 +++ .../clientwrappers/dual_broadcast_client.go | 130 ++++++++++ .../evm/txm/clientwrappers/geth_client.go | 51 ++++ core/chains/evm/txm/dummy_keystore.go | 13 + core/chains/evm/txm/mocks/client.go | 27 ++- core/chains/evm/txm/stuck_tx_detector.go | 115 +++++++++ core/chains/evm/txm/txm.go | 30 +-- core/chains/evm/txm/txm_test.go | 26 +- core/chains/evm/txm/types/transaction.go | 4 + core/chains/evm/txmgr/builder.go | 32 ++- core/chains/legacyevm/evm_txm.go | 43 ++-- core/config/docs/chains-evm.toml | 8 + docs/CONFIG.md | 222 ++++++++++++++++++ 22 files changed, 801 insertions(+), 65 deletions(-) create mode 100644 core/chains/evm/config/chain_scoped_txmv2.go create mode 100644 core/chains/evm/txm/clientwrappers/chain_client.go create mode 100644 core/chains/evm/txm/clientwrappers/dual_broadcast_client.go create mode 100644 core/chains/evm/txm/clientwrappers/geth_client.go create mode 100644 core/chains/evm/txm/stuck_tx_detector.go diff --git a/core/chains/evm/config/chain_scoped.go b/core/chains/evm/config/chain_scoped.go index de89272b5e2..3a7ff43d8a6 100644 --- a/core/chains/evm/config/chain_scoped.go +++ b/core/chains/evm/config/chain_scoped.go @@ -52,6 +52,10 @@ func (e *EVMConfig) BalanceMonitor() BalanceMonitor { return &balanceMonitorConfig{c: e.C.BalanceMonitor} } +func (e *EVMConfig) TxmV2() TxmV2 { + return &txmv2Config{c: e.C.TxmV2} +} + func (e *EVMConfig) Transactions() Transactions { return &transactionsConfig{c: e.C.Transactions} } diff --git a/core/chains/evm/config/chain_scoped_txmv2.go b/core/chains/evm/config/chain_scoped_txmv2.go new file mode 100644 index 00000000000..e50148cfae4 --- /dev/null +++ b/core/chains/evm/config/chain_scoped_txmv2.go @@ -0,0 +1,25 @@ +package config + +import ( + "net/url" + "time" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" +) + +type txmv2Config struct { + c toml.TxmV2 +} + +func (t *txmv2Config) Enabled() bool { + return *t.c.Enabled +} + +func (t *txmv2Config) BlockTime() *time.Duration { + d := t.c.BlockTime.Duration() + return &d +} + +func (t *txmv2Config) CustomURL() *url.URL { + return t.c.CustomURL.URL() +} diff --git a/core/chains/evm/config/chaintype/chaintype.go b/core/chains/evm/config/chaintype/chaintype.go index b2eff02834b..bc2eace8ca3 100644 --- a/core/chains/evm/config/chaintype/chaintype.go +++ b/core/chains/evm/config/chaintype/chaintype.go @@ -23,6 +23,7 @@ const ( ChainZkEvm ChainType = "zkevm" ChainZkSync ChainType = "zksync" ChainZircuit ChainType = "zircuit" + ChainDualBroadcast ChainType = "dualBroadcast" ) // IsL2 returns true if this chain is a Layer 2 chain. Notably: @@ -39,7 +40,7 @@ func (c ChainType) IsL2() bool { func (c ChainType) IsValid() bool { switch c { - case "", ChainArbitrum, ChainAstar, ChainCelo, ChainGnosis, ChainHedera, ChainKroma, ChainMantle, ChainMetis, ChainOptimismBedrock, ChainScroll, ChainWeMix, ChainXLayer, ChainZkEvm, ChainZkSync, ChainZircuit: + case "", ChainArbitrum, ChainAstar, ChainCelo, ChainGnosis, ChainHedera, ChainKroma, ChainMantle, ChainMetis, ChainOptimismBedrock, ChainScroll, ChainWeMix, ChainXLayer, ChainZkEvm, ChainZkSync, ChainZircuit, ChainDualBroadcast: return true } return false @@ -77,6 +78,8 @@ func FromSlug(slug string) ChainType { return ChainZkSync case "zircuit": return ChainZircuit + case "dualBroadcast": + return ChainDualBroadcast default: return ChainType(slug) } @@ -144,4 +147,5 @@ var ErrInvalid = fmt.Errorf("must be one of %s or omitted", strings.Join([]strin string(ChainZkEvm), string(ChainZkSync), string(ChainZircuit), + string(ChainDualBroadcast), }, ", ")) diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index f2a571f94b0..be61bfe02ee 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -18,6 +18,7 @@ import ( type EVM interface { HeadTracker() HeadTracker BalanceMonitor() BalanceMonitor + TxmV2() TxmV2 Transactions() Transactions GasEstimator() GasEstimator OCR() OCR @@ -102,6 +103,12 @@ type ClientErrors interface { TooManyResults() string } +type TxmV2 interface { + Enabled() bool + BlockTime() *time.Duration + CustomURL() *url.URL +} + type Transactions interface { ForwardersEnabled() bool ReaperInterval() time.Duration diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 0505449943e..0f8b1eceee5 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -300,8 +300,10 @@ func (c *EVMConfig) ValidateConfig() (err error) { is := c.ChainType.ChainType() if is != must { if must == "" { - err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), - Msg: "must not be set with this chain id"}) + if c.ChainType.ChainType() != chaintype.ChainDualBroadcast { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), + Msg: "must not be set with this chain id"}) + } } else { err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), Msg: fmt.Sprintf("only %q can be used with this chain id", must)}) @@ -387,6 +389,7 @@ type Chain struct { FinalizedBlockOffset *uint32 NoNewFinalizedHeadsThreshold *commonconfig.Duration + TxmV2 TxmV2 `toml:",omitempty"` Transactions Transactions `toml:",omitempty"` BalanceMonitor BalanceMonitor `toml:",omitempty"` GasEstimator GasEstimator `toml:",omitempty"` @@ -471,6 +474,26 @@ func (c *Chain) ValidateConfig() (err error) { return } +type TxmV2 struct { + Enabled *bool `toml:",omitempty"` + BlockTime *commonconfig.Duration `toml:",omitempty"` + CustomURL *commonconfig.URL `toml:",omitempty"` +} + +func (t *TxmV2) setFrom(f *TxmV2) { + if v := f.Enabled; v != nil { + t.Enabled = f.Enabled + } + + if v := f.BlockTime; v != nil { + t.BlockTime = f.BlockTime + } + + if v := f.CustomURL; v != nil { + t.CustomURL = f.CustomURL + } +} + type Transactions struct { ForwardersEnabled *bool MaxInFlight *uint32 diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 0885d94e6df..5ce014921f4 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -241,6 +241,7 @@ func (c *Chain) SetFrom(f *Chain) { c.NoNewFinalizedHeadsThreshold = v } + c.TxmV2.setFrom(&f.TxmV2) c.Transactions.setFrom(&f.Transactions) c.BalanceMonitor.setFrom(&f.BalanceMonitor) c.GasEstimator.setFrom(&f.GasEstimator) diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index ab349ee4688..44228968fc6 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -18,6 +18,9 @@ FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0' LogBroadcasterEnabled = true +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/core/chains/evm/keystore/eth.go b/core/chains/evm/keystore/eth.go index ff71e0a4f18..9c0986d9c3d 100644 --- a/core/chains/evm/keystore/eth.go +++ b/core/chains/evm/keystore/eth.go @@ -13,5 +13,6 @@ type Eth interface { CheckEnabled(ctx context.Context, address common.Address, chainID *big.Int) error EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) SubscribeToKeyChanges(ctx context.Context) (ch chan struct{}, unsub func()) } diff --git a/core/chains/evm/keystore/mocks/eth.go b/core/chains/evm/keystore/mocks/eth.go index b481be1b5c8..bfc85fc672c 100644 --- a/core/chains/evm/keystore/mocks/eth.go +++ b/core/chains/evm/keystore/mocks/eth.go @@ -133,6 +133,66 @@ func (_c *Eth_EnabledAddressesForChain_Call) RunAndReturn(run func(context.Conte return _c } +// SignMessage provides a mock function with given fields: ctx, address, message +func (_m *Eth) SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) { + ret := _m.Called(ctx, address, message) + + if len(ret) == 0 { + panic("no return value specified for SignMessage") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) ([]byte, error)); ok { + return rf(ctx, address, message) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) []byte); ok { + r0 = rf(ctx, address, message) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, []byte) error); ok { + r1 = rf(ctx, address, message) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Eth_SignMessage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SignMessage' +type Eth_SignMessage_Call struct { + *mock.Call +} + +// SignMessage is a helper method to define mock.On call +// - ctx context.Context +// - address common.Address +// - message []byte +func (_e *Eth_Expecter) SignMessage(ctx interface{}, address interface{}, message interface{}) *Eth_SignMessage_Call { + return &Eth_SignMessage_Call{Call: _e.mock.On("SignMessage", ctx, address, message)} +} + +func (_c *Eth_SignMessage_Call) Run(run func(ctx context.Context, address common.Address, message []byte)) *Eth_SignMessage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].([]byte)) + }) + return _c +} + +func (_c *Eth_SignMessage_Call) Return(_a0 []byte, _a1 error) *Eth_SignMessage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Eth_SignMessage_Call) RunAndReturn(run func(context.Context, common.Address, []byte) ([]byte, error)) *Eth_SignMessage_Call { + _c.Call.Return(run) + return _c +} + // SignTx provides a mock function with given fields: ctx, fromAddress, tx, chainID func (_m *Eth) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { ret := _m.Called(ctx, fromAddress, tx, chainID) diff --git a/core/chains/evm/txm/clientwrappers/chain_client.go b/core/chains/evm/txm/clientwrappers/chain_client.go new file mode 100644 index 00000000000..7638cc53443 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/chain_client.go @@ -0,0 +1,31 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type ChainClient struct { + c client.Client +} + +func NewChainClient(client client.Client) *ChainClient { + return &ChainClient{c: client} +} + +func (c *ChainClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return c.c.NonceAt(ctx, address, blockNumber) +} + +func (c *ChainClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + return c.c.PendingNonceAt(ctx, address) +} + +func (c *ChainClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return c.c.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go new file mode 100644 index 00000000000..0bbd2530765 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -0,0 +1,130 @@ +package clientwrappers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "net/url" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type DualBroadcastClientKeystore interface { + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) +} + +type DualBroadcastClient struct { + c client.Client + keystore DualBroadcastClientKeystore + customURL *url.URL +} + +func NewDualBroadcastClient(c client.Client, keystore DualBroadcastClientKeystore, customURL *url.URL) *DualBroadcastClient { + return &DualBroadcastClient{ + c: c, + keystore: keystore, + customURL: customURL, + } +} + +func (d *DualBroadcastClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return d.c.NonceAt(ctx, address, blockNumber) +} + +func (d *DualBroadcastClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":["%s","pending"]}`, address.String())) + response, err := d.signAndPostMessage(ctx, address, body, "") + if err != nil { + return 0, err + } + + nonce, err := hexutil.DecodeUint64(response) + if err != nil { + return 0, fmt.Errorf("failed to decode response %v into uint64: %w", response, err) + } + return nonce, nil +} + +func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + meta, err := tx.GetMeta() + if err != nil { + return err + } + //nolint:revive //linter nonsense + if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { + data, err := attempt.SignedTransaction.MarshalBinary() + if err != nil { + return err + } + params := "" + if meta.DualBroadcastParams != nil { + params = *meta.DualBroadcastParams + } + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, hexutil.Encode(data))) + if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, params); err != nil { + return err + } + return nil + } else { + return d.c.SendTransaction(ctx, attempt.SignedTransaction) + } +} + +func (d *DualBroadcastClient) signAndPostMessage(ctx context.Context, address common.Address, body []byte, urlParams string) (result string, err error) { + bodyReader := bytes.NewReader(body) + postReq, err := http.NewRequestWithContext(ctx, http.MethodPost, d.customURL.String()+"?"+urlParams, bodyReader) + if err != nil { + return + } + + hashedBody := crypto.Keccak256Hash(body).Hex() + signedMessage, err := d.keystore.SignMessage(ctx, address, []byte(hashedBody)) + if err != nil { + return + } + + postReq.Header.Add("X-Flashbots-signature", address.String()+":"+hexutil.Encode(signedMessage)) + postReq.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(postReq) + if err != nil { + return result, fmt.Errorf("request %v failed: %w", postReq, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("request %v failed with status: %d", postReq, resp.StatusCode) + } + + keyJSON, err := io.ReadAll(resp.Body) + if err != nil { + return + } + var response postResponse + err = json.Unmarshal(keyJSON, &response) + if err != nil { + return result, fmt.Errorf("failed to unmarshal response into struct: %w: %s", err, string(keyJSON)) + } + if response.Error.Message != "" { + return result, errors.New(response.Error.Message) + } + return response.Result, nil +} + +type postResponse struct { + Result string `json:"result,omitempty"` + Error postError +} + +type postError struct { + Message string `json:"message,omitempty"` +} diff --git a/core/chains/evm/txm/clientwrappers/geth_client.go b/core/chains/evm/txm/clientwrappers/geth_client.go new file mode 100644 index 00000000000..d97e5cfae35 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/geth_client.go @@ -0,0 +1,51 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" +) + +type GethClient struct { + *ethclient.Client +} + +func NewGethClient(client *ethclient.Client) *GethClient { + return &GethClient{ + Client: client, + } +} + +func (g *GethClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { + return g.Client.Client().BatchCallContext(ctx, b) +} + +func (g *GethClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + return g.Client.Client().CallContext(ctx, result, method, args...) +} + +func (g *GethClient) CallContract(ctx context.Context, message ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + var hex hexutil.Bytes + err := g.CallContext(ctx, &hex, "eth_call", client.ToBackwardCompatibleCallArg(message), client.ToBackwardCompatibleBlockNumArg(blockNumber)) + return hex, err +} + +func (g *GethClient) HeadByNumber(ctx context.Context, number *big.Int) (*evmtypes.Head, error) { + hexNumber := client.ToBlockNumArg(number) + args := []interface{}{hexNumber, false} + head := new(evmtypes.Head) + err := g.CallContext(ctx, head, "eth_getBlockByNumber", args...) + return head, err +} + +func (g *GethClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return g.Client.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 1161f34058b..01816dfcbbd 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -43,6 +44,18 @@ func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx return nil, fmt.Errorf("private key for address: %v not found", fromAddress) } +func (k *DummyKeystore) SignMessage(ctx context.Context, address common.Address, data []byte) ([]byte, error) { + key, exists := k.privateKeyMap[address] + if !exists { + return nil, fmt.Errorf("private key for address: %v not found", address) + } + signature, err := crypto.Sign(accounts.TextHash(data), key) + if err != nil { + return nil, fmt.Errorf("failed to sign message for address: %v", address) + } + return signature, nil +} + func (k *DummyKeystore) EnabledAddressesForChain(_ context.Context, _ *big.Int) (addresses []common.Address, err error) { for address := range k.privateKeyMap { addresses = append(addresses, address) diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go index 533298625e5..03849ad7e82 100644 --- a/core/chains/evm/txm/mocks/client.go +++ b/core/chains/evm/txm/mocks/client.go @@ -10,7 +10,7 @@ import ( mock "github.com/stretchr/testify/mock" - types "github.com/ethereum/go-ethereum/core/types" + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) // Client is an autogenerated mock type for the Client type @@ -141,17 +141,17 @@ func (_c *Client_PendingNonceAt_Call) RunAndReturn(run func(context.Context, com return _c } -// SendTransaction provides a mock function with given fields: _a0, _a1 -func (_m *Client) SendTransaction(_a0 context.Context, _a1 *types.Transaction) error { - ret := _m.Called(_a0, _a1) +// SendTransaction provides a mock function with given fields: ctx, tx, attempt +func (_m *Client) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + ret := _m.Called(ctx, tx, attempt) if len(ret) == 0 { panic("no return value specified for SendTransaction") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction, *types.Attempt) error); ok { + r0 = rf(ctx, tx, attempt) } else { r0 = ret.Error(0) } @@ -165,15 +165,16 @@ type Client_SendTransaction_Call struct { } // SendTransaction is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *types.Transaction -func (_e *Client_Expecter) SendTransaction(_a0 interface{}, _a1 interface{}) *Client_SendTransaction_Call { - return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", _a0, _a1)} +// - ctx context.Context +// - tx *types.Transaction +// - attempt *types.Attempt +func (_e *Client_Expecter) SendTransaction(ctx interface{}, tx interface{}, attempt interface{}) *Client_SendTransaction_Call { + return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", ctx, tx, attempt)} } -func (_c *Client_SendTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Client_SendTransaction_Call { +func (_c *Client_SendTransaction_Call) Run(run func(ctx context.Context, tx *types.Transaction, attempt *types.Attempt)) *Client_SendTransaction_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Transaction)) + run(args[0].(context.Context), args[1].(*types.Transaction), args[2].(*types.Attempt)) }) return _c } @@ -183,7 +184,7 @@ func (_c *Client_SendTransaction_Call) Return(_a0 error) *Client_SendTransaction return _c } -func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction) error) *Client_SendTransaction_Call { +func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction, *types.Attempt) error) *Client_SendTransaction_Call { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go new file mode 100644 index 00000000000..68d8caf0ed1 --- /dev/null +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -0,0 +1,115 @@ +package txm + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type StuckTxDetectorConfig struct { + BlockTime time.Duration + StuckTxBlockThreshold uint32 + DetectionURL string +} + +type stuckTxDetector struct { + lggr logger.Logger + chainType chaintype.ChainType + config StuckTxDetectorConfig +} + +func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, config StuckTxDetectorConfig) *stuckTxDetector { + return &stuckTxDetector{ + lggr: lggr, + chainType: chaintype, + config: config, + } +} + +func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) { + switch s.chainType { + // TODO: rename + case chaintype.ChainDualBroadcast: + result, err := s.dualBroadcastDetection(ctx, tx) + if result || err != nil { + return result, err + } + return s.timeBasedDetection(tx), nil + default: + return s.timeBasedDetection(tx), nil + } +} + +func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { + threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) + if time.Since(tx.LastBroadcastAt) > threshold && !tx.LastBroadcastAt.IsZero() { + s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", + tx.ID, tx.LastBroadcastAt, threshold) + return true + } + return false +} + +type APIResponse struct { + Status string `json:"status,omitempty"` + Hash common.Hash `json:"hash,omitempty"` +} + +const ( + APIStatusPending = "PENDING" + APIStatusIncluded = "INCLUDED" + APIStatusFailed = "FAILED" + APIStatusCancelled = "CANCELLED" + APIStatusUnknown = "UNKNOWN" +) + +func (s *stuckTxDetector) dualBroadcastDetection(ctx context.Context, tx *types.Transaction) (bool, error) { + for _, attempt := range tx.Attempts { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.config.DetectionURL+attempt.Hash.String(), nil) + if err != nil { + return false, fmt.Errorf("failed to make request for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, fmt.Errorf("failed to get transaction status for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return false, fmt.Errorf("request %v failed with status: %d", req, resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return false, err + } + + var apiResponse APIResponse + err = json.Unmarshal(body, &apiResponse) + if err != nil { + return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w: %s", tx.ID, attempt.Hash, err, string(body)) + } + switch apiResponse.Status { + case APIStatusPending, APIStatusIncluded: + return false, nil + case APIStatusFailed, APIStatusCancelled: + s.lggr.Debugf("TxID: %v with attempHash: %v was marked as failed/cancelled by the RPC. Transaction is now considered stuck and will be purged.", + tx.ID, attempt.Hash) + return true, nil + case APIStatusUnknown: + continue + default: + continue + } + } + return false, nil +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 4153f8072f1..ef8e682ab6a 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -8,7 +8,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - evmtypes "github.com/ethereum/go-ethereum/core/types" "github.com/jpillora/backoff" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -28,7 +27,7 @@ const ( type Client interface { PendingNonceAt(context.Context, common.Address) (uint64, error) NonceAt(context.Context, common.Address, *big.Int) (uint64, error) - SendTransaction(context.Context, *evmtypes.Transaction) error + SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error } type TxStore interface { @@ -57,7 +56,7 @@ type ErrorHandler interface { } type StuckTxDetector interface { - DetectStuckTransaction(tx *types.Transaction) (bool, error) + DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) } type Keystore interface { @@ -91,17 +90,18 @@ type Txm struct { wg sync.WaitGroup } -func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, keystore Keystore) *Txm { +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, stuckTxDetector StuckTxDetector, config Config, keystore Keystore) *Txm { return &Txm{ - lggr: logger.Sugared(logger.Named(lggr, "Txm")), - keystore: keystore, - chainID: chainID, - client: client, - attemptBuilder: attemptBuilder, - txStore: txStore, - config: config, - nonceMap: make(map[common.Address]uint64), - triggerCh: make(map[common.Address]chan struct{}), + lggr: logger.Sugared(logger.Named(lggr, "Txm")), + keystore: keystore, + chainID: chainID, + client: client, + attemptBuilder: attemptBuilder, + txStore: txStore, + stuckTxDetector: stuckTxDetector, + config: config, + nonceMap: make(map[common.Address]uint64), + triggerCh: make(map[common.Address]chan struct{}), } } @@ -311,7 +311,7 @@ func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, a func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transaction, attempt *types.Attempt, address common.Address) (err error) { start := time.Now() - txErr := t.client.SendTransaction(ctx, attempt.SignedTransaction) + txErr := t.client.SendTransaction(ctx, tx, attempt) tx.AttemptCount++ t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { @@ -360,7 +360,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { //nolint:revive //linter nonsense if !tx.IsPurgeable && t.stuckTxDetector != nil { - isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) + isStuck, err := t.stuckTxDetector.DetectStuckTransaction(ctx, tx) if err != nil { return false, err } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 8314bddb811..93742924d58 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -37,7 +37,7 @@ func TestLifecycle(t *testing.T) { keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return(addresses, nil) t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() require.Error(t, txm.Start(tests.Context(t))) }) @@ -46,7 +46,7 @@ func TestLifecycle(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(addresses...)) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) var nonce uint64 // Start client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() @@ -67,7 +67,7 @@ func TestTrigger(t *testing.T) { keystore := mocks.NewKeystore(t) t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.ErrorLevel) - txm := NewTxm(lggr, nil, nil, nil, nil, Config{}, keystore) + txm := NewTxm(lggr, nil, nil, nil, nil, nil, Config{}, keystore) txm.Trigger(address) tests.AssertLogEventually(t, observedLogs, "Txm unstarted") }) @@ -80,7 +80,7 @@ func TestTrigger(t *testing.T) { ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) var nonce uint64 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() @@ -101,7 +101,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) require.Error(t, err) assert.False(t, bo) @@ -112,7 +112,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions+1, nil).Once() - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) assert.True(t, bo) require.NoError(t, err) @@ -122,7 +122,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) txm.setNonce(address, 1) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions/3, nil).Twice() @@ -142,7 +142,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() bo, err := txm.broadcastTransaction(ctx, address) assert.False(t, bo) @@ -154,7 +154,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(address)) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) require.NoError(t, err) assert.False(t, bo) @@ -165,7 +165,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(address)) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) txm.setNonce(address, 8) IDK := "IDK" txRequest := &types.TxRequest{ @@ -183,7 +183,7 @@ func TestBroadcastTransaction(t *testing.T) { GasLimit: 22000, } ab.On("NewAttempt", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(attempt, nil).Once() - client.On("SendTransaction", mock.Anything, mock.Anything).Return(nil).Once() + client.On("SendTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() bo, err := txm.broadcastTransaction(ctx, address) require.NoError(t, err) @@ -210,7 +210,7 @@ func TestBackfillTransactions(t *testing.T) { keystore := mocks.NewKeystore(t) t.Run("fails if latest nonce fetching fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() bo, err := txm.backfillTransactions(ctx, address) require.Error(t, err) @@ -219,7 +219,7 @@ func TestBackfillTransactions(t *testing.T) { }) t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() bo, err := txm.backfillTransactions(ctx, address) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 94ddabce81b..d32b56628e3 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -170,6 +170,10 @@ type TxMeta struct { // CCIP MessageIDs []string `json:"MessageIDs,omitempty"` SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` + + // Dual Broadcast + DualBroadcast *bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams *string `json:"DualBroadcastParams,omitempty"` } type QueueingTxStrategy struct { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 85d2a894ddc..8e1644f3a73 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/clientwrappers" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) @@ -93,12 +94,12 @@ func NewEvmTxm( return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError) } -func NewTxmv2( +func NewTxmV2( ds sqlutil.DataSource, chainConfig ChainConfig, fCfg FeeConfig, - blockTime time.Duration, - fwdEnabled bool, + txConfig config.Transactions, + txmV2Config config.TxmV2, client client.Client, lggr logger.Logger, logPoller logpoller.LogPoller, @@ -106,23 +107,40 @@ func NewTxmv2( estimator gas.EvmFeeEstimator, ) (TxManager, error) { var fwdMgr *forwarders.FwdMgr - if fwdEnabled { + if txConfig.ForwardersEnabled() { fwdMgr = forwarders.NewFwdMgr(ds, client, logPoller, lggr, chainConfig) } else { lggr.Info("ForwarderManager: Disabled") } chainID := client.ConfiguredChainID() + + var stuckTxDetector txm.StuckTxDetector + if txConfig.AutoPurge().Enabled() { + stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ + BlockTime: *txmV2Config.BlockTime(), + StuckTxBlockThreshold: *txConfig.AutoPurge().Threshold(), + DetectionURL: txConfig.AutoPurge().DetectionApiUrl().String(), + } + stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) + } + attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMax(), estimator, keyStore) inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), - BlockTime: blockTime, //TODO: create new config - //nolint:gosec // we want to reuse the existing config until migrations + BlockTime: *txmV2Config.BlockTime(), + //nolint:gosec // reuse existing config until migration RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, keyStore) + var c txm.Client + if chainConfig.ChainType() == chaintype.ChainDualBroadcast { + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomURL()) + } else { + c = clientwrappers.NewChainClient(client) + } + t := txm.NewTxm(lggr, chainID, c, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, keyStore) return txm.NewTxmOrchestrator(lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil } diff --git a/core/chains/legacyevm/evm_txm.go b/core/chains/legacyevm/evm_txm.go index 3a96a9da937..1192462156e 100644 --- a/core/chains/legacyevm/evm_txm.go +++ b/core/chains/legacyevm/evm_txm.go @@ -55,20 +55,35 @@ func newEvmTxm( } if opts.GenTxManager == nil { - txm, err = txmgr.NewTxm( - ds, - cfg, - txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), - cfg.Transactions(), - cfg.NodePool().Errors(), - databaseConfig, - listenerConfig, - client, - lggr, - logPoller, - opts.KeyStore, - estimator, - headTracker) + if cfg.TxmV2().Enabled() { + txm, err = txmgr.NewTxmV2( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.TxmV2(), + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + ) + } else { + txm, err = txmgr.NewTxm( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.NodePool().Errors(), + databaseConfig, + listenerConfig, + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + headTracker) + } } else { txm = opts.GenTxManager(chainID) } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 62360cb02cb..cd99027e611 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -129,6 +129,14 @@ ReaperThreshold = '168h' # Default # ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. ResendAfterThreshold = '1m' # Default +[EVM.TxmV2] +# Enabled enables TxmV2. +Enabled = false # Default +# BlockTime controls the frequency of the backfill loop of TxmV2. +BlockTime = '10s' # Example +# CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomURL = 'https://example.api.io' # Example + [EVM.Transactions.AutoPurge] # Enabled enables or disables automatically purging transactions that have been idenitified as terminally stuck (will never be included on-chain). This feature is only expected to be used by ZK chains. Enabled = false # Default diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 30ada2455ca..baf5832f9d7 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2033,6 +2033,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2137,6 +2140,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2241,6 +2247,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2345,6 +2354,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2450,6 +2462,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '13m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2558,6 +2573,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2662,6 +2680,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2767,6 +2788,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2871,6 +2895,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '45s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2974,6 +3001,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3077,6 +3107,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3181,6 +3214,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3286,6 +3322,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3390,6 +3429,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3494,6 +3536,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '6m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3598,6 +3643,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3703,6 +3751,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3916,6 +3967,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4020,6 +4074,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4128,6 +4185,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4235,6 +4295,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4339,6 +4402,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4443,6 +4509,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4550,6 +4619,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4658,6 +4730,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4874,6 +4949,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4977,6 +5055,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5081,6 +5162,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5185,6 +5269,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5290,6 +5377,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5394,6 +5484,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5713,6 +5806,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5818,6 +5914,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '2h0m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5926,6 +6025,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6034,6 +6136,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6139,6 +6244,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6350,6 +6458,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6454,6 +6565,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6562,6 +6676,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6667,6 +6784,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6775,6 +6895,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6883,6 +7006,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6990,6 +7116,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7094,6 +7223,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7198,6 +7330,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7302,6 +7437,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '45m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7407,6 +7545,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7518,6 +7659,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7627,6 +7771,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7730,6 +7877,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7835,6 +7985,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7941,6 +8094,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8153,6 +8309,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8256,6 +8415,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '12m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8463,6 +8625,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8571,6 +8736,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '12m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8680,6 +8848,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8788,6 +8959,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8895,6 +9069,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9002,6 +9179,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9111,6 +9291,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9328,6 +9511,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9432,6 +9618,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9540,6 +9729,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9644,6 +9836,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9985,6 +10180,33 @@ ResendAfterThreshold = '1m' # Default ``` ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. +## EVM.TxmV2 +```toml +[EVM.TxmV2] +Enabled = false # Default +BlockTime = '10s' # Example +CustomURL = 'https://example.api.io' # Example +``` + + +### Enabled +```toml +Enabled = false # Default +``` +Enabled enables TxmV2. + +### BlockTime +```toml +BlockTime = '10s' # Example +``` +BlockTime controls the frequency of the backfill loop of TxmV2. + +### CustomURL +```toml +CustomURL = 'https://example.api.io' # Example +``` +CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. + ## EVM.Transactions.AutoPurge ```toml [EVM.Transactions.AutoPurge] From 91e35ea6c68ce906ec3f90732798df42b39a2f6f Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 27 Nov 2024 16:44:11 +0200 Subject: [PATCH 51/73] Make nonce nullable --- core/chains/evm/txm/attempt_builder.go | 10 +++++-- core/chains/evm/txm/storage/inmemory_store.go | 26 ++++++++++------- .../evm/txm/storage/inmemory_store_manager.go | 2 +- .../evm/txm/storage/inmemory_store_test.go | 29 +++++++++++-------- core/chains/evm/txm/txm.go | 21 +++++++++----- core/chains/evm/txm/types/transaction.go | 2 +- 6 files changed, 57 insertions(+), 33 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index ceae96f1b28..fd23bf867e7 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -92,8 +92,11 @@ func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transac toAddress = tx.ToAddress value = tx.Value } + if tx.Nonce == nil { + return nil, fmt.Errorf("failed to create attempt for txID: %v: nonce empty", tx.ID) + } legacyTx := evmtypes.LegacyTx{ - Nonce: tx.Nonce, + Nonce: *tx.Nonce, To: &toAddress, Value: value, Gas: estimatedGasLimit, @@ -126,8 +129,11 @@ func (a *attemptBuilder) newDynamicFeeAttempt(ctx context.Context, tx *types.Tra toAddress = tx.ToAddress value = tx.Value } + if tx.Nonce == nil { + return nil, fmt.Errorf("failed to create attempt for txID: %v: nonce empty", tx.ID) + } dynamicTx := evmtypes.DynamicFeeTx{ - Nonce: tx.Nonce, + Nonce: *tx.Nonce, To: &toAddress, Value: value, Gas: estimatedGasLimit, diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 857f3656ddd..06b6475b0cc 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -98,7 +98,7 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit emptyTx := &types.Transaction{ ID: m.txIDCount, ChainID: m.chainID, - Nonce: nonce, + Nonce: &nonce, FromAddress: m.address, ToAddress: common.Address{}, Value: big.NewInt(0), @@ -165,28 +165,34 @@ func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(latestNonce return } -func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, []uint64) { +func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, []uint64, error) { m.Lock() defer m.Unlock() var confirmedTransactionIDs []uint64 for _, tx := range m.UnconfirmedTransactions { - if tx.Nonce < latestNonce { + if tx.Nonce == nil { + return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + if *tx.Nonce < latestNonce { tx.State = types.TxConfirmed confirmedTransactionIDs = append(confirmedTransactionIDs, tx.ID) - m.ConfirmedTransactions[tx.Nonce] = tx - delete(m.UnconfirmedTransactions, tx.Nonce) + m.ConfirmedTransactions[*tx.Nonce] = tx + delete(m.UnconfirmedTransactions, *tx.Nonce) } } var unconfirmedTransactionIDs []uint64 for _, tx := range m.ConfirmedTransactions { - if tx.Nonce >= latestNonce { + if tx.Nonce == nil { + return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + if *tx.Nonce >= latestNonce { tx.State = types.TxUnconfirmed tx.LastBroadcastAt = time.Time{} // Mark reorged transaction as if it wasn't broadcasted before unconfirmedTransactionIDs = append(unconfirmedTransactionIDs, tx.ID) - m.UnconfirmedTransactions[tx.Nonce] = tx - delete(m.ConfirmedTransactions, tx.Nonce) + m.UnconfirmedTransactions[*tx.Nonce] = tx + delete(m.ConfirmedTransactions, *tx.Nonce) } } @@ -197,7 +203,7 @@ func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, } sort.Slice(confirmedTransactionIDs, func(i, j int) bool { return confirmedTransactionIDs[i] < confirmedTransactionIDs[j] }) sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) - return confirmedTransactionIDs, unconfirmedTransactionIDs + return confirmedTransactionIDs, unconfirmedTransactionIDs, nil } func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(nonce uint64) error { @@ -249,7 +255,7 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type } tx := m.UnstartedTransactions[0] - tx.Nonce = nonce + tx.Nonce = &nonce tx.State = types.TxUnconfirmed m.UnstartedTransactions = m.UnstartedTransactions[1:] diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index dfb777ab22b..d6cb1bfb906 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -83,7 +83,7 @@ func (m *InMemoryStoreManager) FetchUnconfirmedTransactionAtNonceWithCount(_ con func (m *InMemoryStoreManager) MarkTransactionsConfirmed(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxIDs []uint64, unconfirmedTxIDs []uint64, err error) { if store, exists := m.InMemoryStoreMap[fromAddress]; exists { - confirmedTxIDs, unconfirmedTxIDs = store.MarkTransactionsConfirmed(nonce) + confirmedTxIDs, unconfirmedTxIDs, err = store.MarkTransactionsConfirmed(nonce) return } return nil, nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 62fc28cea64..efc5111afcb 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -181,7 +181,7 @@ func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { _, err := insertUnconfirmedTransaction(m, nonce) require.NoError(t, err) tx, count = m.FetchUnconfirmedTransactionAtNonceWithCount(0) - assert.Equal(t, tx.Nonce, nonce) + assert.Equal(t, *tx.Nonce, nonce) assert.Equal(t, 1, count) } @@ -192,7 +192,8 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("returns 0 if there are no transactions", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - un, cn := m.MarkTransactionsConfirmed(100) + un, cn, err := m.MarkTransactionsConfirmed(100) + require.NoError(t, err) assert.Empty(t, un) assert.Empty(t, cn) }) @@ -205,7 +206,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { ctx2, err := insertUnconfirmedTransaction(m, 1) require.NoError(t, err) - ctxs, utxs := m.MarkTransactionsConfirmed(1) + ctxs, utxs, err := m.MarkTransactionsConfirmed(1) require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) @@ -221,7 +222,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { ctx2, err := insertConfirmedTransaction(m, 1) require.NoError(t, err) - ctxs, utxs := m.MarkTransactionsConfirmed(1) + ctxs, utxs, err := m.MarkTransactionsConfirmed(1) require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) @@ -236,7 +237,8 @@ func TestMarkTransactionsConfirmed(t *testing.T) { require.NoError(t, err) } assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions) - m.MarkTransactionsConfirmed(maxQueuedTransactions) + _, _, err := m.MarkTransactionsConfirmed(maxQueuedTransactions) + require.NoError(t, err) assert.Len(t, m.ConfirmedTransactions, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset)) }) } @@ -324,7 +326,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { tx, err := m.UpdateUnstartedTransactionWithNonce(nonce) require.NoError(t, err) - assert.Equal(t, nonce, tx.Nonce) + assert.Equal(t, nonce, *tx.Nonce) assert.Equal(t, types.TxUnconfirmed, tx.State) }) } @@ -335,9 +337,10 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { fromAddress := testutils.NewAddress() t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - tx := &types.Transaction{Nonce: 0} + var nonce uint64 + tx := &types.Transaction{Nonce: &nonce} attempt := &types.Attempt{TxID: 0} - err := m.DeleteAttemptForUnconfirmedTx(tx.Nonce, attempt) + err := m.DeleteAttemptForUnconfirmedTx(*tx.Nonce, attempt) require.Error(t, err) }) @@ -388,11 +391,12 @@ func insertUnstartedTransaction(m *InMemoryStore) *types.Transaction { m.Lock() defer m.Unlock() + var nonce uint64 m.txIDCount++ tx := &types.Transaction{ ID: m.txIDCount, ChainID: testutils.FixtureChainID, - Nonce: 0, + Nonce: &nonce, FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), @@ -413,7 +417,7 @@ func insertUnconfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transa tx := &types.Transaction{ ID: m.txIDCount, ChainID: testutils.FixtureChainID, - Nonce: nonce, + Nonce: &nonce, FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), @@ -438,7 +442,7 @@ func insertConfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transact tx := &types.Transaction{ ID: m.txIDCount, ChainID: testutils.FixtureChainID, - Nonce: nonce, + Nonce: &nonce, FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), @@ -459,11 +463,12 @@ func insertFataTransaction(m *InMemoryStore) *types.Transaction { m.Lock() defer m.Unlock() + var nonce uint64 m.txIDCount++ tx := &types.Transaction{ ID: m.txIDCount, ChainID: testutils.FixtureChainID, - Nonce: 0, + Nonce: &nonce, FromAddress: m.address, ToAddress: testutils.NewAddress(), Value: big.NewInt(0), diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index ef8e682ab6a..52c6084833d 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -286,8 +286,9 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) if tx == nil { return false, nil } - tx.Nonce = t.getNonce(address) - t.setNonce(address, tx.Nonce+1) + nonce := t.getNonce(address) + tx.Nonce = &nonce + t.setNonce(address, *tx.Nonce+1) tx.State = types.TxUnconfirmed if err := t.createAndSendAttempt(ctx, tx, address); err != nil { @@ -302,7 +303,10 @@ func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, a return err } - if err = t.txStore.AppendAttemptToTransaction(ctx, tx.Nonce, address, attempt); err != nil { + if tx.Nonce == nil { + return fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } + if err = t.txStore.AppendAttemptToTransaction(ctx, *tx.Nonce, address, attempt); err != nil { return err } @@ -310,6 +314,9 @@ func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, a } func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transaction, attempt *types.Attempt, address common.Address) (err error) { + if tx.Nonce == nil { + return fmt.Errorf("nonce for txID: %v is empty", tx.ID) + } start := time.Now() txErr := t.client.SendTransaction(ctx, tx, attempt) tx.AttemptCount++ @@ -323,13 +330,13 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio if err != nil { return err } - if pendingNonce <= tx.Nonce { + if pendingNonce <= *tx.Nonce { t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, tx.Nonce) return nil } } - return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, tx.Nonce, attempt.Hash, address) + return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, *tx.Nonce, attempt.Hash, address) } func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) (bool, error) { @@ -355,7 +362,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err // TODO: add backoff to optimize requests } - if tx == nil || tx.Nonce != latestNonce { + if tx == nil || *tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { //nolint:revive //linter nonsense @@ -366,7 +373,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) } if isStuck { tx.IsPurgeable = true - err = t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, tx.Nonce, address) + err = t.txStore.MarkUnconfirmedTransactionPurgeable(ctx, *tx.Nonce, address) if err != nil { return false, err } diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index d32b56628e3..1b6110151f3 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -33,7 +33,7 @@ type Transaction struct { ID uint64 IdempotencyKey *string ChainID *big.Int - Nonce uint64 + Nonce *uint64 FromAddress common.Address ToAddress common.Address Value *big.Int From 78e0690cc3d8022082279d0eceac67a0d5c5a7ac Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 27 Nov 2024 18:59:49 +0200 Subject: [PATCH 52/73] Update configs --- core/config/docs/chains-evm.toml | 16 ++-- core/config/docs/docs_test.go | 4 + core/services/chainlink/config_test.go | 16 +++- .../chainlink/testdata/config-full.toml | 3 + .../chainlink/testdata/config-invalid.toml | 6 ++ .../config-multi-chain-effective.toml | 9 +++ core/web/resolver/testdata/config-full.toml | 3 + .../config-multi-chain-effective.toml | 9 +++ docs/CONFIG.md | 78 ++++++++++++------- .../node/validate/defaults-override.txtar | 3 + .../disk-based-logging-disabled.txtar | 3 + .../validate/disk-based-logging-no-dir.txtar | 3 + .../node/validate/disk-based-logging.txtar | 3 + testdata/scripts/node/validate/invalid.txtar | 3 + testdata/scripts/node/validate/valid.txtar | 3 + 15 files changed, 125 insertions(+), 37 deletions(-) diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index cd99027e611..d94a89e29fe 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -105,6 +105,14 @@ LogBroadcasterEnabled = true # Default # Set to zero to disable. NoNewFinalizedHeadsThreshold = '0' # Default +[EVM.TxmV2] +# Enabled enables TxmV2. +Enabled = false # Default +# BlockTime controls the frequency of the backfill loop of TxmV2. +BlockTime = '10s' # Example +# CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomURL = 'https://example.api.io' # Example + [EVM.Transactions] # ForwardersEnabled enables or disables sending transactions through forwarder contracts. ForwardersEnabled = false # Default @@ -129,14 +137,6 @@ ReaperThreshold = '168h' # Default # ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. ResendAfterThreshold = '1m' # Default -[EVM.TxmV2] -# Enabled enables TxmV2. -Enabled = false # Default -# BlockTime controls the frequency of the backfill loop of TxmV2. -BlockTime = '10s' # Example -# CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. -CustomURL = 'https://example.api.io' # Example - [EVM.Transactions.AutoPurge] # Enabled enables or disables automatically purging transactions that have been idenitified as terminally stuck (will never be included on-chain). This feature is only expected to be used by ZK chains. Enabled = false # Default diff --git a/core/config/docs/docs_test.go b/core/config/docs/docs_test.go index 9fca08ee99b..6ec0e2acb6b 100644 --- a/core/config/docs/docs_test.go +++ b/core/config/docs/docs_test.go @@ -92,6 +92,10 @@ func TestDoc(t *testing.T) { docDefaults.Workflow.GasLimitDefault = &gasLimitDefault docDefaults.NodePool.Errors = evmcfg.ClientErrors{} + // TxmV2 configs are only set if the feature is enabled + docDefaults.TxmV2.BlockTime = nil + docDefaults.TxmV2.CustomURL = nil + // Transactions.AutoPurge configs are only set if the feature is enabled docDefaults.Transactions.AutoPurge.DetectionApiUrl = nil docDefaults.Transactions.AutoPurge.Threshold = nil diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 769005feb72..1dbc46d069d 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -653,6 +653,9 @@ func TestConfig_Marshal(t *testing.T) { RPCBlockQueryDelay: ptr[uint16](10), NoNewFinalizedHeadsThreshold: &hour, + TxmV2: evmcfg.TxmV2{ + Enabled: ptr(false), + }, Transactions: evmcfg.Transactions{ MaxInFlight: ptr[uint32](19), MaxQueued: ptr[uint32](99), @@ -1117,6 +1120,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 16 NoNewFinalizedHeadsThreshold = '1h0m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = true MaxInFlight = 19 @@ -1402,6 +1408,12 @@ func TestConfig_full(t *testing.T) { got.EVM[c].Nodes[n].Order = ptr(int32(100)) } } + if got.EVM[c].TxmV2.BlockTime == nil { + got.EVM[c].TxmV2.BlockTime = new(commoncfg.Duration) + } + if got.EVM[c].TxmV2.CustomURL == nil { + got.EVM[c].TxmV2.CustomURL = new(commoncfg.URL) + } if got.EVM[c].Transactions.AutoPurge.Threshold == nil { got.EVM[c].Transactions.AutoPurge.Threshold = ptr(uint32(0)) } @@ -1468,7 +1480,7 @@ func TestConfig_Validate(t *testing.T) { - 1: 10 errors: - ChainType: invalid value (Foo): must not be set with this chain id - Nodes: missing: must have at least one node - - ChainType: invalid value (Foo): must be one of arbitrum, astar, celo, gnosis, hedera, kroma, mantle, metis, optimismBedrock, scroll, wemix, xlayer, zkevm, zksync, zircuit or omitted + - ChainType: invalid value (Foo): must be one of arbitrum, astar, celo, gnosis, hedera, kroma, mantle, metis, optimismBedrock, scroll, wemix, xlayer, zkevm, zksync, zircuit, dualBroadcast or omitted - HeadTracker.HistoryDepth: invalid value (30): must be greater than or equal to FinalizedBlockOffset - GasEstimator.BumpThreshold: invalid value (0): cannot be 0 if auto-purge feature is enabled for Foo - Transactions.AutoPurge.Threshold: missing: needs to be set if auto-purge feature is enabled for Foo @@ -1481,7 +1493,7 @@ func TestConfig_Validate(t *testing.T) { - 2: 5 errors: - ChainType: invalid value (Arbitrum): only "optimismBedrock" can be used with this chain id - Nodes: missing: must have at least one node - - ChainType: invalid value (Arbitrum): must be one of arbitrum, astar, celo, gnosis, hedera, kroma, mantle, metis, optimismBedrock, scroll, wemix, xlayer, zkevm, zksync, zircuit or omitted + - ChainType: invalid value (Arbitrum): must be one of arbitrum, astar, celo, gnosis, hedera, kroma, mantle, metis, optimismBedrock, scroll, wemix, xlayer, zkevm, zksync, zircuit, dualBroadcast or omitted - FinalityDepth: invalid value (0): must be greater than or equal to 1 - MinIncomingConfirmations: invalid value (0): must be greater than or equal to 1 - 3: 3 errors: diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 47193f80184..9b78aceb032 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -336,6 +336,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 16 NoNewFinalizedHeadsThreshold = '1h0m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = true MaxInFlight = 19 diff --git a/core/services/chainlink/testdata/config-invalid.toml b/core/services/chainlink/testdata/config-invalid.toml index 967ef76de8e..8dc38dcb7d9 100644 --- a/core/services/chainlink/testdata/config-invalid.toml +++ b/core/services/chainlink/testdata/config-invalid.toml @@ -55,6 +55,9 @@ ChainType = 'Foo' FinalityDepth = 32 FinalizedBlockOffset = 64 +[EVM.TxmV2] +Enabled = true + [EVM.Transactions.AutoPurge] Enabled = true @@ -108,6 +111,9 @@ WSURL = 'ws://dupe.com' ChainID = '534352' ChainType = 'scroll' +[EVM.TxmV2] +Enabled = true + [EVM.Transactions.AutoPurge] Enabled = true DetectionApiUrl = '' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 7e658b170db..4ceaee2f4cc 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -319,6 +319,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 12 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -429,6 +432,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -533,6 +539,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '6m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index ef26bfea75a..79c85e3025d 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -336,6 +336,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = true MaxInFlight = 19 diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 7bdf50b9080..eb016a7c831 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -319,6 +319,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -429,6 +432,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -533,6 +539,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '6m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/docs/CONFIG.md b/docs/CONFIG.md index baf5832f9d7..342ce69d82a 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -3859,6 +3859,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '1h10m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4841,6 +4844,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '1h30m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5591,6 +5597,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '1h10m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5699,6 +5708,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '45m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6351,6 +6363,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '1h30m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8201,6 +8216,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '1h50m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8521,6 +8539,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '5m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9403,6 +9424,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '1h50m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -10122,6 +10146,33 @@ out-of-sync. Only applicable if `FinalityTagEnabled=true` Set to zero to disable. +## EVM.TxmV2 +```toml +[EVM.TxmV2] +Enabled = false # Default +BlockTime = '10s' # Example +CustomURL = 'https://example.api.io' # Example +``` + + +### Enabled +```toml +Enabled = false # Default +``` +Enabled enables TxmV2. + +### BlockTime +```toml +BlockTime = '10s' # Example +``` +BlockTime controls the frequency of the backfill loop of TxmV2. + +### CustomURL +```toml +CustomURL = 'https://example.api.io' # Example +``` +CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. + ## EVM.Transactions ```toml [EVM.Transactions] @@ -10180,33 +10231,6 @@ ResendAfterThreshold = '1m' # Default ``` ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. -## EVM.TxmV2 -```toml -[EVM.TxmV2] -Enabled = false # Default -BlockTime = '10s' # Example -CustomURL = 'https://example.api.io' # Example -``` - - -### Enabled -```toml -Enabled = false # Default -``` -Enabled enables TxmV2. - -### BlockTime -```toml -BlockTime = '10s' # Example -``` -BlockTime controls the frequency of the backfill loop of TxmV2. - -### CustomURL -```toml -CustomURL = 'https://example.api.io' # Example -``` -CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. - ## EVM.Transactions.AutoPurge ```toml [EVM.Transactions.AutoPurge] diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 336f170bd1b..0297c9ed73a 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -392,6 +392,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 677058e1c08..7575b351a81 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -375,6 +375,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 0e5a78f4a39..985340b68be 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -375,6 +375,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 7fc05533a47..253501356f0 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -375,6 +375,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index b048af38a3b..fdc61742bf3 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -365,6 +365,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index bc84a9b2a37..1da3a20cde1 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -372,6 +372,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[EVM.TxmV2] +Enabled = false + [EVM.Transactions] ForwardersEnabled = false MaxInFlight = 16 From 272ba06fdc83bd44e1420c47dfa241ca57f8a622 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Thu, 28 Nov 2024 17:38:50 +0200 Subject: [PATCH 53/73] Add prom metrics --- .../evm/txm/docs/TRANSACTION_MANAGER_V2.md | 7 ++++++- core/chains/evm/txm/txm.go | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md index b7c350a2a17..638b8474f3d 100644 --- a/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md +++ b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md @@ -5,4 +5,9 @@ - `EIP1559`: enables EIP-1559 mode. This means the transaction manager will create and broadcast Dynamic attempts. Set this to false to broadcast Legacy transactions. - `BlockTime`: controls the interval of the backfill loop. This dictates how frequently the transaction manager will check for confirmed transactions, rebroadcast stuck ones, and fill any nonce gaps. Transactions are getting confirmed only during new blocks so it's best if you set this to a value close to the block time. At least one RPC call is made during each BlockTime interval so the absolute minimum should be 2s. A small jitter is applied so the timeout won't be exactly the same each time. - `RetryBlockThreshold`: is the number of blocks to wait for a transaction stuck in the mempool before automatically rebroadcasting it with a new attempt. -- `EmptyTxLimitDefault`: sets default gas limit for empty transactions. Empty transactions are created in case there is a nonce gap or another stuck transaction in the mempool to fill a given nonce. These are empty transactions and they don't have any data or value. \ No newline at end of file +- `EmptyTxLimitDefault`: sets default gas limit for empty transactions. Empty transactions are created in case there is a nonce gap or another stuck transaction in the mempool to fill a given nonce. These are empty transactions and they don't have any data or value. + +## Metrics +- `txm_num_broadcasted_transactions` : total number of successful broadcasted transactions. +- `txm_num_confirmed_transactions` : total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs. +- `txm_num_nonce_gaps` : total number of nonce gaps created that the transaction manager had to fill. \ No newline at end of file diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 52c6084833d..3347a32e51c 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -9,6 +9,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/jpillora/backoff" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -63,6 +65,21 @@ type Keystore interface { EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } +var ( + promNumBroadcastedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_broadcasted_transactions", + Help: "Total number of successful broadcasted transactions.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs.", + }, []string{"chainID"}) + promNumNonceGaps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_nonce_gaps", + Help: "Total number of nonce gaps created that the transaction manager had to fill.", + }, []string{"chainID"}) +) + type Config struct { EIP1559 bool BlockTime time.Duration @@ -336,6 +353,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio } } + promNumBroadcastedTxs.WithLabelValues(t.chainID.String()).Add(float64(1)) return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, *tx.Nonce, attempt.Hash, address) } @@ -350,6 +368,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err } if len(confirmedTransactionIDs) > 0 || len(unconfirmedTransactionIDs) > 0 { + promNumConfirmedTxs.WithLabelValues(t.chainID.String()).Add(float64(len(confirmedTransactionIDs))) t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) } @@ -364,6 +383,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if tx == nil || *tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) + promNumNonceGaps.WithLabelValues(t.chainID.String()).Add(float64(1)) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { //nolint:revive //linter nonsense if !tx.IsPurgeable && t.stuckTxDetector != nil { From 79895a7d96516b143907237bc76b78497166cf81 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Thu, 28 Nov 2024 17:38:50 +0200 Subject: [PATCH 54/73] Add prom metrics --- .../evm/txm/docs/TRANSACTION_MANAGER_V2.md | 7 ++++++- core/chains/evm/txm/txm.go | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md index b7c350a2a17..638b8474f3d 100644 --- a/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md +++ b/core/chains/evm/txm/docs/TRANSACTION_MANAGER_V2.md @@ -5,4 +5,9 @@ - `EIP1559`: enables EIP-1559 mode. This means the transaction manager will create and broadcast Dynamic attempts. Set this to false to broadcast Legacy transactions. - `BlockTime`: controls the interval of the backfill loop. This dictates how frequently the transaction manager will check for confirmed transactions, rebroadcast stuck ones, and fill any nonce gaps. Transactions are getting confirmed only during new blocks so it's best if you set this to a value close to the block time. At least one RPC call is made during each BlockTime interval so the absolute minimum should be 2s. A small jitter is applied so the timeout won't be exactly the same each time. - `RetryBlockThreshold`: is the number of blocks to wait for a transaction stuck in the mempool before automatically rebroadcasting it with a new attempt. -- `EmptyTxLimitDefault`: sets default gas limit for empty transactions. Empty transactions are created in case there is a nonce gap or another stuck transaction in the mempool to fill a given nonce. These are empty transactions and they don't have any data or value. \ No newline at end of file +- `EmptyTxLimitDefault`: sets default gas limit for empty transactions. Empty transactions are created in case there is a nonce gap or another stuck transaction in the mempool to fill a given nonce. These are empty transactions and they don't have any data or value. + +## Metrics +- `txm_num_broadcasted_transactions` : total number of successful broadcasted transactions. +- `txm_num_confirmed_transactions` : total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs. +- `txm_num_nonce_gaps` : total number of nonce gaps created that the transaction manager had to fill. \ No newline at end of file diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 52c6084833d..3347a32e51c 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -9,6 +9,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/jpillora/backoff" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -63,6 +65,21 @@ type Keystore interface { EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } +var ( + promNumBroadcastedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_broadcasted_transactions", + Help: "Total number of successful broadcasted transactions.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs.", + }, []string{"chainID"}) + promNumNonceGaps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_nonce_gaps", + Help: "Total number of nonce gaps created that the transaction manager had to fill.", + }, []string{"chainID"}) +) + type Config struct { EIP1559 bool BlockTime time.Duration @@ -336,6 +353,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio } } + promNumBroadcastedTxs.WithLabelValues(t.chainID.String()).Add(float64(1)) return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, *tx.Nonce, attempt.Hash, address) } @@ -350,6 +368,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err } if len(confirmedTransactionIDs) > 0 || len(unconfirmedTransactionIDs) > 0 { + promNumConfirmedTxs.WithLabelValues(t.chainID.String()).Add(float64(len(confirmedTransactionIDs))) t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) } @@ -364,6 +383,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if tx == nil || *tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) + promNumNonceGaps.WithLabelValues(t.chainID.String()).Add(float64(1)) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { //nolint:revive //linter nonsense if !tx.IsPurgeable && t.stuckTxDetector != nil { From 7b0a1d1fedd221d251c2f76b32787d07cf8ffc6c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 2 Dec 2024 14:28:42 +0200 Subject: [PATCH 55/73] Add transaction confirmation metric --- core/chains/evm/txm/mocks/tx_store.go | 14 +++++------ core/chains/evm/txm/storage/inmemory_store.go | 13 ++++++---- .../evm/txm/storage/inmemory_store_manager.go | 4 ++-- .../evm/txm/storage/inmemory_store_test.go | 2 +- core/chains/evm/txm/txm.go | 24 +++++++++++++++---- core/chains/evm/txm/txm_test.go | 2 +- core/chains/evm/txm/types/transaction.go | 5 ++-- 7 files changed, 42 insertions(+), 22 deletions(-) diff --git a/core/chains/evm/txm/mocks/tx_store.go b/core/chains/evm/txm/mocks/tx_store.go index 866e095b1a1..86d4e29304f 100644 --- a/core/chains/evm/txm/mocks/tx_store.go +++ b/core/chains/evm/txm/mocks/tx_store.go @@ -358,24 +358,24 @@ func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) RunAndReturn } // MarkTransactionsConfirmed provides a mock function with given fields: _a0, _a1, _a2 -func (_m *TxStore) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]uint64, []uint64, error) { +func (_m *TxStore) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]*types.Transaction, []uint64, error) { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { panic("no return value specified for MarkTransactionsConfirmed") } - var r0 []uint64 + var r0 []*types.Transaction var r1 []uint64 var r2 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) ([]uint64, []uint64, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error)); ok { return rf(_a0, _a1, _a2) } - if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) []uint64); ok { + if rf, ok := ret.Get(0).(func(context.Context, uint64, common.Address) []*types.Transaction); ok { r0 = rf(_a0, _a1, _a2) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]uint64) + r0 = ret.Get(0).([]*types.Transaction) } } @@ -416,12 +416,12 @@ func (_c *TxStore_MarkTransactionsConfirmed_Call) Run(run func(_a0 context.Conte return _c } -func (_c *TxStore_MarkTransactionsConfirmed_Call) Return(_a0 []uint64, _a1 []uint64, _a2 error) *TxStore_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkTransactionsConfirmed_Call) Return(_a0 []*types.Transaction, _a1 []uint64, _a2 error) *TxStore_MarkTransactionsConfirmed_Call { _c.Call.Return(_a0, _a1, _a2) return _c } -func (_c *TxStore_MarkTransactionsConfirmed_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]uint64, []uint64, error)) *TxStore_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkTransactionsConfirmed_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error)) *TxStore_MarkTransactionsConfirmed_Call { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 06b6475b0cc..b030ddfec90 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -165,18 +165,18 @@ func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(latestNonce return } -func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, []uint64, error) { +func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]*types.Transaction, []uint64, error) { m.Lock() defer m.Unlock() - var confirmedTransactionIDs []uint64 + var confirmedTransactions []*types.Transaction for _, tx := range m.UnconfirmedTransactions { if tx.Nonce == nil { return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) } if *tx.Nonce < latestNonce { tx.State = types.TxConfirmed - confirmedTransactionIDs = append(confirmedTransactionIDs, tx.ID) + confirmedTransactions = append(confirmedTransactions, tx.DeepCopy()) m.ConfirmedTransactions[*tx.Nonce] = tx delete(m.UnconfirmedTransactions, *tx.Nonce) } @@ -201,9 +201,9 @@ func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]uint64, m.lggr.Debugf("Confirmed transactions map for address: %v reached max limit of: %d. Pruned 1/3 of the oldest confirmed transactions. TxIDs: %v", m.address, maxQueuedTransactions, prunedTxIDs) } - sort.Slice(confirmedTransactionIDs, func(i, j int) bool { return confirmedTransactionIDs[i] < confirmedTransactionIDs[j] }) + sort.Slice(confirmedTransactions, func(i, j int) bool { return confirmedTransactions[i].ID < confirmedTransactions[j].ID }) sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) - return confirmedTransactionIDs, unconfirmedTransactionIDs, nil + return confirmedTransactions, unconfirmedTransactionIDs, nil } func (m *InMemoryStore) MarkUnconfirmedTransactionPurgeable(nonce uint64) error { @@ -232,6 +232,9 @@ func (m *InMemoryStore) UpdateTransactionBroadcast(txID uint64, txNonce uint64, // Set the same time for both the tx and its attempt now := time.Now() unconfirmedTx.LastBroadcastAt = now + if unconfirmedTx.InitialBroadcastAt.IsZero() { + unconfirmedTx.InitialBroadcastAt = now + } a, err := unconfirmedTx.FindAttemptByHash(attemptHash) if err != nil { return err diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index d6cb1bfb906..5d0327de359 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -81,9 +81,9 @@ func (m *InMemoryStoreManager) FetchUnconfirmedTransactionAtNonceWithCount(_ con return nil, 0, fmt.Errorf(StoreNotFoundForAddress, fromAddress) } -func (m *InMemoryStoreManager) MarkTransactionsConfirmed(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxIDs []uint64, unconfirmedTxIDs []uint64, err error) { +func (m *InMemoryStoreManager) MarkTransactionsConfirmed(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxs []*types.Transaction, unconfirmedTxIDs []uint64, err error) { if store, exists := m.InMemoryStoreMap[fromAddress]; exists { - confirmedTxIDs, unconfirmedTxIDs, err = store.MarkTransactionsConfirmed(nonce) + confirmedTxs, unconfirmedTxIDs, err = store.MarkTransactionsConfirmed(nonce) return } return nil, nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index efc5111afcb..51b22a3631f 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -210,7 +210,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) - assert.Equal(t, ctxs[0], ctx1.ID) + assert.Equal(t, ctxs[0].ID, ctx1.ID) // Ensure order assert.Empty(t, utxs) }) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 3347a32e51c..ba836640933 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -38,7 +38,7 @@ type TxStore interface { CreateEmptyUnconfirmedTransaction(context.Context, common.Address, uint64, uint64) (*types.Transaction, error) CreateTransaction(context.Context, *types.TxRequest) (*types.Transaction, error) FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*types.Transaction, int, error) - MarkTransactionsConfirmed(context.Context, uint64, common.Address) ([]uint64, []uint64, error) + MarkTransactionsConfirmed(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error) MarkUnconfirmedTransactionPurgeable(context.Context, uint64, common.Address) error UpdateTransactionBroadcast(context.Context, uint64, uint64, common.Hash, common.Address) error UpdateUnstartedTransactionWithNonce(context.Context, common.Address, uint64) (*types.Transaction, error) @@ -78,6 +78,10 @@ var ( Name: "txm_num_nonce_gaps", Help: "Total number of nonce gaps created that the transaction manager had to fill.", }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "txm_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + }, []string{"chainID"}) ) type Config struct { @@ -363,12 +367,13 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err } - confirmedTransactionIDs, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(ctx, latestNonce, address) + confirmedTransactions, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(ctx, latestNonce, address) if err != nil { return false, err } - if len(confirmedTransactionIDs) > 0 || len(unconfirmedTransactionIDs) > 0 { - promNumConfirmedTxs.WithLabelValues(t.chainID.String()).Add(float64(len(confirmedTransactionIDs))) + if len(confirmedTransactions) > 0 || len(unconfirmedTransactionIDs) > 0 { + promNumConfirmedTxs.WithLabelValues(t.chainID.String()).Add(float64(len(confirmedTransactions))) + confirmedTransactionIDs := extractMetrics(confirmedTransactions, t.chainID) t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) } @@ -424,3 +429,14 @@ func (t *Txm) createAndSendEmptyTx(ctx context.Context, latestNonce uint64, addr } return t.createAndSendAttempt(ctx, tx, address) } + +func extractMetrics(txs []*types.Transaction, chainID *big.Int) []uint64 { + confirmedTxIDs := make([]uint64, 0, len(txs)) + for _, tx := range txs { + confirmedTxIDs = append(confirmedTxIDs, tx.ID) + if !tx.InitialBroadcastAt.IsZero() { + promTimeUntilTxConfirmed.WithLabelValues(chainID.String()).Observe(float64(time.Since(tx.InitialBroadcastAt))) + } + } + return confirmedTxIDs +} diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 93742924d58..ebdf9d42b1e 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -221,7 +221,7 @@ func TestBackfillTransactions(t *testing.T) { t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() - storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() + storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]*types.Transaction{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() bo, err := txm.backfillTransactions(ctx, address) require.Error(t, err) assert.False(t, bo) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 1b6110151f3..136b801a7e0 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -40,8 +40,9 @@ type Transaction struct { Data []byte SpecifiedGasLimit uint64 - CreatedAt time.Time - LastBroadcastAt time.Time + CreatedAt time.Time + InitialBroadcastAt time.Time + LastBroadcastAt time.Time State TxState IsPurgeable bool From 5f5524538ef1cf3781c8295b081686574f9119c2 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 2 Dec 2024 17:22:45 +0200 Subject: [PATCH 56/73] Improve logs --- core/chains/evm/txm/txm.go | 5 ++-- core/chains/evm/txm/types/transaction.go | 29 ++++++++++++++++-------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index ba836640933..833b3111ef4 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -341,7 +341,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio start := time.Now() txErr := t.client.SendTransaction(ctx, tx, attempt) tx.AttemptCount++ - t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) + t.lggr.Infow("Broadcasted attempt", "tx", tx.PrettyPrint(), "attempt", attempt.PrettyPrint(), "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore, t.setNonce, false); err != nil { return @@ -410,7 +410,8 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if tx.AttemptCount >= maxAllowedAttempts { return true, fmt.Errorf("reached max allowed attempts for txID: %d. TXM won't broadcast any more attempts."+ "If this error persists, it means the transaction won't be confirmed and the TXM needs to be restarted."+ - "Look for any error messages from previous attempts that may indicate why this happened, i.e. wallet is out of funds. Tx: %v", tx.ID, tx) + "Look for any error messages from previous broadcasted attempts that may indicate why this happened, i.e. wallet is out of funds. Tx: %v", tx.ID, + tx.PrettyPrintWithAttempts()) } if time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero() { diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 136b801a7e0..6ae7271ff8b 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -59,13 +59,24 @@ type Transaction struct { CallbackCompleted bool } -// func (t *Transaction) String() string { -// return fmt.Sprintf(`{"ID":%d, "IdempotencyKey":%v, "ChainID":%v, "Nonce":%d, "FromAddress":%v, "ToAddress":%v, "Value":%v, `+ -// `"Data":%v, "SpecifiedGasLimit":%d, "CreatedAt":%v, "LastBroadcastAt":%v, "State":%v, "IsPurgeable":%v, "AttemptCount":%d, `+ -// `"Meta":%v, "Subject":%v, "PipelineTaskRunID":%v, "MinConfirmations":%v, "SignalCallback":%v, "CallbackCompleted":%v`, -// t.ID, *t.IdempotencyKey, t.ChainID, t.Nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.LastBroadcastAt, -// t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject, t.PipelineTaskRunID, t.MinConfirmations, t.SignalCallback, t.CallbackCompleted) -// } +func (t *Transaction) PrettyPrint() string { + return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%d, FromAddress:%v, ToAddress:%v, Value:%v, `+ + `Data:%v, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ + `Meta:%v, Subject:%v}`, + t.ID, *t.IdempotencyKey, t.ChainID, t.Nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.InitialBroadcastAt, + t.LastBroadcastAt, t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject) +} + +func (t *Transaction) PrettyPrintWithAttempts() string { + attempts := " Attempts: [" + for _, a := range t.Attempts { + attempts += a.PrettyPrint() + ", " + } + attempts += "]" + + return t.PrettyPrint() + attempts +} + func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, error) { for _, a := range t.Attempts { if a.Hash == attemptHash { @@ -117,8 +128,8 @@ func (a *Attempt) DeepCopy() *Attempt { return &txCopy } -func (a *Attempt) String() string { - return fmt.Sprintf(`{"ID":%d, "TxID":%d, "Hash":%v, "Fee":%v, "GasLimit":%d, "Type":%v, "CreatedAt":%v, "BroadcastAt":%v}`, +func (a *Attempt) PrettyPrint() string { + return fmt.Sprintf(`{ID:%d, TxID:%d, Hash:%v, Fee:%v, GasLimit:%d, Type:%v, CreatedAt:%v, BroadcastAt:%v}`, a.ID, a.TxID, a.Hash, a.Fee, a.GasLimit, a.Type, a.CreatedAt, a.BroadcastAt) } From 6bc27a260489ed3f3d42143dc317e5b7afc79586 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 2 Dec 2024 18:55:38 +0200 Subject: [PATCH 57/73] Add Abandon support --- common/txmgr/txmgr.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index b05cd2e77ff..4d3288878fb 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -76,6 +76,7 @@ type TxmV2Wrapper[ ] interface { services.Service CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + Reset(addr ADDR, abandon bool) error } type reset struct { @@ -259,6 +260,11 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Reset(addr f := func() { if abandon { err = b.abandon(addr) + if b.txmv2wrapper != nil { + if err2 := b.txmv2wrapper.Reset(addr, abandon); err2 != nil { + b.logger.Error("failed to abandon transactions for dual broadcast", "err", err2) + } + } } } From 926bf67393f0cb7c1a0f75fa2c3b3e1c3e58dcd9 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 3 Dec 2024 14:33:17 +0200 Subject: [PATCH 58/73] Address feedback --- core/chains/evm/txm/mocks/tx_store.go | 22 +++++++-------- core/chains/evm/txm/orchestrator.go | 2 +- core/chains/evm/txm/storage/inmemory_store.go | 19 ++++++++++--- .../evm/txm/storage/inmemory_store_manager.go | 4 +-- .../evm/txm/storage/inmemory_store_test.go | 28 +++++++++++++++---- core/chains/evm/txm/txm.go | 14 ++++------ core/chains/evm/txm/txm_test.go | 5 ++-- core/chains/evm/txm/types/transaction.go | 12 ++++++-- 8 files changed, 71 insertions(+), 35 deletions(-) diff --git a/core/chains/evm/txm/mocks/tx_store.go b/core/chains/evm/txm/mocks/tx_store.go index 86d4e29304f..c164e43a6c5 100644 --- a/core/chains/evm/txm/mocks/tx_store.go +++ b/core/chains/evm/txm/mocks/tx_store.go @@ -357,12 +357,12 @@ func (_c *TxStore_FetchUnconfirmedTransactionAtNonceWithCount_Call) RunAndReturn return _c } -// MarkTransactionsConfirmed provides a mock function with given fields: _a0, _a1, _a2 -func (_m *TxStore) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]*types.Transaction, []uint64, error) { +// MarkConfirmedAndReorgedTransactions provides a mock function with given fields: _a0, _a1, _a2 +func (_m *TxStore) MarkConfirmedAndReorgedTransactions(_a0 context.Context, _a1 uint64, _a2 common.Address) ([]*types.Transaction, []uint64, error) { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { - panic("no return value specified for MarkTransactionsConfirmed") + panic("no return value specified for MarkConfirmedAndReorgedTransactions") } var r0 []*types.Transaction @@ -396,32 +396,32 @@ func (_m *TxStore) MarkTransactionsConfirmed(_a0 context.Context, _a1 uint64, _a return r0, r1, r2 } -// TxStore_MarkTransactionsConfirmed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkTransactionsConfirmed' -type TxStore_MarkTransactionsConfirmed_Call struct { +// TxStore_MarkConfirmedAndReorgedTransactions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkConfirmedAndReorgedTransactions' +type TxStore_MarkConfirmedAndReorgedTransactions_Call struct { *mock.Call } -// MarkTransactionsConfirmed is a helper method to define mock.On call +// MarkConfirmedAndReorgedTransactions is a helper method to define mock.On call // - _a0 context.Context // - _a1 uint64 // - _a2 common.Address -func (_e *TxStore_Expecter) MarkTransactionsConfirmed(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkTransactionsConfirmed_Call { - return &TxStore_MarkTransactionsConfirmed_Call{Call: _e.mock.On("MarkTransactionsConfirmed", _a0, _a1, _a2)} +func (_e *TxStore_Expecter) MarkConfirmedAndReorgedTransactions(_a0 interface{}, _a1 interface{}, _a2 interface{}) *TxStore_MarkConfirmedAndReorgedTransactions_Call { + return &TxStore_MarkConfirmedAndReorgedTransactions_Call{Call: _e.mock.On("MarkConfirmedAndReorgedTransactions", _a0, _a1, _a2)} } -func (_c *TxStore_MarkTransactionsConfirmed_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkConfirmedAndReorgedTransactions_Call) Run(run func(_a0 context.Context, _a1 uint64, _a2 common.Address)) *TxStore_MarkConfirmedAndReorgedTransactions_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(uint64), args[2].(common.Address)) }) return _c } -func (_c *TxStore_MarkTransactionsConfirmed_Call) Return(_a0 []*types.Transaction, _a1 []uint64, _a2 error) *TxStore_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkConfirmedAndReorgedTransactions_Call) Return(_a0 []*types.Transaction, _a1 []uint64, _a2 error) *TxStore_MarkConfirmedAndReorgedTransactions_Call { _c.Call.Return(_a0, _a1, _a2) return _c } -func (_c *TxStore_MarkTransactionsConfirmed_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error)) *TxStore_MarkTransactionsConfirmed_Call { +func (_c *TxStore_MarkConfirmedAndReorgedTransactions_Call) RunAndReturn(run func(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error)) *TxStore_MarkConfirmedAndReorgedTransactions_Call { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 694cce3d14b..e4b3437b7bc 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -171,7 +171,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, } if wrappedTx != nil { - o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", *wrappedTx.IdempotencyKey) + o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", wrappedTx.IdempotencyKey) } else { var pipelineTaskRunID uuid.NullUUID if request.PipelineTaskRunID != nil { diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index b030ddfec90..013f7844a44 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -15,8 +15,11 @@ import ( ) const ( + // maxQueuedTransactions is the max limit of UnstartedTransactions and ConfirmedTransactions structures. maxQueuedTransactions = 250 - pruneSubset = 3 + // pruneSubset controls the subset of confirmed transactions to prune when the structure reaches its max limit. + // i.e. if the value is 3 and the limit is 90, 30 transactions will be pruned. + pruneSubset = 3 ) type InMemoryStore struct { @@ -111,6 +114,10 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) } + if _, exists := m.Transactions[nonce]; exists { + return nil, fmt.Errorf("a tx with the same nonce already exists: %v", m.Transactions[nonce]) + } + m.UnconfirmedTransactions[nonce] = emptyTx m.Transactions[emptyTx.ID] = emptyTx @@ -165,7 +172,7 @@ func (m *InMemoryStore) FetchUnconfirmedTransactionAtNonceWithCount(latestNonce return } -func (m *InMemoryStore) MarkTransactionsConfirmed(latestNonce uint64) ([]*types.Transaction, []uint64, error) { +func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) ([]*types.Transaction, []uint64, error) { m.Lock() defer m.Unlock() @@ -257,6 +264,10 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) } + if _, exists := m.Transactions[nonce]; exists { + return nil, fmt.Errorf("a tx with the same nonce already exists: %v", m.Transactions[nonce]) + } + tx := m.UnstartedTransactions[0] tx.Nonce = &nonce tx.State = types.TxUnconfirmed @@ -318,8 +329,8 @@ func (m *InMemoryStore) MarkTxFatal(*types.Transaction) error { // Orchestrator func (m *InMemoryStore) FindTxWithIdempotencyKey(idempotencyKey *string) *types.Transaction { - m.Lock() - defer m.Unlock() + m.RLock() + defer m.RUnlock() if idempotencyKey != nil { for _, tx := range m.Transactions { diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index 5d0327de359..7e0871c3a7f 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -81,9 +81,9 @@ func (m *InMemoryStoreManager) FetchUnconfirmedTransactionAtNonceWithCount(_ con return nil, 0, fmt.Errorf(StoreNotFoundForAddress, fromAddress) } -func (m *InMemoryStoreManager) MarkTransactionsConfirmed(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxs []*types.Transaction, unconfirmedTxIDs []uint64, err error) { +func (m *InMemoryStoreManager) MarkConfirmedAndReorgedTransactions(_ context.Context, nonce uint64, fromAddress common.Address) (confirmedTxs []*types.Transaction, unconfirmedTxIDs []uint64, err error) { if store, exists := m.InMemoryStoreMap[fromAddress]; exists { - confirmedTxs, unconfirmedTxIDs, err = store.MarkTransactionsConfirmed(nonce) + confirmedTxs, unconfirmedTxIDs, err = store.MarkConfirmedAndReorgedTransactions(nonce) return } return nil, nil, fmt.Errorf(StoreNotFoundForAddress, fromAddress) diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index 51b22a3631f..b2a3a068018 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -192,7 +192,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { t.Run("returns 0 if there are no transactions", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - un, cn, err := m.MarkTransactionsConfirmed(100) + un, cn, err := m.MarkConfirmedAndReorgedTransactions(100) require.NoError(t, err) assert.Empty(t, un) assert.Empty(t, cn) @@ -206,7 +206,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { ctx2, err := insertUnconfirmedTransaction(m, 1) require.NoError(t, err) - ctxs, utxs, err := m.MarkTransactionsConfirmed(1) + ctxs, utxs, err := m.MarkConfirmedAndReorgedTransactions(1) require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) @@ -214,6 +214,22 @@ func TestMarkTransactionsConfirmed(t *testing.T) { assert.Empty(t, utxs) }) + t.Run("state remains the same if nonce didn't change", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + ctx1, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + + ctx2, err := insertUnconfirmedTransaction(m, 1) + require.NoError(t, err) + + ctxs, utxs, err := m.MarkConfirmedAndReorgedTransactions(1) + require.NoError(t, err) + assert.Equal(t, types.TxConfirmed, ctx1.State) + assert.Equal(t, types.TxUnconfirmed, ctx2.State) + assert.Empty(t, ctxs) + assert.Empty(t, utxs) + }) + t.Run("unconfirms transaction with nonce equal to or higher than the latest", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) ctx1, err := insertConfirmedTransaction(m, 0) @@ -222,7 +238,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { ctx2, err := insertConfirmedTransaction(m, 1) require.NoError(t, err) - ctxs, utxs, err := m.MarkTransactionsConfirmed(1) + ctxs, utxs, err := m.MarkConfirmedAndReorgedTransactions(1) require.NoError(t, err) assert.Equal(t, types.TxConfirmed, ctx1.State) assert.Equal(t, types.TxUnconfirmed, ctx2.State) @@ -237,7 +253,7 @@ func TestMarkTransactionsConfirmed(t *testing.T) { require.NoError(t, err) } assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions) - _, _, err := m.MarkTransactionsConfirmed(maxQueuedTransactions) + _, _, err := m.MarkConfirmedAndReorgedTransactions(maxQueuedTransactions) require.NoError(t, err) assert.Len(t, m.ConfirmedTransactions, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset)) }) @@ -294,6 +310,7 @@ func TestUpdateTransactionBroadcast(t *testing.T) { require.NoError(t, m.UpdateTransactionBroadcast(0, nonce, hash)) assert.False(t, tx.LastBroadcastAt.IsZero()) assert.False(t, attempt.BroadcastAt.IsZero()) + assert.False(t, tx.InitialBroadcastAt.IsZero()) }) } @@ -308,7 +325,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { assert.Nil(t, tx) }) - t.Run("fails if there is already another unstarted transaction with the same nonce", func(t *testing.T) { + t.Run("fails if there is already another unconfirmed transaction with the same nonce", func(t *testing.T) { var nonce uint64 m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) insertUnstartedTransaction(m) @@ -328,6 +345,7 @@ func TestUpdateUnstartedTransactionWithNonce(t *testing.T) { require.NoError(t, err) assert.Equal(t, nonce, *tx.Nonce) assert.Equal(t, types.TxUnconfirmed, tx.State) + assert.Empty(t, m.UnstartedTransactions) }) } diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 833b3111ef4..340bd222d58 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -38,7 +38,7 @@ type TxStore interface { CreateEmptyUnconfirmedTransaction(context.Context, common.Address, uint64, uint64) (*types.Transaction, error) CreateTransaction(context.Context, *types.TxRequest) (*types.Transaction, error) FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*types.Transaction, int, error) - MarkTransactionsConfirmed(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error) + MarkConfirmedAndReorgedTransactions(context.Context, uint64, common.Address) ([]*types.Transaction, []uint64, error) MarkUnconfirmedTransactionPurgeable(context.Context, uint64, common.Address) error UpdateTransactionBroadcast(context.Context, uint64, uint64, common.Hash, common.Address) error UpdateUnstartedTransactionWithNonce(context.Context, common.Address, uint64) (*types.Transaction, error) @@ -279,7 +279,7 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) return false, err } - // Optimistically send up to 1/3 of the maxInFlightTransactions. After that threshold, broadcast more cautiously + // Optimistically send up to 1/maxInFlightSubset of the maxInFlightTransactions. After that threshold, broadcast more cautiously // by checking the pending nonce so no more than maxInFlightTransactions/3 can get stuck simultaneously i.e. due // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. @@ -300,17 +300,15 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) } } - tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(ctx, address, t.getNonce(address)) + nonce := t.getNonce(address) + tx, err := t.txStore.UpdateUnstartedTransactionWithNonce(ctx, address, nonce) if err != nil { return false, err } if tx == nil { return false, nil } - nonce := t.getNonce(address) - tx.Nonce = &nonce - t.setNonce(address, *tx.Nonce+1) - tx.State = types.TxUnconfirmed + t.setNonce(address, nonce+1) if err := t.createAndSendAttempt(ctx, tx, address); err != nil { return true, err @@ -367,7 +365,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err } - confirmedTransactions, unconfirmedTransactionIDs, err := t.txStore.MarkTransactionsConfirmed(ctx, latestNonce, address) + confirmedTransactions, unconfirmedTransactionIDs, err := t.txStore.MarkConfirmedAndReorgedTransactions(ctx, latestNonce, address) if err != nil { return false, err } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index ebdf9d42b1e..94710266104 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -218,10 +218,11 @@ func TestBackfillTransactions(t *testing.T) { require.ErrorContains(t, err, "latest nonce fail") }) - t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { + t.Run("fails if MarkConfirmedAndReorgedTransactions fails", func(t *testing.T) { txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() - storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]*types.Transaction{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() + storage.On("MarkConfirmedAndReorgedTransactions", mock.Anything, mock.Anything, address). + Return([]*types.Transaction{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() bo, err := txm.backfillTransactions(ctx, address) require.Error(t, err) assert.False(t, bo) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 6ae7271ff8b..1b0a91646b5 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "math/big" + "strconv" "time" "github.com/google/uuid" @@ -60,10 +61,17 @@ type Transaction struct { } func (t *Transaction) PrettyPrint() string { - return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%d, FromAddress:%v, ToAddress:%v, Value:%v, `+ + idk, nonce := "", "" + if t.IdempotencyKey != nil { + idk = *t.IdempotencyKey + } + if t.Nonce != nil { + nonce = strconv.FormatUint(*t.Nonce, 10) + } + return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%s, FromAddress:%v, ToAddress:%v, Value:%v, `+ `Data:%v, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ `Meta:%v, Subject:%v}`, - t.ID, *t.IdempotencyKey, t.ChainID, t.Nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.InitialBroadcastAt, + t.ID, idk, t.ChainID, nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.InitialBroadcastAt, t.LastBroadcastAt, t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject) } From a4eacaa6c19d6653f931727f30c53c696a7a3e9a Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 4 Dec 2024 14:08:14 +0200 Subject: [PATCH 59/73] Update tests --- core/chains/evm/txm/txm_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 94710266104..8410271f55c 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -85,6 +85,7 @@ func TestTrigger(t *testing.T) { // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() servicetest.Run(t, txm) + txm.Trigger(address) }) } @@ -195,6 +196,7 @@ func TestBroadcastTransaction(t *testing.T) { var zeroTime time.Time assert.Greater(t, tx.LastBroadcastAt, zeroTime) assert.Greater(t, tx.Attempts[0].BroadcastAt, zeroTime) + assert.Greater(t, tx.InitialBroadcastAt, zeroTime) }) } From ecfdfb9410aab2f1b45ab9313ca5e783d6ed5859 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 4 Dec 2024 14:27:22 +0200 Subject: [PATCH 60/73] Fix orchestrator log --- core/chains/evm/txm/orchestrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index e4b3437b7bc..694cce3d14b 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -171,7 +171,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, } if wrappedTx != nil { - o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", wrappedTx.IdempotencyKey) + o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", *wrappedTx.IdempotencyKey) } else { var pipelineTaskRunID uuid.NullUUID if request.PipelineTaskRunID != nil { From 0ff44e965fc57d0be644544aab478addaf9ab78c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 4 Dec 2024 15:23:18 +0200 Subject: [PATCH 61/73] Fix Nonce log --- core/chains/evm/txm/txm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 340bd222d58..4b1d85a5545 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -350,7 +350,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio return err } if pendingNonce <= *tx.Nonce { - t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, tx.Nonce) + t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, *tx.Nonce) return nil } } From cecb7f65dc243143380c77e11f30c0a2170b0d79 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 4 Dec 2024 18:15:24 +0200 Subject: [PATCH 62/73] Improvements --- core/chains/evm/txm/txm.go | 37 +++++++++++++++++++----- core/chains/evm/txm/txm_test.go | 22 +++++++++----- core/chains/evm/txm/types/transaction.go | 2 +- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 4b1d85a5545..1c0cca07b76 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -20,10 +20,12 @@ import ( ) const ( - broadcastInterval time.Duration = 30 * time.Second - maxInFlightTransactions int = 16 - maxInFlightSubset int = 3 - maxAllowedAttempts uint16 = 10 + broadcastInterval time.Duration = 30 * time.Second + maxInFlightTransactions int = 16 + maxInFlightSubset int = 3 + maxAllowedAttempts uint16 = 10 + pendingNonceDefaultTimeout time.Duration = 30 * time.Second + pendingNonceRecheckInterval time.Duration = 2 * time.Second ) type Client interface { @@ -72,7 +74,7 @@ var ( }, []string{"chainID"}) promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "txm_num_confirmed_transactions", - Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs.", + Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs or when filling the nonce for untracked transactions.", }, []string{"chainID"}) promNumNonceGaps = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "txm_num_nonce_gaps", @@ -135,7 +137,7 @@ func (t *Txm) Start(ctx context.Context) error { return err } for _, address := range addresses { - err := t.startAddress(address) + err := t.startAddress(ctx, address) if err != nil { return err } @@ -144,10 +146,10 @@ func (t *Txm) Start(ctx context.Context) error { }) } -func (t *Txm) startAddress(address common.Address) error { +func (t *Txm) startAddress(ctx context.Context, address common.Address) error { triggerCh := make(chan struct{}, 1) t.triggerCh[address] = triggerCh - pendingNonce, err := t.client.PendingNonceAt(context.TODO(), address) + pendingNonce, err := t.pollForPendingNonce(ctx, address) if err != nil { return err } @@ -159,6 +161,24 @@ func (t *Txm) startAddress(address common.Address) error { return nil } +func (t *Txm) pollForPendingNonce(ctx context.Context, address common.Address) (pendingNonce uint64, err error) { + ctxWithTimeout, cancel := context.WithTimeout(ctx, pendingNonceDefaultTimeout) + defer cancel() + for { + pendingNonce, err = t.client.PendingNonceAt(ctxWithTimeout, address) + if err != nil { + t.lggr.Errorw("Error when fetching initial pending nonce", "address", address, "err", err) + select { + case <-time.After(pendingNonceRecheckInterval): + case <-ctx.Done(): + return 0, context.Cause(ctx) + } + continue + } + return pendingNonce, nil + } +} + func (t *Txm) Close() error { return t.StopOnce("Txm", func() error { close(t.stopCh) @@ -188,6 +208,7 @@ func (t *Txm) Trigger(address common.Address) { } func (t *Txm) Abandon(address common.Address) error { + t.lggr.Infof("Dropping unstarted and unconfirmed transactions for address: %v", address) return t.txStore.AbandonPendingTransactions(context.TODO(), address) } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 8410271f55c..22f700d13e6 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -28,21 +28,28 @@ func TestLifecycle(t *testing.T) { client := mocks.NewClient(t) ab := mocks.NewAttemptBuilder(t) - config := Config{BlockTime: 10 * time.Millisecond} address1 := testutils.NewAddress() address2 := testutils.NewAddress() assert.NotEqual(t, address1, address2) addresses := []common.Address{address1, address2} keystore := mocks.NewKeystore(t) - keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return(addresses, nil) - t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, nil, config, keystore) + t.Run("retries if initial pending nonce call fails", func(t *testing.T) { + config := Config{BlockTime: 1 * time.Minute} + txStore := mocks.NewTxStore(t) + txStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 20, nil) + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address1}, nil).Once() + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + txm := NewTxm(lggr, testutils.FixtureChainID, client, nil, txStore, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() - require.Error(t, txm.Start(tests.Context(t))) + client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), nil).Once() + require.NoError(t, txm.Start(tests.Context(t))) + tests.AssertLogEventually(t, observedLogs, "Error when fetching initial pending nonce") }) t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { + config := Config{BlockTime: 200 * time.Millisecond} + keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return(addresses, nil).Once() lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(addresses...)) @@ -52,8 +59,8 @@ func TestLifecycle(t *testing.T) { client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() client.On("PendingNonceAt", mock.Anything, address2).Return(nonce, nil).Once() // backfill loop (may or may not be executed multiple times) - client.On("NonceAt", mock.Anything, address1, mock.Anything).Return(nonce, nil) - client.On("NonceAt", mock.Anything, address2, mock.Anything).Return(nonce, nil) + client.On("NonceAt", mock.Anything, address1, mock.Anything).Return(nonce, nil).Maybe() + client.On("NonceAt", mock.Anything, address2, mock.Anything).Return(nonce, nil).Maybe() servicetest.Run(t, txm) tests.AssertLogEventually(t, observedLogs, "Backfill time elapsed") @@ -170,6 +177,7 @@ func TestBroadcastTransaction(t *testing.T) { txm.setNonce(address, 8) IDK := "IDK" txRequest := &types.TxRequest{ + Data: []byte{100}, IdempotencyKey: &IDK, ChainID: testutils.FixtureChainID, FromAddress: address, diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 1b0a91646b5..0c1d2861daa 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -69,7 +69,7 @@ func (t *Transaction) PrettyPrint() string { nonce = strconv.FormatUint(*t.Nonce, 10) } return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%s, FromAddress:%v, ToAddress:%v, Value:%v, `+ - `Data:%v, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ + `Data:%s, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ `Meta:%v, Subject:%v}`, t.ID, idk, t.ChainID, nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.InitialBroadcastAt, t.LastBroadcastAt, t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject) From 103f0d8b026c40e93bb64f9608e2447236a3d156 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Wed, 4 Dec 2024 18:39:17 +0200 Subject: [PATCH 63/73] Update tests --- core/chains/evm/txm/txm.go | 2 +- core/chains/evm/txm/txm_test.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 1c0cca07b76..c37099d3783 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -25,7 +25,7 @@ const ( maxInFlightSubset int = 3 maxAllowedAttempts uint16 = 10 pendingNonceDefaultTimeout time.Duration = 30 * time.Second - pendingNonceRecheckInterval time.Duration = 2 * time.Second + pendingNonceRecheckInterval time.Duration = 1 * time.Second ) type Client interface { diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 22f700d13e6..fef90f9c344 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -35,11 +35,10 @@ func TestLifecycle(t *testing.T) { keystore := mocks.NewKeystore(t) t.Run("retries if initial pending nonce call fails", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) config := Config{BlockTime: 1 * time.Minute} - txStore := mocks.NewTxStore(t) - txStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 20, nil) + txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address1}, nil).Once() - lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) txm := NewTxm(lggr, testutils.FixtureChainID, client, nil, txStore, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), nil).Once() From cc0b9eb35b38ed5b8671870d9408aca7aa8290c2 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 6 Dec 2024 16:55:21 +0200 Subject: [PATCH 64/73] Add fixes --- .../evm/txm/clientwrappers/dual_broadcast_client.go | 4 ++-- core/chains/evm/txm/storage/inmemory_store.go | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go index 0bbd2530765..77111c2d48c 100644 --- a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -42,7 +42,7 @@ func (d *DualBroadcastClient) NonceAt(ctx context.Context, address common.Addres } func (d *DualBroadcastClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { - body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":["%s","pending"]}`, address.String())) + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":["%s","pending"], "id":1}`, address.String())) response, err := d.signAndPostMessage(ctx, address, body, "") if err != nil { return 0, err @@ -70,7 +70,7 @@ func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Tra if meta.DualBroadcastParams != nil { params = *meta.DualBroadcastParams } - body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, hexutil.Encode(data))) + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"], "id":1}`, hexutil.Encode(data))) if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, params); err != nil { return err } diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 013f7844a44..49bb54e98af 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -114,8 +114,8 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) } - if _, exists := m.Transactions[nonce]; exists { - return nil, fmt.Errorf("a tx with the same nonce already exists: %v", m.Transactions[nonce]) + if _, exists := m.ConfirmedTransactions[nonce]; exists { + return nil, fmt.Errorf("a confirmed tx with the same nonce already exists: %v", m.ConfirmedTransactions[nonce]) } m.UnconfirmedTransactions[nonce] = emptyTx @@ -264,10 +264,6 @@ func (m *InMemoryStore) UpdateUnstartedTransactionWithNonce(nonce uint64) (*type return nil, fmt.Errorf("an unconfirmed tx with the same nonce already exists: %v", m.UnconfirmedTransactions[nonce]) } - if _, exists := m.Transactions[nonce]; exists { - return nil, fmt.Errorf("a tx with the same nonce already exists: %v", m.Transactions[nonce]) - } - tx := m.UnstartedTransactions[0] tx.Nonce = &nonce tx.State = types.TxUnconfirmed From 6a17e0a26f0b2faea74017689b2ead24fb85ea0d Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 6 Dec 2024 18:25:03 +0200 Subject: [PATCH 65/73] Improvements --- core/chains/evm/txm/attempt_builder.go | 1 + core/chains/evm/txm/orchestrator.go | 18 ++++---- core/chains/evm/txm/storage/inmemory_store.go | 43 ++++++++++--------- .../evm/txm/storage/inmemory_store_manager.go | 9 ++-- .../evm/txm/storage/inmemory_store_test.go | 6 +-- core/chains/evm/txm/stuck_tx_detector.go | 2 +- core/chains/evm/txm/txm.go | 14 +++--- core/chains/evm/txm/txm_test.go | 14 +++--- core/chains/evm/txm/types/transaction.go | 30 ++++++------- 9 files changed, 71 insertions(+), 66 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index fd23bf867e7..6be849c8dcb 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -114,6 +114,7 @@ func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transac Fee: gas.EvmFee{GasPrice: gasPrice}, Hash: signedTx.Hash(), GasLimit: estimatedGasLimit, + Type: evmtypes.LegacyTxType, SignedTransaction: signedTx, } diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 694cce3d14b..914a14f981b 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -30,7 +30,7 @@ import ( type OrchestratorTxStore interface { Add(addresses ...common.Address) error FetchUnconfirmedTransactionAtNonceWithCount(context.Context, uint64, common.Address) (*txmtypes.Transaction, int, error) - FindTxWithIdempotencyKey(context.Context, *string) (*txmtypes.Transaction, error) + FindTxWithIdempotencyKey(context.Context, string) (*txmtypes.Transaction, error) } type OrchestratorKeystore interface { @@ -116,12 +116,12 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) Close() (merr error) { merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop ForwarderManager: %w", err)) } } - if err := o.attemptBuilder.Close(); err != nil { - merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop AttemptBuilder: %w", err)) - } if err := o.txm.Close(); err != nil { merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop Txm: %w", err)) } + if err := o.attemptBuilder.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Orchestrator failed to stop AttemptBuilder: %w", err)) + } return merr }) } @@ -165,9 +165,11 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) OnNewLongestChain(ctx context.Context, func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, request txmgrtypes.TxRequest[common.Address, common.Hash]) (tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { var wrappedTx *txmtypes.Transaction - wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(ctx, request.IdempotencyKey) - if err != nil { - return + if request.IdempotencyKey != nil { + wrappedTx, err = o.txStore.FindTxWithIdempotencyKey(ctx, *request.IdempotencyKey) + if err != nil { + return + } } if wrappedTx != nil { @@ -317,7 +319,7 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) GetForwarderForEOAOCR2Feeds(ctx context func (o *Orchestrator[BLOCK_HASH, HEAD]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { // Loads attempts and receipts in the transaction - tx, err := o.txStore.FindTxWithIdempotencyKey(ctx, &transactionID) + tx, err := o.txStore.FindTxWithIdempotencyKey(ctx, transactionID) if err != nil || tx == nil { return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s: %w", transactionID, err) } diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 49bb54e98af..6451f6c3068 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -42,6 +42,7 @@ func NewInMemoryStore(lggr logger.Logger, address common.Address, chainID *big.I lggr: logger.Named(lggr, "InMemoryStore"), address: address, chainID: chainID, + UnstartedTransactions: make([]*types.Transaction, 0, maxQueuedTransactions), UnconfirmedTransactions: make(map[uint64]*types.Transaction), ConfirmedTransactions: make(map[uint64]*types.Transaction), Transactions: make(map[uint64]*types.Transaction), @@ -97,7 +98,6 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit m.Lock() defer m.Unlock() - m.txIDCount++ emptyTx := &types.Transaction{ ID: m.txIDCount, ChainID: m.chainID, @@ -118,6 +118,7 @@ func (m *InMemoryStore) CreateEmptyUnconfirmedTransaction(nonce uint64, gasLimit return nil, fmt.Errorf("a confirmed tx with the same nonce already exists: %v", m.ConfirmedTransactions[nonce]) } + m.txIDCount++ m.UnconfirmedTransactions[nonce] = emptyTx m.Transactions[emptyTx.ID] = emptyTx @@ -128,8 +129,6 @@ func (m *InMemoryStore) CreateTransaction(txRequest *types.TxRequest) *types.Tra m.Lock() defer m.Unlock() - m.txIDCount++ - tx := &types.Transaction{ ID: m.txIDCount, IdempotencyKey: txRequest.IdempotencyKey, @@ -147,13 +146,17 @@ func (m *InMemoryStore) CreateTransaction(txRequest *types.TxRequest) *types.Tra SignalCallback: txRequest.SignalCallback, } - if len(m.UnstartedTransactions) == maxQueuedTransactions { - m.lggr.Warnf("Unstarted transactions queue for address: %v reached max limit of: %d. Dropping oldest transaction: %v.", - m.address, maxQueuedTransactions, m.UnstartedTransactions[0]) - delete(m.Transactions, m.UnstartedTransactions[0].ID) - m.UnstartedTransactions = m.UnstartedTransactions[1:maxQueuedTransactions] + uLen := len(m.UnstartedTransactions) + if uLen >= maxQueuedTransactions { + m.lggr.Warnw(fmt.Sprintf("Unstarted transactions queue for address: %v reached max limit of: %d. Dropping oldest transactions", m.address, maxQueuedTransactions), + "txs", m.UnstartedTransactions[0:uLen-maxQueuedTransactions+1]) // need to make room for the new tx + for _, tx := range m.UnstartedTransactions[0 : uLen-maxQueuedTransactions+1] { + delete(m.Transactions, tx.ID) + } + m.UnstartedTransactions = m.UnstartedTransactions[uLen-maxQueuedTransactions+1:] } + m.txIDCount++ txCopy := tx.DeepCopy() m.Transactions[txCopy.ID] = txCopy m.UnstartedTransactions = append(m.UnstartedTransactions, txCopy) @@ -196,7 +199,7 @@ func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) } if *tx.Nonce >= latestNonce { tx.State = types.TxUnconfirmed - tx.LastBroadcastAt = time.Time{} // Mark reorged transaction as if it wasn't broadcasted before + tx.LastBroadcastAt = nil // Mark reorged transaction as if it wasn't broadcasted before unconfirmedTransactionIDs = append(unconfirmedTransactionIDs, tx.ID) m.UnconfirmedTransactions[*tx.Nonce] = tx delete(m.ConfirmedTransactions, *tx.Nonce) @@ -205,8 +208,8 @@ func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) if len(m.ConfirmedTransactions) >= maxQueuedTransactions { prunedTxIDs := m.pruneConfirmedTransactions() - m.lggr.Debugf("Confirmed transactions map for address: %v reached max limit of: %d. Pruned 1/3 of the oldest confirmed transactions. TxIDs: %v", - m.address, maxQueuedTransactions, prunedTxIDs) + m.lggr.Debugf("Confirmed transactions map for address: %v reached max limit of: %d. Pruned 1/%d of the oldest confirmed transactions. TxIDs: %v", + m.address, maxQueuedTransactions, pruneSubset, prunedTxIDs) } sort.Slice(confirmedTransactions, func(i, j int) bool { return confirmedTransactions[i].ID < confirmedTransactions[j].ID }) sort.Slice(unconfirmedTransactionIDs, func(i, j int) bool { return unconfirmedTransactionIDs[i] < unconfirmedTransactionIDs[j] }) @@ -238,15 +241,15 @@ func (m *InMemoryStore) UpdateTransactionBroadcast(txID uint64, txNonce uint64, // Set the same time for both the tx and its attempt now := time.Now() - unconfirmedTx.LastBroadcastAt = now - if unconfirmedTx.InitialBroadcastAt.IsZero() { - unconfirmedTx.InitialBroadcastAt = now + unconfirmedTx.LastBroadcastAt = &now + if unconfirmedTx.InitialBroadcastAt == nil { + unconfirmedTx.InitialBroadcastAt = &now } a, err := unconfirmedTx.FindAttemptByHash(attemptHash) if err != nil { return err } - a.BroadcastAt = now + a.BroadcastAt = &now return nil } @@ -324,15 +327,13 @@ func (m *InMemoryStore) MarkTxFatal(*types.Transaction) error { } // Orchestrator -func (m *InMemoryStore) FindTxWithIdempotencyKey(idempotencyKey *string) *types.Transaction { +func (m *InMemoryStore) FindTxWithIdempotencyKey(idempotencyKey string) *types.Transaction { m.RLock() defer m.RUnlock() - if idempotencyKey != nil { - for _, tx := range m.Transactions { - if tx.IdempotencyKey != nil && tx.IdempotencyKey == idempotencyKey { - return tx.DeepCopy() - } + for _, tx := range m.Transactions { + if tx.IdempotencyKey != nil && *tx.IdempotencyKey == idempotencyKey { + return tx.DeepCopy() } } diff --git a/core/chains/evm/txm/storage/inmemory_store_manager.go b/core/chains/evm/txm/storage/inmemory_store_manager.go index 7e0871c3a7f..86abaf4b7cc 100644 --- a/core/chains/evm/txm/storage/inmemory_store_manager.go +++ b/core/chains/evm/txm/storage/inmemory_store_manager.go @@ -6,6 +6,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "go.uber.org/multierr" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" @@ -35,14 +36,14 @@ func (m *InMemoryStoreManager) AbandonPendingTransactions(_ context.Context, fro return fmt.Errorf(StoreNotFoundForAddress, fromAddress) } -func (m *InMemoryStoreManager) Add(addresses ...common.Address) error { +func (m *InMemoryStoreManager) Add(addresses ...common.Address) (err error) { for _, address := range addresses { if _, exists := m.InMemoryStoreMap[address]; exists { - return fmt.Errorf("address %v already exists in store manager", address) + err = multierr.Append(err, fmt.Errorf("address %v already exists in store manager", address)) } m.InMemoryStoreMap[address] = NewInMemoryStore(m.lggr, address, m.chainID) } - return nil + return } func (m *InMemoryStoreManager) AppendAttemptToTransaction(_ context.Context, txNonce uint64, fromAddress common.Address, attempt *types.Attempt) error { @@ -124,7 +125,7 @@ func (m *InMemoryStoreManager) MarkTxFatal(_ context.Context, tx *types.Transact return fmt.Errorf(StoreNotFoundForAddress, fromAddress) } -func (m *InMemoryStoreManager) FindTxWithIdempotencyKey(_ context.Context, idempotencyKey *string) (*types.Transaction, error) { +func (m *InMemoryStoreManager) FindTxWithIdempotencyKey(_ context.Context, idempotencyKey string) (*types.Transaction, error) { for _, store := range m.InMemoryStoreMap { tx := store.FindTxWithIdempotencyKey(idempotencyKey) if tx != nil { diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index b2a3a068018..db1f54d4093 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -138,11 +138,11 @@ func TestCreateTransaction(t *testing.T) { txR1 := &types.TxRequest{} txR2 := &types.TxRequest{} tx1 := m.CreateTransaction(txR1) - assert.Equal(t, uint64(1), tx1.ID) + assert.Equal(t, uint64(0), tx1.ID) assert.LessOrEqual(t, now, tx1.CreatedAt) tx2 := m.CreateTransaction(txR2) - assert.Equal(t, uint64(2), tx2.ID) + assert.Equal(t, uint64(1), tx2.ID) assert.LessOrEqual(t, now, tx2.CreatedAt) assert.Equal(t, 2, m.CountUnstartedTransactions()) @@ -151,7 +151,7 @@ func TestCreateTransaction(t *testing.T) { t.Run("prunes oldest unstarted transactions if limit is reached", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) overshot := 5 - for i := 1; i < maxQueuedTransactions+overshot; i++ { + for i := 0; i < maxQueuedTransactions+overshot; i++ { r := &types.TxRequest{} tx := m.CreateTransaction(r) //nolint:gosec // this won't overflow diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go index 68d8caf0ed1..87b78d5cc04 100644 --- a/core/chains/evm/txm/stuck_tx_detector.go +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -52,7 +52,7 @@ func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types. func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) - if time.Since(tx.LastBroadcastAt) > threshold && !tx.LastBroadcastAt.IsZero() { + if tx.LastBroadcastAt != nil && time.Since(*tx.LastBroadcastAt) > threshold { s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", tx.ID, tx.LastBroadcastAt, threshold) return true diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index c37099d3783..3ac899efd80 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -22,7 +22,7 @@ import ( const ( broadcastInterval time.Duration = 30 * time.Second maxInFlightTransactions int = 16 - maxInFlightSubset int = 3 + maxInFlightSubset int = 5 maxAllowedAttempts uint16 = 10 pendingNonceDefaultTimeout time.Duration = 30 * time.Second pendingNonceRecheckInterval time.Duration = 1 * time.Second @@ -300,11 +300,11 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) return false, err } - // Optimistically send up to 1/maxInFlightSubset of the maxInFlightTransactions. After that threshold, broadcast more cautiously - // by checking the pending nonce so no more than maxInFlightTransactions/3 can get stuck simultaneously i.e. due + // Optimistically send up to maxInFlightSubset of the maxInFlightTransactions. After that threshold, broadcast more cautiously + // by checking the pending nonce so no more than maxInFlightSubset can get stuck simultaneously i.e. due // to insufficient balance. We're making this trade-off to avoid storing stuck transactions and making unnecessary // RPC calls. The upper limit is always maxInFlightTransactions regardless of the pending nonce. - if unconfirmedCount >= maxInFlightTransactions/maxInFlightSubset { + if unconfirmedCount >= maxInFlightSubset { if unconfirmedCount > maxInFlightTransactions { t.lggr.Warnf("Reached transaction limit: %d for unconfirmed transactions", maxInFlightTransactions) return true, nil @@ -433,7 +433,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) tx.PrettyPrintWithAttempts()) } - if time.Since(tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) || tx.LastBroadcastAt.IsZero() { + if tx.LastBroadcastAt == nil || time.Since(*tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) { // TODO: add optional graceful bumping strategy t.lggr.Info("Rebroadcasting attempt for txID: ", tx.ID) return false, t.createAndSendAttempt(ctx, tx, address) @@ -454,8 +454,8 @@ func extractMetrics(txs []*types.Transaction, chainID *big.Int) []uint64 { confirmedTxIDs := make([]uint64, 0, len(txs)) for _, tx := range txs { confirmedTxIDs = append(confirmedTxIDs, tx.ID) - if !tx.InitialBroadcastAt.IsZero() { - promTimeUntilTxConfirmed.WithLabelValues(chainID.String()).Observe(float64(time.Since(tx.InitialBroadcastAt))) + if tx.InitialBroadcastAt != nil { + promTimeUntilTxConfirmed.WithLabelValues(chainID.String()).Observe(float64(time.Since(*tx.InitialBroadcastAt))) } } return confirmedTxIDs diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index fef90f9c344..41c6c4d0741 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -126,12 +126,12 @@ func TestBroadcastTransaction(t *testing.T) { tests.AssertLogEventually(t, observedLogs, "Reached transaction limit") }) - t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { + t.Run("checks pending nonce if unconfirmed transactions are equal or more than maxInFlightSubset", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) txm.setNonce(address, 1) - mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions/3, nil).Twice() + mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightSubset, nil).Twice() client.On("PendingNonceAt", mock.Anything, address).Return(uint64(0), nil).Once() // LocalNonce: 1, PendingNonce: 0 bo, err := txm.broadcastTransaction(ctx, address) @@ -176,7 +176,7 @@ func TestBroadcastTransaction(t *testing.T) { txm.setNonce(address, 8) IDK := "IDK" txRequest := &types.TxRequest{ - Data: []byte{100}, + Data: []byte{100, 200}, IdempotencyKey: &IDK, ChainID: testutils.FixtureChainID, FromAddress: address, @@ -197,13 +197,13 @@ func TestBroadcastTransaction(t *testing.T) { require.NoError(t, err) assert.False(t, bo) assert.Equal(t, uint64(9), txm.getNonce(address)) - tx, err = txStore.FindTxWithIdempotencyKey(tests.Context(t), &IDK) + tx, err = txStore.FindTxWithIdempotencyKey(tests.Context(t), IDK) require.NoError(t, err) assert.Len(t, tx.Attempts, 1) var zeroTime time.Time - assert.Greater(t, tx.LastBroadcastAt, zeroTime) - assert.Greater(t, tx.Attempts[0].BroadcastAt, zeroTime) - assert.Greater(t, tx.InitialBroadcastAt, zeroTime) + assert.Greater(t, *tx.LastBroadcastAt, zeroTime) + assert.Greater(t, *tx.Attempts[0].BroadcastAt, zeroTime) + assert.Greater(t, *tx.InitialBroadcastAt, zeroTime) }) } diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 0c1d2861daa..c16c92a3aa7 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "math/big" - "strconv" "time" "github.com/google/uuid" @@ -42,8 +41,8 @@ type Transaction struct { SpecifiedGasLimit uint64 CreatedAt time.Time - InitialBroadcastAt time.Time - LastBroadcastAt time.Time + InitialBroadcastAt *time.Time + LastBroadcastAt *time.Time State TxState IsPurgeable bool @@ -61,18 +60,19 @@ type Transaction struct { } func (t *Transaction) PrettyPrint() string { - idk, nonce := "", "" - if t.IdempotencyKey != nil { - idk = *t.IdempotencyKey - } - if t.Nonce != nil { - nonce = strconv.FormatUint(*t.Nonce, 10) - } return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%s, FromAddress:%v, ToAddress:%v, Value:%v, `+ - `Data:%s, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ + `Data:%X, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ `Meta:%v, Subject:%v}`, - t.ID, idk, t.ChainID, nonce, t.FromAddress, t.ToAddress, t.Value, t.Data, t.SpecifiedGasLimit, t.CreatedAt, t.InitialBroadcastAt, - t.LastBroadcastAt, t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject) + t.ID, stringOrNull(t.IdempotencyKey), t.ChainID, stringOrNull(t.Nonce), t.FromAddress, t.ToAddress, t.Value, + t.Data, t.SpecifiedGasLimit, t.CreatedAt, stringOrNull(t.InitialBroadcastAt), stringOrNull(t.LastBroadcastAt), t.State, t.IsPurgeable, t.AttemptCount, + t.Meta, t.Subject) +} + +func stringOrNull[T any](t *T) string { + if t != nil { + return fmt.Sprintf("%v", t) + } + return "null" } func (t *Transaction) PrettyPrintWithAttempts() string { @@ -125,7 +125,7 @@ type Attempt struct { SignedTransaction *types.Transaction CreatedAt time.Time - BroadcastAt time.Time + BroadcastAt *time.Time } func (a *Attempt) DeepCopy() *Attempt { @@ -138,7 +138,7 @@ func (a *Attempt) DeepCopy() *Attempt { func (a *Attempt) PrettyPrint() string { return fmt.Sprintf(`{ID:%d, TxID:%d, Hash:%v, Fee:%v, GasLimit:%d, Type:%v, CreatedAt:%v, BroadcastAt:%v}`, - a.ID, a.TxID, a.Hash, a.Fee, a.GasLimit, a.Type, a.CreatedAt, a.BroadcastAt) + a.ID, a.TxID, a.Hash, a.Fee, a.GasLimit, a.Type, a.CreatedAt, stringOrNull(a.BroadcastAt)) } type TxRequest struct { From 8d870fbaad46ee2cb4c058e0ade23f86683a1d1b Mon Sep 17 00:00:00 2001 From: Dimitris Date: Mon, 9 Dec 2024 13:25:24 +0200 Subject: [PATCH 66/73] Improve logs --- core/chains/evm/txm/txm.go | 4 ++-- core/chains/evm/txm/types/transaction.go | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 3ac899efd80..12a81042e13 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -360,7 +360,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio start := time.Now() txErr := t.client.SendTransaction(ctx, tx, attempt) tx.AttemptCount++ - t.lggr.Infow("Broadcasted attempt", "tx", tx.PrettyPrint(), "attempt", attempt.PrettyPrint(), "duration", time.Since(start), "txErr: ", txErr) + t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { if err = t.errorHandler.HandleError(tx, txErr, t.attemptBuilder, t.client, t.txStore, t.setNonce, false); err != nil { return @@ -430,7 +430,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return true, fmt.Errorf("reached max allowed attempts for txID: %d. TXM won't broadcast any more attempts."+ "If this error persists, it means the transaction won't be confirmed and the TXM needs to be restarted."+ "Look for any error messages from previous broadcasted attempts that may indicate why this happened, i.e. wallet is out of funds. Tx: %v", tx.ID, - tx.PrettyPrintWithAttempts()) + tx.PrintWithAttempts()) } if tx.LastBroadcastAt == nil || time.Since(*tx.LastBroadcastAt) > (t.config.BlockTime*time.Duration(t.config.RetryBlockThreshold)) { diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index c16c92a3aa7..c2be729e509 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "math/big" + "reflect" "time" "github.com/google/uuid" @@ -59,30 +60,30 @@ type Transaction struct { CallbackCompleted bool } -func (t *Transaction) PrettyPrint() string { +func (t *Transaction) String() string { return fmt.Sprintf(`{txID:%d, IdempotencyKey:%v, ChainID:%v, Nonce:%s, FromAddress:%v, ToAddress:%v, Value:%v, `+ `Data:%X, SpecifiedGasLimit:%d, CreatedAt:%v, InitialBroadcastAt:%v, LastBroadcastAt:%v, State:%v, IsPurgeable:%v, AttemptCount:%d, `+ `Meta:%v, Subject:%v}`, t.ID, stringOrNull(t.IdempotencyKey), t.ChainID, stringOrNull(t.Nonce), t.FromAddress, t.ToAddress, t.Value, - t.Data, t.SpecifiedGasLimit, t.CreatedAt, stringOrNull(t.InitialBroadcastAt), stringOrNull(t.LastBroadcastAt), t.State, t.IsPurgeable, t.AttemptCount, - t.Meta, t.Subject) + reflect.ValueOf(&t.Data).Elem(), t.SpecifiedGasLimit, t.CreatedAt, stringOrNull(t.InitialBroadcastAt), stringOrNull(t.LastBroadcastAt), + t.State, t.IsPurgeable, t.AttemptCount, t.Meta, t.Subject) } func stringOrNull[T any](t *T) string { if t != nil { - return fmt.Sprintf("%v", t) + return fmt.Sprintf("%v", *t) } return "null" } -func (t *Transaction) PrettyPrintWithAttempts() string { +func (t *Transaction) PrintWithAttempts() string { attempts := " Attempts: [" for _, a := range t.Attempts { - attempts += a.PrettyPrint() + ", " + attempts += a.String() + ", " } attempts += "]" - return t.PrettyPrint() + attempts + return t.String() + attempts } func (t *Transaction) FindAttemptByHash(attemptHash common.Hash) (*Attempt, error) { @@ -136,7 +137,7 @@ func (a *Attempt) DeepCopy() *Attempt { return &txCopy } -func (a *Attempt) PrettyPrint() string { +func (a *Attempt) String() string { return fmt.Sprintf(`{ID:%d, TxID:%d, Hash:%v, Fee:%v, GasLimit:%d, Type:%v, CreatedAt:%v, BroadcastAt:%v}`, a.ID, a.TxID, a.Hash, a.Fee, a.GasLimit, a.Type, a.CreatedAt, stringOrNull(a.BroadcastAt)) } From a2f1544f229413a264e123ace5b24602f90f44c9 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 10 Dec 2024 16:34:23 +0200 Subject: [PATCH 67/73] Move initialization of nonce --- core/chains/evm/txm/txm.go | 28 ++++++++++++---------------- core/chains/evm/txm/txm_test.go | 9 ++++++--- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 12a81042e13..e3beac1f890 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -137,45 +137,39 @@ func (t *Txm) Start(ctx context.Context) error { return err } for _, address := range addresses { - err := t.startAddress(ctx, address) - if err != nil { - return err - } + t.startAddress(address) } return nil }) } -func (t *Txm) startAddress(ctx context.Context, address common.Address) error { +func (t *Txm) startAddress(address common.Address) { triggerCh := make(chan struct{}, 1) t.triggerCh[address] = triggerCh - pendingNonce, err := t.pollForPendingNonce(ctx, address) - if err != nil { - return err - } - t.setNonce(address, pendingNonce) t.wg.Add(2) go t.broadcastLoop(address, triggerCh) go t.backfillLoop(address) - return nil } -func (t *Txm) pollForPendingNonce(ctx context.Context, address common.Address) (pendingNonce uint64, err error) { +func (t *Txm) initializeNonce(ctx context.Context, address common.Address) { ctxWithTimeout, cancel := context.WithTimeout(ctx, pendingNonceDefaultTimeout) defer cancel() for { - pendingNonce, err = t.client.PendingNonceAt(ctxWithTimeout, address) + pendingNonce, err := t.client.PendingNonceAt(ctxWithTimeout, address) if err != nil { - t.lggr.Errorw("Error when fetching initial pending nonce", "address", address, "err", err) + t.lggr.Errorw("Error when fetching initial nonce", "address", address, "err", err) select { case <-time.After(pendingNonceRecheckInterval): case <-ctx.Done(): - return 0, context.Cause(ctx) + t.lggr.Errorw("context error", "err", context.Cause(ctx)) + return } continue } - return pendingNonce, nil + t.setNonce(address, pendingNonce) + t.lggr.Debugf("Set initial nonce for address: %v to %d", address, pendingNonce) + return } } @@ -239,6 +233,8 @@ func (t *Txm) broadcastLoop(address common.Address, triggerCh chan struct{}) { broadcastWithBackoff := newBackoff(1 * time.Second) var broadcastCh <-chan time.Time + t.initializeNonce(ctx, address) + for { start := time.Now() bo, err := t.broadcastTransaction(ctx, address) diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 41c6c4d0741..703f4c9b5db 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -2,6 +2,7 @@ package txm import ( "errors" + "fmt" "testing" "time" @@ -38,12 +39,14 @@ func TestLifecycle(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) config := Config{BlockTime: 1 * time.Minute} txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) + require.NoError(t, txStore.Add(address1)) keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address1}, nil).Once() txm := NewTxm(lggr, testutils.FixtureChainID, client, nil, txStore, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() - client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), nil).Once() + client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(100), nil).Once() require.NoError(t, txm.Start(tests.Context(t))) - tests.AssertLogEventually(t, observedLogs, "Error when fetching initial pending nonce") + tests.AssertLogEventually(t, observedLogs, "Error when fetching initial nonce") + tests.AssertLogEventually(t, observedLogs, fmt.Sprintf("Set initial nonce for address: %v to %d", address1, 100)) }) t.Run("tests lifecycle successfully without any transactions", func(t *testing.T) { @@ -89,7 +92,7 @@ func TestTrigger(t *testing.T) { txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) var nonce uint64 // Start - client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() + client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Maybe() servicetest.Run(t, txm) txm.Trigger(address) }) From 5b023a4870799d692e6db138ed038e4d6c3ed0e8 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Tue, 10 Dec 2024 18:01:58 +0200 Subject: [PATCH 68/73] Add Beholder metrics --- core/chains/evm/txm/metrics.go | 93 +++++++++++++++++++++++++++++++++ core/chains/evm/txm/txm.go | 39 +++++--------- core/chains/evm/txm/txm_test.go | 3 ++ 3 files changed, 108 insertions(+), 27 deletions(-) create mode 100644 core/chains/evm/txm/metrics.go diff --git a/core/chains/evm/txm/metrics.go b/core/chains/evm/txm/metrics.go new file mode 100644 index 00000000000..5ccc711ef09 --- /dev/null +++ b/core/chains/evm/txm/metrics.go @@ -0,0 +1,93 @@ +package txm + +import ( + "context" + "fmt" + "math/big" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/metrics" +) + +var ( + promNumBroadcastedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_broadcasted_transactions", + Help: "Total number of successful broadcasted transactions.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs or when filling the nonce for untracked transactions.", + }, []string{"chainID"}) + promNumNonceGaps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "txm_num_nonce_gaps", + Help: "Total number of nonce gaps created that the transaction manager had to fill.", + }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "txm_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + }, []string{"chainID"}) +) + +type txmMetrics struct { + metrics.Labeler + chainID *big.Int + numBroadcastedTxs metric.Int64Counter + numConfirmedTxs metric.Int64Counter + numNonceGaps metric.Int64Counter + timeUntilTxConfirmed metric.Float64Histogram +} + +func NewTxmMetrics(chainID *big.Int) (*txmMetrics, error) { + numBroadcastedTxs, err := beholder.GetMeter().Int64Counter("txm_num_broadcasted_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register broadcasted txs number: %w", err) + } + + numConfirmedTxs, err := beholder.GetMeter().Int64Counter("txm_num_confirmed_transactions") + if err != nil { + return nil, fmt.Errorf("failed to register confirmed txs number: %w", err) + } + + numNonceGaps, err := beholder.GetMeter().Int64Counter("txm_num_nonce_gaps") + if err != nil { + return nil, fmt.Errorf("failed to register nonce gaps number: %w", err) + } + + timeUntilTxConfirmed, err := beholder.GetMeter().Float64Histogram("txm_time_until_tx_confirmed") + if err != nil { + return nil, fmt.Errorf("failed to register time until tx confirmed: %w", err) + } + + return &txmMetrics{ + chainID: chainID, + Labeler: metrics.NewLabeler().With("chainID", chainID.String()), + numBroadcastedTxs: numBroadcastedTxs, + numConfirmedTxs: numConfirmedTxs, + numNonceGaps: numNonceGaps, + timeUntilTxConfirmed: timeUntilTxConfirmed, + }, nil +} + +func (m *txmMetrics) IncrementNumBroadcastedTxs(ctx context.Context) { + promNumBroadcastedTxs.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.numBroadcastedTxs.Add(ctx, 1) +} + +func (m *txmMetrics) IncrementNumConfirmedTxs(ctx context.Context, confirmedTransactions int) { + promNumConfirmedTxs.WithLabelValues(m.chainID.String()).Add(float64(confirmedTransactions)) + m.numConfirmedTxs.Add(ctx, int64(confirmedTransactions)) +} + +func (m *txmMetrics) IncrementNumNonceGaps(ctx context.Context) { + promNumNonceGaps.WithLabelValues(m.chainID.String()).Add(float64(1)) + m.numNonceGaps.Add(ctx, 1) +} + +func (m *txmMetrics) RecordTimeUntilTxConfirmed(ctx context.Context, duration float64) { + promTimeUntilTxConfirmed.WithLabelValues(m.chainID.String()).Observe(duration) + m.timeUntilTxConfirmed.Record(ctx, duration) +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index e3beac1f890..054e31aeb79 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -9,8 +9,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/jpillora/backoff" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -67,25 +65,6 @@ type Keystore interface { EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } -var ( - promNumBroadcastedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "txm_num_broadcasted_transactions", - Help: "Total number of successful broadcasted transactions.", - }, []string{"chainID"}) - promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "txm_num_confirmed_transactions", - Help: "Total number of confirmed transactions. Note that this can happen multiple times per transaction in the case of re-orgs or when filling the nonce for untracked transactions.", - }, []string{"chainID"}) - promNumNonceGaps = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "txm_num_nonce_gaps", - Help: "Total number of nonce gaps created that the transaction manager had to fill.", - }, []string{"chainID"}) - promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "txm_time_until_tx_confirmed", - Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", - }, []string{"chainID"}) -) - type Config struct { EIP1559 bool BlockTime time.Duration @@ -104,6 +83,7 @@ type Txm struct { txStore TxStore keystore Keystore config Config + metrics *txmMetrics nonceMapMu sync.Mutex nonceMap map[common.Address]uint64 @@ -130,6 +110,11 @@ func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder func (t *Txm) Start(ctx context.Context) error { return t.StartOnce("Txm", func() error { + tm, err := NewTxmMetrics(t.chainID) + if err != nil { + return err + } + t.metrics = tm t.stopCh = make(chan struct{}) addresses, err := t.keystore.EnabledAddressesForChain(ctx, t.chainID) @@ -372,7 +357,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio } } - promNumBroadcastedTxs.WithLabelValues(t.chainID.String()).Add(float64(1)) + t.metrics.IncrementNumBroadcastedTxs(ctx) return t.txStore.UpdateTransactionBroadcast(ctx, attempt.TxID, *tx.Nonce, attempt.Hash, address) } @@ -387,8 +372,8 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, err } if len(confirmedTransactions) > 0 || len(unconfirmedTransactionIDs) > 0 { - promNumConfirmedTxs.WithLabelValues(t.chainID.String()).Add(float64(len(confirmedTransactions))) - confirmedTransactionIDs := extractMetrics(confirmedTransactions, t.chainID) + t.metrics.IncrementNumConfirmedTxs(ctx, len(confirmedTransactions)) + confirmedTransactionIDs := t.extractMetrics(ctx, confirmedTransactions) t.lggr.Infof("Confirmed transaction IDs: %v . Re-orged transaction IDs: %v", confirmedTransactionIDs, unconfirmedTransactionIDs) } @@ -403,7 +388,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) if tx == nil || *tx.Nonce != latestNonce { t.lggr.Warnf("Nonce gap at nonce: %d - address: %v. Creating a new transaction\n", latestNonce, address) - promNumNonceGaps.WithLabelValues(t.chainID.String()).Add(float64(1)) + t.metrics.IncrementNumNonceGaps(ctx) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { //nolint:revive //linter nonsense if !tx.IsPurgeable && t.stuckTxDetector != nil { @@ -446,12 +431,12 @@ func (t *Txm) createAndSendEmptyTx(ctx context.Context, latestNonce uint64, addr return t.createAndSendAttempt(ctx, tx, address) } -func extractMetrics(txs []*types.Transaction, chainID *big.Int) []uint64 { +func (t *Txm) extractMetrics(ctx context.Context, txs []*types.Transaction) []uint64 { confirmedTxIDs := make([]uint64, 0, len(txs)) for _, tx := range txs { confirmedTxIDs = append(confirmedTxIDs, tx.ID) if tx.InitialBroadcastAt != nil { - promTimeUntilTxConfirmed.WithLabelValues(chainID.String()).Observe(float64(time.Since(*tx.InitialBroadcastAt))) + t.metrics.RecordTimeUntilTxConfirmed(ctx, float64(time.Since(*tx.InitialBroadcastAt))) } } return confirmedTxIDs diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 703f4c9b5db..458c0ca97ef 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -177,6 +177,9 @@ func TestBroadcastTransaction(t *testing.T) { require.NoError(t, txStore.Add(address)) txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) txm.setNonce(address, 8) + metrics, err := NewTxmMetrics(testutils.FixtureChainID) + require.NoError(t, err) + txm.metrics = metrics IDK := "IDK" txRequest := &types.TxRequest{ Data: []byte{100, 200}, From 86ce03ad08432aaa45d3e4ed093be0cd36492490 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Thu, 12 Dec 2024 16:44:17 +0200 Subject: [PATCH 69/73] Improve InMemoryStorage --- core/chains/evm/txm/storage/inmemory_store.go | 20 +++- .../evm/txm/storage/inmemory_store_test.go | 94 +++++++++++++++---- 2 files changed, 93 insertions(+), 21 deletions(-) diff --git a/core/chains/evm/txm/storage/inmemory_store.go b/core/chains/evm/txm/storage/inmemory_store.go index 6451f6c3068..2f5cde53a56 100644 --- a/core/chains/evm/txm/storage/inmemory_store.go +++ b/core/chains/evm/txm/storage/inmemory_store.go @@ -44,18 +44,22 @@ func NewInMemoryStore(lggr logger.Logger, address common.Address, chainID *big.I chainID: chainID, UnstartedTransactions: make([]*types.Transaction, 0, maxQueuedTransactions), UnconfirmedTransactions: make(map[uint64]*types.Transaction), - ConfirmedTransactions: make(map[uint64]*types.Transaction), + ConfirmedTransactions: make(map[uint64]*types.Transaction, maxQueuedTransactions), Transactions: make(map[uint64]*types.Transaction), } } func (m *InMemoryStore) AbandonPendingTransactions() { + // TODO: append existing fatal transactions and cap the size m.Lock() defer m.Unlock() for _, tx := range m.UnstartedTransactions { tx.State = types.TxFatalError } + for _, tx := range m.FatalTransactions { + delete(m.Transactions, tx.ID) + } m.FatalTransactions = m.UnstartedTransactions m.UnstartedTransactions = []*types.Transaction{} @@ -184,6 +188,11 @@ func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) if tx.Nonce == nil { return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) } + existingTx, exists := m.ConfirmedTransactions[*tx.Nonce] + if exists { + m.lggr.Errorw("Another confirmed transaction with the same nonce exists. Transaction will overwritten.", + "existingTx", existingTx, "newTx", tx) + } if *tx.Nonce < latestNonce { tx.State = types.TxConfirmed confirmedTransactions = append(confirmedTransactions, tx.DeepCopy()) @@ -197,6 +206,11 @@ func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) if tx.Nonce == nil { return nil, nil, fmt.Errorf("nonce for txID: %v is empty", tx.ID) } + existingTx, exists := m.UnconfirmedTransactions[*tx.Nonce] + if exists { + m.lggr.Errorw("Another unconfirmed transaction with the same nonce exists. Transaction will overwritten.", + "existingTx", existingTx, "newTx", tx) + } if *tx.Nonce >= latestNonce { tx.State = types.TxUnconfirmed tx.LastBroadcastAt = nil // Mark reorged transaction as if it wasn't broadcasted before @@ -206,7 +220,7 @@ func (m *InMemoryStore) MarkConfirmedAndReorgedTransactions(latestNonce uint64) } } - if len(m.ConfirmedTransactions) >= maxQueuedTransactions { + if len(m.ConfirmedTransactions) > maxQueuedTransactions { prunedTxIDs := m.pruneConfirmedTransactions() m.lggr.Debugf("Confirmed transactions map for address: %v reached max limit of: %d. Pruned 1/%d of the oldest confirmed transactions. TxIDs: %v", m.address, maxQueuedTransactions, pruneSubset, prunedTxIDs) @@ -247,7 +261,7 @@ func (m *InMemoryStore) UpdateTransactionBroadcast(txID uint64, txNonce uint64, } a, err := unconfirmedTx.FindAttemptByHash(attemptHash) if err != nil { - return err + return fmt.Errorf("UpdateTransactionBroadcast failed to find attempt. %w", err) } a.BroadcastAt = &now diff --git a/core/chains/evm/txm/storage/inmemory_store_test.go b/core/chains/evm/txm/storage/inmemory_store_test.go index db1f54d4093..919a36dde50 100644 --- a/core/chains/evm/txm/storage/inmemory_store_test.go +++ b/core/chains/evm/txm/storage/inmemory_store_test.go @@ -8,8 +8,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" @@ -19,8 +21,8 @@ func TestAbandonPendingTransactions(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() - m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) t.Run("abandons unstarted and unconfirmed transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) // Unstarted tx1 := insertUnstartedTransaction(m) tx2 := insertUnstartedTransaction(m) @@ -40,6 +42,7 @@ func TestAbandonPendingTransactions(t *testing.T) { }) t.Run("skips all types apart from unstarted and unconfirmed transactions", func(t *testing.T) { + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) // Fatal tx1 := insertFataTransaction(m) tx2 := insertFataTransaction(m) @@ -56,6 +59,7 @@ func TestAbandonPendingTransactions(t *testing.T) { assert.Equal(t, types.TxFatalError, tx2.State) assert.Equal(t, types.TxConfirmed, tx3.State) assert.Equal(t, types.TxConfirmed, tx4.State) + assert.Len(t, m.Transactions, 2) // tx1, tx2 were dropped }) } @@ -65,33 +69,39 @@ func TestAppendAttemptToTransaction(t *testing.T) { fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - _, err := insertUnconfirmedTransaction(m, 0) // txID = 1 + _, err := insertUnconfirmedTransaction(m, 10) // txID = 1, nonce = 10 require.NoError(t, err) - _, err = insertConfirmedTransaction(m, 2) // txID = 1 + _, err = insertConfirmedTransaction(m, 2) // txID = 2, nonce = 2 require.NoError(t, err) t.Run("fails if corresponding unconfirmed transaction for attempt was not found", func(t *testing.T) { var nonce uint64 = 1 - newAttempt := &types.Attempt{ - TxID: 1, - } - require.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + newAttempt := &types.Attempt{} + err := m.AppendAttemptToTransaction(nonce, newAttempt) + require.Error(t, err) + require.ErrorContains(t, err, "unconfirmed tx was not found") }) - t.Run("fails if unconfirmed transaction was found but has doesn't match the txID", func(t *testing.T) { - var nonce uint64 + t.Run("fails if unconfirmed transaction was found but doesn't match the txID", func(t *testing.T) { + var nonce uint64 = 10 newAttempt := &types.Attempt{ TxID: 2, } - require.Error(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + err := m.AppendAttemptToTransaction(nonce, newAttempt) + require.Error(t, err) + require.ErrorContains(t, err, "attempt points to a different txID") }) t.Run("appends attempt to transaction", func(t *testing.T) { - var nonce uint64 + var nonce uint64 = 10 newAttempt := &types.Attempt{ TxID: 1, } require.NoError(t, m.AppendAttemptToTransaction(nonce, newAttempt)) + tx, _ := m.FetchUnconfirmedTransactionAtNonceWithCount(10) + assert.Len(t, tx.Attempts, 1) + assert.Equal(t, uint16(1), tx.AttemptCount) + assert.False(t, tx.Attempts[0].CreatedAt.IsZero()) }) } @@ -105,6 +115,10 @@ func TestCountUnstartedTransactions(t *testing.T) { insertUnstartedTransaction(m) assert.Equal(t, 1, m.CountUnstartedTransactions()) + + _, err := insertConfirmedTransaction(m, 10) + require.NoError(t, err) + assert.Equal(t, 1, m.CountUnstartedTransactions()) } func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { @@ -112,16 +126,23 @@ func TestCreateEmptyUnconfirmedTransaction(t *testing.T) { fromAddress := testutils.NewAddress() m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - _, err := insertUnconfirmedTransaction(m, 0) + _, err := insertUnconfirmedTransaction(m, 1) + require.NoError(t, err) + _, err = insertConfirmedTransaction(m, 0) require.NoError(t, err) t.Run("fails if unconfirmed transaction with the same nonce exists", func(t *testing.T) { + _, err := m.CreateEmptyUnconfirmedTransaction(1, 0) + require.Error(t, err) + }) + + t.Run("fails if confirmed transaction with the same nonce exists", func(t *testing.T) { _, err := m.CreateEmptyUnconfirmedTransaction(0, 0) require.Error(t, err) }) t.Run("creates a new empty unconfirmed transaction", func(t *testing.T) { - tx, err := m.CreateEmptyUnconfirmedTransaction(1, 0) + tx, err := m.CreateEmptyUnconfirmedTransaction(2, 0) require.NoError(t, err) assert.Equal(t, types.TxUnconfirmed, tx.State) }) @@ -185,7 +206,7 @@ func TestFetchUnconfirmedTransactionAtNonceWithCount(t *testing.T) { assert.Equal(t, 1, count) } -func TestMarkTransactionsConfirmed(t *testing.T) { +func TestMarkConfirmedAndReorgedTransactions(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() @@ -245,17 +266,33 @@ func TestMarkTransactionsConfirmed(t *testing.T) { assert.Equal(t, utxs[0], ctx2.ID) assert.Empty(t, ctxs) }) + + t.Run("logs an error during confirmation if a transaction with the same nonce already exists", func(t *testing.T) { + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + m := NewInMemoryStore(lggr, fromAddress, testutils.FixtureChainID) + _, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + _, err = insertUnconfirmedTransaction(m, 0) + require.NoError(t, err) + + _, _, err = m.MarkConfirmedAndReorgedTransactions(1) + require.NoError(t, err) + tests.AssertLogEventually(t, observedLogs, "Another confirmed transaction with the same nonce exists") + }) + t.Run("prunes confirmed transactions map if it reaches the limit", func(t *testing.T) { m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) - for i := 0; i < maxQueuedTransactions; i++ { + overshot := 5 + for i := 0; i < maxQueuedTransactions+overshot; i++ { //nolint:gosec // this won't overflow _, err := insertConfirmedTransaction(m, uint64(i)) require.NoError(t, err) } - assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions) - _, _, err := m.MarkConfirmedAndReorgedTransactions(maxQueuedTransactions) + assert.Len(t, m.ConfirmedTransactions, maxQueuedTransactions+overshot) + //nolint:gosec // this won't overflow + _, _, err := m.MarkConfirmedAndReorgedTransactions(uint64(maxQueuedTransactions + overshot)) require.NoError(t, err) - assert.Len(t, m.ConfirmedTransactions, (maxQueuedTransactions - maxQueuedTransactions/pruneSubset)) + assert.Len(t, m.ConfirmedTransactions, 170) }) } @@ -389,6 +426,23 @@ func TestDeleteAttemptForUnconfirmedTx(t *testing.T) { }) } +func TestFindTxWithIdempotencyKey(t *testing.T) { + t.Parallel() + fromAddress := testutils.NewAddress() + m := NewInMemoryStore(logger.Test(t), fromAddress, testutils.FixtureChainID) + tx, err := insertConfirmedTransaction(m, 0) + require.NoError(t, err) + + ik := "IK" + tx.IdempotencyKey = &ik + itx := m.FindTxWithIdempotencyKey(ik) + assert.Equal(t, ik, *itx.IdempotencyKey) + + uik := "Unknown" + itx = m.FindTxWithIdempotencyKey(uik) + assert.Nil(t, itx) +} + func TestPruneConfirmedTransactions(t *testing.T) { t.Parallel() fromAddress := testutils.NewAddress() @@ -424,6 +478,7 @@ func insertUnstartedTransaction(m *InMemoryStore) *types.Transaction { } m.UnstartedTransactions = append(m.UnstartedTransactions, tx) + m.Transactions[tx.ID] = tx return tx } @@ -449,6 +504,7 @@ func insertUnconfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transa } m.UnconfirmedTransactions[nonce] = tx + m.Transactions[tx.ID] = tx return tx, nil } @@ -474,6 +530,7 @@ func insertConfirmedTransaction(m *InMemoryStore, nonce uint64) (*types.Transact } m.ConfirmedTransactions[nonce] = tx + m.Transactions[tx.ID] = tx return tx, nil } @@ -496,5 +553,6 @@ func insertFataTransaction(m *InMemoryStore) *types.Transaction { } m.FatalTransactions = append(m.FatalTransactions, tx) + m.Transactions[tx.ID] = tx return tx } From 5b73b73f81677e3444cfa8c3d0cea0b883ef8d52 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Thu, 12 Dec 2024 17:20:12 +0200 Subject: [PATCH 70/73] Support different priceMax per key --- core/chains/evm/txm/attempt_builder.go | 14 +++++++------- core/chains/evm/txmgr/builder.go | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/chains/evm/txm/attempt_builder.go b/core/chains/evm/txm/attempt_builder.go index 6be849c8dcb..16ed0f1a86a 100644 --- a/core/chains/evm/txm/attempt_builder.go +++ b/core/chains/evm/txm/attempt_builder.go @@ -20,23 +20,23 @@ type AttemptBuilderKeystore interface { } type attemptBuilder struct { - chainID *big.Int - priceMax *assets.Wei gas.EvmFeeEstimator - keystore AttemptBuilderKeystore + chainID *big.Int + priceMaxKey func(common.Address) *assets.Wei + keystore AttemptBuilderKeystore } -func NewAttemptBuilder(chainID *big.Int, priceMax *assets.Wei, estimator gas.EvmFeeEstimator, keystore AttemptBuilderKeystore) *attemptBuilder { +func NewAttemptBuilder(chainID *big.Int, priceMaxKey func(common.Address) *assets.Wei, estimator gas.EvmFeeEstimator, keystore AttemptBuilderKeystore) *attemptBuilder { return &attemptBuilder{ chainID: chainID, - priceMax: priceMax, + priceMaxKey: priceMaxKey, EvmFeeEstimator: estimator, keystore: keystore, } } func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (*types.Attempt, error) { - fee, estimatedGasLimit, err := a.EvmFeeEstimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMax, &tx.FromAddress, &tx.ToAddress) + fee, estimatedGasLimit, err := a.EvmFeeEstimator.GetFee(ctx, tx.Data, tx.SpecifiedGasLimit, a.priceMaxKey(tx.FromAddress), &tx.FromAddress, &tx.ToAddress) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (a *attemptBuilder) NewAttempt(ctx context.Context, lggr logger.Logger, tx } func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, previousAttempt types.Attempt) (*types.Attempt, error) { - bumpedFee, bumpedFeeLimit, err := a.EvmFeeEstimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMax, nil) + bumpedFee, bumpedFeeLimit, err := a.EvmFeeEstimator.BumpFee(ctx, previousAttempt.Fee, tx.SpecifiedGasLimit, a.priceMaxKey(tx.FromAddress), nil) if err != nil { return nil, err } diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 8e1644f3a73..9de3bd79636 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -125,7 +125,7 @@ func NewTxmV2( stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) } - attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMax(), estimator, keyStore) + attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMaxKey, estimator, keyStore) inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), From 08f4851de5b90332eec9a3d18ad39d5f34159c4c Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 13 Dec 2024 16:57:12 +0200 Subject: [PATCH 71/73] Upgrades --- core/chains/evm/config/toml/config.go | 27 +++++++ core/chains/evm/txm/orchestrator.go | 5 ++ core/chains/evm/txm/stuck_tx_detector.go | 30 ++++--- core/chains/evm/txm/stuck_tx_detector_test.go | 80 +++++++++++++++++++ core/chains/evm/txm/txm.go | 6 +- 5 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 core/chains/evm/txm/stuck_tx_detector_test.go diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 0f8b1eceee5..36c7d0f052b 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -6,6 +6,7 @@ import ( "net/url" "slices" "strconv" + "time" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/pelletier/go-toml/v2" @@ -451,6 +452,20 @@ func (c *Chain) ValidateConfig() (err error) { err = multierr.Append(err, commonconfig.ErrInvalid{Name: "GasEstimator.BumpThreshold", Value: 0, Msg: fmt.Sprintf("cannot be 0 if Transactions.AutoPurge.MinAttempts is set for %s", chainType)}) } } + case chaintype.ChainDualBroadcast: + if c.Transactions.AutoPurge.DetectionApiUrl == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "Transactions.AutoPurge.DetectionApiUrl", Msg: fmt.Sprintf("must be set for %s", chainType)}) + } + if c.Transactions.AutoPurge.Threshold == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "Transactions.AutoPurge.Threshold", Msg: fmt.Sprintf("needs to be set if auto-purge feature is enabled for %s", chainType)}) + } else if *c.Transactions.AutoPurge.Threshold == 0 { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "Transactions.AutoPurge.Threshold", Value: 0, Msg: fmt.Sprintf("cannot be 0 if auto-purge feature is enabled for %s", chainType)}) + } + if c.TxmV2.Enabled != nil && *c.TxmV2.Enabled { + if c.TxmV2.CustomURL == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "TxmV2.CustomURL", Msg: fmt.Sprintf("must be set for %s", chainType)}) + } + } default: // Bump Threshold is required because the stuck tx heuristic relies on a minimum number of bump attempts to exist if c.GasEstimator.BumpThreshold == nil { @@ -494,6 +509,18 @@ func (t *TxmV2) setFrom(f *TxmV2) { } } +func (t *TxmV2) ValidateConfig() (err error) { + if t.Enabled != nil && *t.Enabled { + if t.BlockTime == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "TxmV2.BlockTime", Msg: "must be set if txmv2 feature is enabled"}) + } + if t.BlockTime.Duration() < 2*time.Second { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "TxmV2.BlockTime", Msg: "must be equal to or greater than 2 seconds"}) + } + } + return +} + type Transactions struct { ForwardersEnabled *bool MaxInFlight *uint32 diff --git a/core/chains/evm/txm/orchestrator.go b/core/chains/evm/txm/orchestrator.go index 914a14f981b..aa8c5e9e2c6 100644 --- a/core/chains/evm/txm/orchestrator.go +++ b/core/chains/evm/txm/orchestrator.go @@ -34,6 +34,7 @@ type OrchestratorTxStore interface { } type OrchestratorKeystore interface { + CheckEnabled(ctx context.Context, address common.Address, chainID *big.Int) error EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) } @@ -175,6 +176,10 @@ func (o *Orchestrator[BLOCK_HASH, HEAD]) CreateTransaction(ctx context.Context, if wrappedTx != nil { o.lggr.Infof("Found Tx with IdempotencyKey: %v. Returning existing Tx without creating a new one.", *wrappedTx.IdempotencyKey) } else { + if kErr := o.keystore.CheckEnabled(ctx, request.FromAddress, o.chainID); kErr != nil { + return tx, fmt.Errorf("cannot send transaction from %s on chain ID %s: %w", request.FromAddress, o.chainID.String(), kErr) + } + var pipelineTaskRunID uuid.NullUUID if request.PipelineTaskRunID != nil { pipelineTaskRunID.UUID = *request.PipelineTaskRunID diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go index 87b78d5cc04..33905ead80c 100644 --- a/core/chains/evm/txm/stuck_tx_detector.go +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -23,22 +23,23 @@ type StuckTxDetectorConfig struct { } type stuckTxDetector struct { - lggr logger.Logger - chainType chaintype.ChainType - config StuckTxDetectorConfig + lggr logger.Logger + chainType chaintype.ChainType + config StuckTxDetectorConfig + lastPurgeMap map[common.Address]time.Time } func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, config StuckTxDetectorConfig) *stuckTxDetector { return &stuckTxDetector{ - lggr: lggr, - chainType: chaintype, - config: config, + lggr: lggr, + chainType: chaintype, + config: config, + lastPurgeMap: make(map[common.Address]time.Time), } } func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) { switch s.chainType { - // TODO: rename case chaintype.ChainDualBroadcast: result, err := s.dualBroadcastDetection(ctx, tx) if result || err != nil { @@ -50,11 +51,20 @@ func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types. } } +// timeBasedDetection marks a transaction if all the following conditions are met: +// - LastBroadcastAt is not nil +// - Time since last broadcast is above the threshold +// - Time since last purge is above threshold +// +// NOTE: Potentially we can use a subset of threhsold for last purge check, because the transaction would have already been broadcasted to the mempool +// so it is more likely to be picked up compared to a transaction that hasn't been broadcasted before. This would avoid slowing down TXM for sebsequent transactions +// in case the current one is stuck. func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) - if tx.LastBroadcastAt != nil && time.Since(*tx.LastBroadcastAt) > threshold { - s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", - tx.ID, tx.LastBroadcastAt, threshold) + if tx.LastBroadcastAt != nil && min(time.Since(*tx.LastBroadcastAt), time.Since(s.lastPurgeMap[tx.FromAddress])) > threshold { + s.lggr.Debugf("TxID: %v last broadcast was: %v and last purge: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", + tx.ID, tx.LastBroadcastAt, s.lastPurgeMap[tx.FromAddress], threshold) + s.lastPurgeMap[tx.FromAddress] = time.Now() return true } return false diff --git a/core/chains/evm/txm/stuck_tx_detector_test.go b/core/chains/evm/txm/stuck_tx_detector_test.go new file mode 100644 index 00000000000..af5a765dcdb --- /dev/null +++ b/core/chains/evm/txm/stuck_tx_detector_test.go @@ -0,0 +1,80 @@ +package txm + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +func TestTimeBasedDetection(t *testing.T) { + t.Parallel() + + t.Run("returns false if transaction is not stuck", func(t *testing.T) { + config := StuckTxDetectorConfig{ + BlockTime: 10 * time.Second, + StuckTxBlockThreshold: 5, + } + fromAddress := testutils.NewAddress() + s := NewStuckTxDetector(logger.Test(t), "", config) + + // No previous broadcast + tx := &types.Transaction{ + ID: 1, + LastBroadcastAt: nil, + FromAddress: fromAddress, + } + assert.False(t, s.timeBasedDetection(tx)) + // Not enough time has passed since last broadcast + now := time.Now() + tx.LastBroadcastAt = &now + assert.False(t, s.timeBasedDetection(tx)) + // Not enough time has passed since last purge + tx.LastBroadcastAt = &time.Time{} + s.lastPurgeMap[fromAddress] = now + assert.False(t, s.timeBasedDetection(tx)) + }) + + t.Run("returns true if transaction is stuck", func(t *testing.T) { + config := StuckTxDetectorConfig{ + BlockTime: 10 * time.Second, + StuckTxBlockThreshold: 5, + } + fromAddress := testutils.NewAddress() + s := NewStuckTxDetector(logger.Test(t), "", config) + + tx := &types.Transaction{ + ID: 1, + LastBroadcastAt: &time.Time{}, + FromAddress: fromAddress, + } + assert.True(t, s.timeBasedDetection(tx)) + }) + + t.Run("marks first tx as stuck, updates purge time for address, and returns false for the second tx with the same broadcast time", func(t *testing.T) { + config := StuckTxDetectorConfig{ + BlockTime: 1 * time.Second, + StuckTxBlockThreshold: 10, + } + fromAddress := testutils.NewAddress() + s := NewStuckTxDetector(logger.Test(t), "", config) + + tx1 := &types.Transaction{ + ID: 1, + LastBroadcastAt: &time.Time{}, + FromAddress: fromAddress, + } + tx2 := &types.Transaction{ + ID: 2, + LastBroadcastAt: &time.Time{}, + FromAddress: fromAddress, + } + assert.True(t, s.timeBasedDetection(tx1)) + assert.False(t, s.timeBasedDetection(tx2)) + }) +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 054e31aeb79..bf53e00e81a 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -187,6 +187,7 @@ func (t *Txm) Trigger(address common.Address) { } func (t *Txm) Abandon(address common.Address) error { + // TODO: restart txm t.lggr.Infof("Dropping unstarted and unconfirmed transactions for address: %v", address) return t.txStore.AbandonPendingTransactions(context.TODO(), address) } @@ -313,7 +314,7 @@ func (t *Txm) broadcastTransaction(ctx context.Context, address common.Address) t.setNonce(address, nonce+1) if err := t.createAndSendAttempt(ctx, tx, address); err != nil { - return true, err + return false, err } } } @@ -352,8 +353,7 @@ func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transactio return err } if pendingNonce <= *tx.Nonce { - t.lggr.Debugf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d", tx.ID, pendingNonce, *tx.Nonce) - return nil + return fmt.Errorf("Pending nonce for txID: %v didn't increase. PendingNonce: %d, TxNonce: %d. TxErr: %w", tx.ID, pendingNonce, *tx.Nonce, txErr) } } From ccf8be7dfff5ba538655da7c62b25349f11fbce3 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Fri, 13 Dec 2024 19:36:34 +0200 Subject: [PATCH 72/73] Fix config tests --- core/chains/evm/config/toml/config.go | 1 + core/services/chainlink/config_test.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 36c7d0f052b..6e360d886c1 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -513,6 +513,7 @@ func (t *TxmV2) ValidateConfig() (err error) { if t.Enabled != nil && *t.Enabled { if t.BlockTime == nil { err = multierr.Append(err, commonconfig.ErrMissing{Name: "TxmV2.BlockTime", Msg: "must be set if txmv2 feature is enabled"}) + return } if t.BlockTime.Duration() < 2*time.Second { err = multierr.Append(err, commonconfig.ErrInvalid{Name: "TxmV2.BlockTime", Msg: "must be equal to or greater than 2 seconds"}) diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 1dbc46d069d..895c8809e23 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -1477,7 +1477,7 @@ func TestConfig_Validate(t *testing.T) { - Nodes: 2 errors: - 0.HTTPURL: missing: required for all nodes - 1.HTTPURL: missing: required for all nodes - - 1: 10 errors: + - 1: 11 errors: - ChainType: invalid value (Foo): must not be set with this chain id - Nodes: missing: must have at least one node - ChainType: invalid value (Foo): must be one of arbitrum, astar, celo, gnosis, hedera, kroma, mantle, metis, optimismBedrock, scroll, wemix, xlayer, zkevm, zksync, zircuit, dualBroadcast or omitted @@ -1485,6 +1485,7 @@ func TestConfig_Validate(t *testing.T) { - GasEstimator.BumpThreshold: invalid value (0): cannot be 0 if auto-purge feature is enabled for Foo - Transactions.AutoPurge.Threshold: missing: needs to be set if auto-purge feature is enabled for Foo - Transactions.AutoPurge.MinAttempts: missing: needs to be set if auto-purge feature is enabled for Foo + - TxmV2.TxmV2.BlockTime: missing: must be set if txmv2 feature is enabled - GasEstimator: 2 errors: - FeeCapDefault: invalid value (101 wei): must be equal to PriceMax (99 wei) since you are using FixedPrice estimation with gas bumping disabled in EIP1559 mode - PriceMax will be used as the FeeCap for transactions instead of FeeCapDefault - PriceMax: invalid value (1 gwei): must be greater than or equal to PriceDefault @@ -1515,7 +1516,9 @@ func TestConfig_Validate(t *testing.T) { - 4: 2 errors: - ChainID: missing: required for all chains - Nodes: missing: must have at least one node - - 5.Transactions.AutoPurge.DetectionApiUrl: invalid value (): must be set for scroll + - 5: 2 errors: + - Transactions.AutoPurge.DetectionApiUrl: invalid value (): must be set for scroll + - TxmV2.TxmV2.BlockTime: missing: must be set if txmv2 feature is enabled - 6.Nodes: missing: 0th node (primary) must have a valid WSURL when http polling is disabled - Cosmos: 5 errors: - 1.ChainID: invalid value (Malaga-420): duplicate - must be unique From 98d0b53cf1f0947fdbd7f817d034a9df4bef5cf8 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Sat, 14 Dec 2024 15:27:42 +0200 Subject: [PATCH 73/73] Fix docs --- docs/CONFIG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 2a8dfa3b44f..a9f7da56fa9 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -6037,6 +6037,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6141,6 +6144,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16