From 0ac790b37fca951dfb4b4093c00c5adbd6987668 Mon Sep 17 00:00:00 2001 From: amit-momin <108959691+amit-momin@users.noreply.github.com> Date: Wed, 19 Jun 2024 11:20:27 -0500 Subject: [PATCH] Add transaction status function for external components to use (#13040) * Added transaction status function for external components to use * Added changeset and fixed linting * Updated changeset * Made internal error regex matching explicit * Updated common client error list * Defined a generalized tx error interface for chain agnostic compatibility * Adjusted to align as close as possible with the ChainWriter interface * Updated to align transaction status method to ChainWriter expectations * Updated to use the TransactionStatus type from chainlink-common * Simplified the TxError interface * Updated FindTxWithIdempotencyKey to return error if no rows found * Removed sql error checks outside of txstore * Updated method signature to accept string instead of uuid * Updated mocks * Moved tx finality check to Confirmer and renamed fields * Updated tx finalized check to consider re-orgs * Fixed linting * Removed finality from GetTransactionStatus method * Cleaned out unused client method * Rephrased comment to match others --- .changeset/funny-snails-shake.md | 5 + common/client/models.go | 6 +- common/client/multi_node_test.go | 2 +- common/txmgr/mocks/tx_manager.go | 30 +++ common/txmgr/txmgr.go | 96 +++++++--- common/txmgr/types/tx.go | 7 + common/txmgr/types/tx_store.go | 1 + core/chains/evm/client/errors.go | 31 +++- core/chains/evm/client/tx_simulator_test.go | 4 +- core/chains/evm/txmgr/builder.go | 6 +- core/chains/evm/txmgr/confirmer_test.go | 2 +- core/chains/evm/txmgr/evm_tx_store.go | 28 ++- core/chains/evm/txmgr/evm_tx_store_test.go | 2 +- core/chains/evm/txmgr/stuck_tx_detector.go | 10 +- core/chains/evm/txmgr/txmgr_test.go | 193 ++++++++++++++++++++ 15 files changed, 362 insertions(+), 61 deletions(-) create mode 100644 .changeset/funny-snails-shake.md diff --git a/.changeset/funny-snails-shake.md b/.changeset/funny-snails-shake.md new file mode 100644 index 00000000000..dad6c90872b --- /dev/null +++ b/.changeset/funny-snails-shake.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +Added API for products to query a transaction's status in the TXM #internal diff --git a/common/client/models.go b/common/client/models.go index fd0c3915940..8b616137669 100644 --- a/common/client/models.go +++ b/common/client/models.go @@ -18,7 +18,7 @@ const ( InsufficientFunds // Tx was rejected due to insufficient funds. ExceedsMaxFee // Attempt's fee was higher than the node's limit and got rejected. FeeOutOfValidRange // This error is returned when we use a fee price suggested from an RPC, but the network rejects the attempt due to an invalid range(mostly used by L2 chains). Retry by requesting a new suggested fee price. - OutOfCounters // The error returned when a transaction is too complex to be proven by zk circuits. This error is mainly returned by zk chains. + TerminallyStuck // The error returned when a transaction is or could get terminally stuck in the mempool without any chance of inclusion. sendTxReturnCodeLen // tracks the number of errors. Must always be last ) @@ -50,8 +50,8 @@ func (c SendTxReturnCode) String() string { return "ExceedsMaxFee" case FeeOutOfValidRange: return "FeeOutOfValidRange" - case OutOfCounters: - return "OutOfCounters" + case TerminallyStuck: + return "TerminallyStuck" default: return fmt.Sprintf("SendTxReturnCode(%d)", c) } diff --git a/common/client/multi_node_test.go b/common/client/multi_node_test.go index 4504b5071e1..3076d99b618 100644 --- a/common/client/multi_node_test.go +++ b/common/client/multi_node_test.go @@ -848,7 +848,7 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) { ExpectedTxResult: "not enough keccak counters to continue the execution", ExpectedCriticalErr: "", ResultsByCode: sendTxErrors{ - OutOfCounters: {errors.New("not enough keccak counters to continue the execution")}, + TerminallyStuck: {errors.New("not enough keccak counters to continue the execution")}, }, }, } diff --git a/common/txmgr/mocks/tx_manager.go b/common/txmgr/mocks/tx_manager.go index 974fd455903..502f6fe8a25 100644 --- a/common/txmgr/mocks/tx_manager.go +++ b/common/txmgr/mocks/tx_manager.go @@ -11,6 +11,8 @@ import ( null "gopkg.in/guregu/null.v4" + pkgtypes "github.com/smartcontractkit/chainlink-common/pkg/types" + txmgr "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" @@ -329,6 +331,34 @@ func (_m *TxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetFor return r0, r1 } +// GetTransactionStatus provides a mock function with given fields: ctx, transactionID +func (_m *TxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetTransactionStatus(ctx context.Context, transactionID string) (pkgtypes.TransactionStatus, error) { + ret := _m.Called(ctx, transactionID) + + if len(ret) == 0 { + panic("no return value specified for GetTransactionStatus") + } + + var r0 pkgtypes.TransactionStatus + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (pkgtypes.TransactionStatus, error)); ok { + return rf(ctx, transactionID) + } + if rf, ok := ret.Get(0).(func(context.Context, string) pkgtypes.TransactionStatus); ok { + r0 = rf(ctx, transactionID) + } else { + r0 = ret.Get(0).(pkgtypes.TransactionStatus) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, transactionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // HealthReport provides a mock function with given fields: func (_m *TxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) HealthReport() map[string]error { ret := _m.Called() diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index 44b518fdaab..9cefa5c15ba 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -2,7 +2,6 @@ package txmgr import ( "context" - "database/sql" "errors" "fmt" "math/big" @@ -14,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/utils" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" @@ -29,6 +29,8 @@ import ( // ResumeCallback is assumed to be idempotent type ResumeCallback func(ctx context.Context, id uuid.UUID, result interface{}, err error) error +type NewErrorClassifier func(err error) txmgrtypes.ErrorClassifier + // TxManager is the main component of the transaction manager. // It is also the interface to external callers. // @@ -62,6 +64,7 @@ type TxManager[ FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (count uint32, err error) + GetTransactionStatus(ctx context.Context, transactionID string) (state commontypes.TransactionStatus, err error) } type reset struct { @@ -102,13 +105,14 @@ type Txm[ chSubbed chan struct{} wg sync.WaitGroup - reaper *Reaper[CHAIN_ID] - resender *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - broadcaster *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - confirmer *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - fwdMgr txmgrtypes.ForwarderManager[ADDR] - txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + reaper *Reaper[CHAIN_ID] + resender *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + broadcaster *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + confirmer *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + fwdMgr txmgrtypes.ForwarderManager[ADDR] + txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + newErrorClassifier NewErrorClassifier } func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RegisterResumeCallback(fn ResumeCallback) { @@ -141,26 +145,28 @@ func NewTxm[ confirmer *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], resender *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + newErrorClassifierFunc NewErrorClassifier, ) *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { b := Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - logger: logger.Sugared(lggr), - txStore: txStore, - config: cfg, - txConfig: txCfg, - keyStore: keyStore, - chainID: chainId, - checkerFactory: checkerFactory, - chHeads: make(chan HEAD), - trigger: make(chan ADDR), - chStop: make(chan struct{}), - chSubbed: make(chan struct{}), - reset: make(chan reset), - fwdMgr: fwdMgr, - txAttemptBuilder: txAttemptBuilder, - broadcaster: broadcaster, - confirmer: confirmer, - resender: resender, - tracker: tracker, + logger: logger.Sugared(lggr), + txStore: txStore, + config: cfg, + txConfig: txCfg, + keyStore: keyStore, + chainID: chainId, + checkerFactory: checkerFactory, + chHeads: make(chan HEAD), + trigger: make(chan ADDR), + chStop: make(chan struct{}), + chSubbed: make(chan struct{}), + reset: make(chan reset), + fwdMgr: fwdMgr, + txAttemptBuilder: txAttemptBuilder, + broadcaster: broadcaster, + confirmer: confirmer, + resender: resender, + tracker: tracker, + newErrorClassifier: newErrorClassifierFunc, } if txCfg.ResendAfterThreshold() <= 0 { @@ -498,7 +504,7 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTran if txRequest.IdempotencyKey != nil { var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) - if err != nil && !errors.Is(err, sql.ErrNoRows) { + if err != nil { return tx, fmt.Errorf("Failed to search for transaction with IdempotencyKey: %w", err) } if existingTx != nil { @@ -625,6 +631,38 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTrans return b.txStore.CountTransactionsByState(ctx, state, b.chainID) } +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { + // Loads attempts and receipts in the transaction + tx, err := b.txStore.FindTxWithIdempotencyKey(ctx, transactionID, b.chainID) + if err != nil { + return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s: %w", transactionID, err) + } + // This check is required since a no-rows error returns nil err + if tx == nil { + return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s", transactionID) + } + switch tx.State { + case TxUnconfirmed, TxConfirmedMissingReceipt: + // Return unconfirmed for ConfirmedMissingReceipt since a receipt is required to determine if it is finalized + return commontypes.Unconfirmed, nil + case TxConfirmed: + // TODO: Check for finality and return finalized status + // Return unconfirmed if tx receipt's block is newer than the latest finalized block + return commontypes.Unconfirmed, nil + case TxFatalError: + // Use an ErrorClassifier to determine if the transaction is considered Fatal + txErr := b.newErrorClassifier(tx.GetError()) + if txErr != nil && txErr.IsFatal() { + return commontypes.Fatal, tx.GetError() + } + // Return failed for all other tx's marked as FatalError + return commontypes.Failed, tx.GetError() + default: + // Unstarted and InProgress are classified as unknown since they are not supported by the ChainWriter interface + return commontypes.Unknown, nil + } +} + type NullTxManager[ CHAIN_ID types.ID, HEAD types.Head[BLOCK_HASH], @@ -708,6 +746,10 @@ func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Cou return count, errors.New(n.ErrMsg) } +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { + return +} + func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pruneQueueAndCreateTxn( ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], diff --git a/common/txmgr/types/tx.go b/common/txmgr/types/tx.go index e04ebe95871..d2afbd209d8 100644 --- a/common/txmgr/types/tx.go +++ b/common/txmgr/types/tx.go @@ -339,3 +339,10 @@ func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetChecker() (Transm return t, nil } + +// Provides error classification to external components in a chain agnostic way +// Only exposes the error types that could be set in the transaction error field +type ErrorClassifier interface { + error + IsFatal() bool +} diff --git a/common/txmgr/types/tx_store.go b/common/txmgr/types/tx_store.go index 2b82e1f6483..c102fb5912a 100644 --- a/common/txmgr/types/tx_store.go +++ b/common/txmgr/types/tx_store.go @@ -53,6 +53,7 @@ type TxStore[ FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) // Find transactions loaded with transaction attempts and receipts by transaction IDs and states FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []TxState, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) } // TransactionStore contains the persistence layer methods needed to manage Txs and TxAttempts diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index f43b5189c7e..22fac5f7287 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -15,6 +15,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" commonclient "github.com/smartcontractkit/chainlink/v2/common/client" + commontypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/label" ) @@ -60,7 +61,7 @@ const ( TransactionAlreadyMined Fatal ServiceUnavailable - OutOfCounters + TerminallyStuck ) type ClientErrors map[int]*regexp.Regexp @@ -246,10 +247,17 @@ var zkSync = ClientErrors{ } var zkEvm = ClientErrors{ - OutOfCounters: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`), + TerminallyStuck: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`), } -var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm} +const TerminallyStuckMsg = "transaction terminally stuck" + +// Tx.Error messages that are set internally so they are not chain or client specific +var internal = ClientErrors{ + TerminallyStuck: regexp.MustCompile(TerminallyStuckMsg), +} + +var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm, internal} // ClientErrorRegexes returns a map of compiled regexes for each error type func ClientErrorRegexes(errsRegex config.ClientErrors) *ClientErrors { @@ -353,9 +361,16 @@ func (s *SendError) IsServiceUnavailable(configErrors *ClientErrors) bool { return s.is(ServiceUnavailable, configErrors) } -// IsOutOfCounters is a zk chain specific error returned if the transaction is too complex to prove on zk circuits -func (s *SendError) IsOutOfCounters(configErrors *ClientErrors) bool { - return s.is(OutOfCounters, configErrors) +// IsTerminallyStuck indicates if a transaction was stuck without any chance of inclusion +func (s *SendError) IsTerminallyStuckConfigError(configErrors *ClientErrors) bool { + return s.is(TerminallyStuck, configErrors) +} + +// IsFatal indicates if a transaction error is considered fatal for external callers +// The naming discrepancy is due to the generic transaction statuses introduced by ChainWriter +func (s *SendError) IsFatal() bool { + // An error classified as terminally stuck is considered fatal since the transaction payload should NOT be retried by external callers + return s.IsTerminallyStuckConfigError(nil) } // IsTimeout indicates if the error was caused by an exceeded context deadline @@ -399,6 +414,10 @@ func NewSendError(e error) *SendError { return &SendError{err: pkgerrors.WithStack(e), fatal: fatal} } +func NewTxError(e error) commontypes.ErrorClassifier { + return NewSendError(e) +} + // Geth/parity returns these errors if the transaction failed in such a way that: // 1. It will never be included into a block as a result of this send // 2. Resending the transaction at a different gas price will never change the outcome diff --git a/core/chains/evm/client/tx_simulator_test.go b/core/chains/evm/client/tx_simulator_test.go index ddd0a5fcff9..d82db154e04 100644 --- a/core/chains/evm/client/tx_simulator_test.go +++ b/core/chains/evm/client/tx_simulator_test.go @@ -78,7 +78,7 @@ func TestSimulateTx_Default(t *testing.T) { Data: []byte("0x00"), } sendErr := client.SimulateTransaction(ctx, ethClient, logger.TestSugared(t), "", msg) - require.Equal(t, true, sendErr.IsOutOfCounters(nil)) + require.Equal(t, true, sendErr.IsTerminallyStuckConfigError(nil)) }) t.Run("returns without error if simulation returns non-OOC error", func(t *testing.T) { @@ -108,6 +108,6 @@ func TestSimulateTx_Default(t *testing.T) { Data: []byte("0x00"), } sendErr := client.SimulateTransaction(ctx, ethClient, logger.TestSugared(t), "", msg) - require.Equal(t, false, sendErr.IsOutOfCounters(nil)) + require.Equal(t, false, sendErr.IsTerminallyStuckConfigError(nil)) }) } diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index caba4d3806c..dcf15a4fa23 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -8,7 +8,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" - evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/forwarders" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" @@ -26,7 +26,7 @@ func NewTxm( clientErrors config.ClientErrors, dbConfig DatabaseConfig, listenerConfig ListenerConfig, - client evmclient.Client, + client client.Client, lggr logger.Logger, logPoller logpoller.LogPoller, keyStore keystore.Eth, @@ -77,7 +77,7 @@ func NewEvmTxm( resender *Resender, tracker *Tracker, ) *Txm { - return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker) + return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, client.NewTxError) } // NewEvmResender creates a new concrete EvmResender diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index a3ae0a0a5db..6b107b222a6 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -3223,7 +3223,7 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { require.NoError(t, err) require.NotNil(t, dbTx) require.Equal(t, txmgrcommon.TxFatalError, dbTx.State) - require.Equal(t, "transaction terminally stuck", dbTx.Error.String) + require.Equal(t, client.TerminallyStuckMsg, dbTx.Error.String) }) } diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index ce64f816cd9..fd38eb7a8c9 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -1061,16 +1061,26 @@ func (o *evmTxStore) FindTxWithIdempotencyKey(ctx context.Context, idempotencyKe var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - var dbEtx DbEthTx - err = o.q.GetContext(ctx, &dbEtx, `SELECT * FROM evm.txes WHERE idempotency_key = $1 and evm_chain_id = $2`, idempotencyKey, chainID.String()) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil + err = o.Transact(ctx, true, func(orm *evmTxStore) error { + var dbEtx DbEthTx + err = o.q.GetContext(ctx, &dbEtx, `SELECT * FROM evm.txes WHERE idempotency_key = $1 and evm_chain_id = $2`, idempotencyKey, chainID.String()) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return pkgerrors.Wrap(err, "FindTxWithIdempotencyKey failed to load evm.txes") } - return nil, pkgerrors.Wrap(err, "FindTxWithIdempotencyKey failed to load evm.txes") - } - etx = new(Tx) - dbEtx.ToTx(etx) + etx = new(Tx) + dbEtx.ToTx(etx) + etxArr := []*Tx{etx} + if err = orm.LoadTxesAttempts(ctx, etxArr); err != nil { + return fmt.Errorf("FindTxWithIdempotencyKey failed to load evm.tx_attempts: %w", err) + } + if err = orm.loadEthTxesAttemptsReceipts(ctx, etxArr); err != nil { + return fmt.Errorf("FindTxWithIdempotencyKey failed to load evm.receipts: %w", err) + } + return nil + }) return } diff --git a/core/chains/evm/txmgr/evm_tx_store_test.go b/core/chains/evm/txmgr/evm_tx_store_test.go index 71b1bd52851..afb8de4ca52 100644 --- a/core/chains/evm/txmgr/evm_tx_store_test.go +++ b/core/chains/evm/txmgr/evm_tx_store_test.go @@ -698,7 +698,7 @@ func Test_FindTxWithIdempotencyKey(t *testing.T) { ethKeyStore := cltest.NewKeyStore(t, db).Eth() _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - t.Run("returns nil if no results", func(t *testing.T) { + t.Run("returns nil error if no results", func(t *testing.T) { idempotencyKey := "777" etx, err := txStore.FindTxWithIdempotencyKey(tests.Context(t), idempotencyKey, big.NewInt(0)) require.NoError(t, err) diff --git a/core/chains/evm/txmgr/stuck_tx_detector.go b/core/chains/evm/txmgr/stuck_tx_detector.go index 23cb9b2698a..baadda8a5f9 100644 --- a/core/chains/evm/txmgr/stuck_tx_detector.go +++ b/core/chains/evm/txmgr/stuck_tx_detector.go @@ -18,6 +18,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/common/txmgr" "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" ) @@ -363,13 +364,6 @@ func (d *stuckTxDetector) SetPurgeBlockNum(fromAddress common.Address, blockNum } func (d *stuckTxDetector) StuckTxFatalError() *string { - var errorMsg string - switch d.chainType { - case chaintype.ChainScroll, chaintype.ChainZkEvm: - errorMsg = "transaction skipped by chain" - default: - errorMsg = "transaction terminally stuck" - } - + errorMsg := client.TerminallyStuckMsg return &errorMsg } diff --git a/core/chains/evm/txmgr/txmgr_test.go b/core/chains/evm/txmgr/txmgr_test.go index de5847dc715..dac0ef29cd8 100644 --- a/core/chains/evm/txmgr/txmgr_test.go +++ b/core/chains/evm/txmgr/txmgr_test.go @@ -22,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" commonutils "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" @@ -33,6 +34,7 @@ import ( evmconfig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/forwarders" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + gasmocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" ksmocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" @@ -586,6 +588,197 @@ func TestTxm_Reset(t *testing.T) { }) } +func TestTxm_GetTransactionStatus(t *testing.T) { + t.Parallel() + + ctx := tests.Context(t) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + feeLimit := uint64(10_000) + gcfg := configtest.NewTestGeneralConfig(t) + cfg := evmtest.NewChainScopedConfig(t, gcfg) + + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + ethClient.On("PendingNonceAt", mock.Anything, mock.Anything).Return(uint64(0), nil).Maybe() + feeEstimator := gasmocks.NewEvmFeeEstimator(t) + feeEstimator.On("Start", mock.Anything).Return(nil).Once() + feeEstimator.On("OnNewLongestChain", mock.Anything, mock.Anything).Once() + txm, err := makeTestEvmTxm(t, db, ethClient, feeEstimator, cfg.EVM(), cfg.EVM().GasEstimator(), cfg.EVM().Transactions(), gcfg.Database(), gcfg.Database().Listener(), ethKeyStore) + require.NoError(t, err) + err = txm.Start(ctx) + require.NoError(t, err) + + head := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 100, + Parent: &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 99, + IsFinalized: true, + }, + } + txm.OnNewLongestChain(ctx, head) + + t.Run("returns error if transaction not found", func(t *testing.T) { + idempotencyKey := uuid.New().String() + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.Error(t, err, fmt.Sprintf("failed to find transaction with IdempotencyKey: %s", idempotencyKey)) + require.Equal(t, commontypes.Unknown, state) + }) + + t.Run("returns unknown for unstarted state", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + tx := &txmgr.Tx{ + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxUnstarted, + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.NoError(t, err) + require.Equal(t, commontypes.Unknown, state) + }) + + t.Run("returns unknown for in-progress state", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + nonce := evmtypes.Nonce(0) + tx := &txmgr.Tx{ + Sequence: &nonce, + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxInProgress, + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.NoError(t, err) + require.Equal(t, commontypes.Unknown, state) + }) + + t.Run("returns unconfirmed for unconfirmed state", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + nonce := evmtypes.Nonce(0) + broadcast := time.Now() + tx := &txmgr.Tx{ + Sequence: &nonce, + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxUnconfirmed, + BroadcastAt: &broadcast, + InitialBroadcastAt: &broadcast, + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.NoError(t, err) + require.Equal(t, commontypes.Unconfirmed, state) + }) + + t.Run("returns unconfirmed for confirmed state", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + nonce := evmtypes.Nonce(0) + broadcast := time.Now() + tx := &txmgr.Tx{ + Sequence: &nonce, + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxConfirmed, + BroadcastAt: &broadcast, + InitialBroadcastAt: &broadcast, + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + tx, err = txStore.FindTxWithIdempotencyKey(ctx, idempotencyKey, testutils.FixtureChainID) + require.NoError(t, err) + attempt := cltest.NewLegacyEthTxAttempt(t, tx.ID) + err = txStore.InsertTxAttempt(ctx, &attempt) + require.NoError(t, err) + // Insert receipt for finalized block num + mustInsertEthReceipt(t, txStore, head.Parent.Number, head.ParentHash, attempt.Hash) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.NoError(t, err) + require.Equal(t, commontypes.Unconfirmed, state) + }) + + t.Run("returns unconfirmed for confirmed missing receipt state", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + nonce := evmtypes.Nonce(0) + broadcast := time.Now() + tx := &txmgr.Tx{ + Sequence: &nonce, + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxConfirmedMissingReceipt, + BroadcastAt: &broadcast, + InitialBroadcastAt: &broadcast, + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.NoError(t, err) + require.Equal(t, commontypes.Unconfirmed, state) + }) + + t.Run("returns fatal for fatal error state with terminally stuck error", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + nonce := evmtypes.Nonce(0) + broadcast := time.Now() + tx := &txmgr.Tx{ + Sequence: &nonce, + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxFatalError, + Error: null.NewString(evmclient.TerminallyStuckMsg, true), + BroadcastAt: &broadcast, + InitialBroadcastAt: &broadcast, + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.Equal(t, commontypes.Fatal, state) + require.Error(t, err, evmclient.TerminallyStuckMsg) + }) + + t.Run("returns failed for fatal error state with other error", func(t *testing.T) { + idempotencyKey := uuid.New().String() + _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) + errorMsg := "something went wrong" + tx := &txmgr.Tx{ + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: feeLimit, + State: txmgrcommon.TxFatalError, + Error: null.NewString(errorMsg, true), + } + err := txStore.InsertTx(ctx, tx) + require.NoError(t, err) + state, err := txm.GetTransactionStatus(ctx, idempotencyKey) + require.Equal(t, commontypes.Failed, state) + require.Error(t, err, errorMsg) + }) +} + func newTxStore(t *testing.T, db *sqlx.DB) txmgr.EvmTxStore { return txmgr.NewTxStore(db, logger.Test(t)) }