Merge remote-tracking branch 'origin/llo-feeds-config-count-from-sour…

…ce' into llo-feeds-config-count-from-source

.github/workflows/solidity-foundry.yml

@@ -23,6 +23,9 @@ jobs:
               - 'contracts/src/v0.8/**/*'
               - '.github/workflows/solidity-foundry.yml'
               - 'contracts/foundry.toml'
+              - 'contracts/gas-snapshots/*.gas-snapshot'
+              - '.gitmodules'
+              - 'contracts/foundry-lib'
 
   tests:
     strategy:

common/txmgr/txmgr.go

@@ -2,6 +2,7 @@ package txmgr
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"math/big"
 	"sync"
@@ -435,6 +436,20 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Trigger(ad
 
 // CreateTransaction inserts a new transaction
 func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], qs ...pg.QOpt) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) {
+	// Check for existing Tx with IdempotencyKey. If found, return the Tx and do nothing
+	// Skipping CreateTransaction to avoid double send
+	if txRequest.IdempotencyKey != nil {
+		var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]
+		existingTx, err = b.txStore.FindTxWithIdempotencyKey(*txRequest.IdempotencyKey, b.chainID)
+		if err != nil && !errors.Is(err, sql.ErrNoRows) {
+			return tx, errors.Wrap(err, "Failed to search for transaction with IdempotencyKey")
+		}
+		if existingTx != nil {
+			b.logger.Infow("Found a Tx with IdempotencyKey. Returning existing Tx without creating a new one.", "IdempotencyKey", *txRequest.IdempotencyKey)
+			return *existingTx, nil
+		}
+	}
+
 	if err = b.checkEnabled(txRequest.FromAddress); err != nil {
 		return tx, err
 	}

common/txmgr/types/tx.go

@@ -66,6 +66,14 @@ func (s TxAttemptState) String() (str string) {
 }
 
 type TxRequest[ADDR types.Hashable, TX_HASH types.Hashable] struct {
+	// IdempotencyKey is a globally unique ID set by the caller, to prevent accidental creation of duplicated Txs during retries or crash recovery.
+	// If this field is set, the TXM will first search existing Txs with this field.
+	// If found, it will return the existing Tx, without creating a new one. TXM will not validate or ensure that existing Tx is same as the incoming TxRequest.
+	// If not found, TXM will create a new Tx.
+	// If IdempotencyKey is set to null, TXM will always create a new Tx.
+	// Since IdempotencyKey has to be globally unique, consider prepending the service or component's name it is being used by
+	// Such as {service}-{ID}. E.g vrf-12345
+	IdempotencyKey   *string
 	FromAddress      ADDR
 	ToAddress        ADDR
 	EncodedPayload   []byte
@@ -178,6 +186,7 @@ type Tx[
 	FEE feetypes.Fee,
 ] struct {
 	ID             int64
+	IdempotencyKey *string
 	Sequence       *SEQ
 	FromAddress    ADDR
 	ToAddress      ADDR

common/txmgr/types/tx_store.go

@@ -64,6 +64,9 @@ type TransactionStore[
 	FindTxAttemptsConfirmedMissingReceipt(chainID CHAIN_ID) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error)
 	FindTxAttemptsRequiringReceiptFetch(chainID CHAIN_ID) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error)
 	FindTxAttemptsRequiringResend(olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error)
+	// Search for Tx using the idempotencyKey and chainID
+	FindTxWithIdempotencyKey(idempotencyKey string, chainID CHAIN_ID) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error)
+	// Search for Tx using the fromAddress and sequence
 	FindTxWithSequence(fromAddress ADDR, seq SEQ) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error)
 	FindNextUnstartedTransactionFromAddress(etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID, qopts ...pg.QOpt) error
 	FindTransactionsConfirmedInBlockRange(highBlockNumber, lowBlockNumber int64, chainID CHAIN_ID) (etxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error)

contracts/gas-snapshots/automation-dev.gas-snapshot

@@ -1,5 +1,5 @@
 AutomationForwarder_forward:testBasicSuccess() (gas: 87630)
-AutomationForwarder_forward:testNotAuthorizedReverts() (gas: 21681)
+AutomationForwarder_forward:testNotAuthorizedReverts() (gas: 25427)
 AutomationForwarder_forward:testWrongFunctionSelectorSuccess() (gas: 17958)
 AutomationForwarder_updateRegistry:testBasicSuccess() (gas: 14577)
-AutomationForwarder_updateRegistry:testNotFromRegistryNotAuthorizedReverts() (gas: 13893)
+AutomationForwarder_updateRegistry:testNotFromRegistryNotAuthorizedReverts() (gas: 17665)

contracts/gas-snapshots/functions.gas-snapshot

@@ -1,17 +1,17 @@
-FunctionsOracle_sendRequest:testEmptyRequestDataReverts() (gas: 13430)
+FunctionsOracle_sendRequest:testEmptyRequestDataReverts() (gas: 13452)
 FunctionsOracle_setDONPublicKey:testEmptyPublicKeyReverts() (gas: 10974)
 FunctionsOracle_setDONPublicKey:testOnlyOwnerReverts() (gas: 11255)
 FunctionsOracle_setDONPublicKey:testSetDONPublicKeySuccess() (gas: 126453)
-FunctionsOracle_setDONPublicKey:testSetDONPublicKey_gas() (gas: 97558)
+FunctionsOracle_setDONPublicKey:testSetDONPublicKey_gas() (gas: 97580)
 FunctionsOracle_setRegistry:testEmptyPublicKeyReverts() (gas: 10635)
 FunctionsOracle_setRegistry:testOnlyOwnerReverts() (gas: 10927)
 FunctionsOracle_setRegistry:testSetRegistrySuccess() (gas: 35791)
 FunctionsOracle_setRegistry:testSetRegistry_gas() (gas: 31987)
 FunctionsOracle_typeAndVersion:testTypeAndVersionSuccess() (gas: 6905)
-FunctionsRouter_Pause:test_Pause_RevertIfNotOwner() (gas: 13293)
-FunctionsRouter_Pause:test_Pause_Success() (gas: 20205)
-FunctionsRouter_Unpause:test_Unpause_RevertIfNotOwner() (gas: 13338)
-FunctionsRouter_Unpause:test_Unpause_Success() (gas: 77279)
-FunctionsSubscriptions_createSubscription:test_CreateSubscription_RevertIfNotAllowedSender() (gas: 26347)
-FunctionsSubscriptions_createSubscription:test_CreateSubscription_RevertIfPaused() (gas: 15699)
+FunctionsRouter_Pause:test_Pause_RevertIfNotOwner() (gas: 13315)
+FunctionsRouter_Pause:test_Pause_Success() (gas: 20254)
+FunctionsRouter_Unpause:test_Unpause_RevertIfNotOwner() (gas: 13294)
+FunctionsRouter_Unpause:test_Unpause_Success() (gas: 77334)
+FunctionsSubscriptions_createSubscription:test_CreateSubscription_RevertIfNotAllowedSender() (gas: 26368)
+FunctionsSubscriptions_createSubscription:test_CreateSubscription_RevertIfPaused() (gas: 15714)
 FunctionsSubscriptions_createSubscription:test_CreateSubscription_Success() (gas: 152436)

contracts/gas-snapshots/shared.gas-snapshot

@@ -1,32 +1,32 @@
-BurnMintERC677_approve:testApproveSuccess() (gas: 52254)
-BurnMintERC677_approve:testInvalidAddressReverts() (gas: 10641)
-BurnMintERC677_burn:testBasicBurnSuccess() (gas: 164344)
-BurnMintERC677_burn:testBurnFromZeroAddressReverts() (gas: 43462)
-BurnMintERC677_burn:testExceedsBalanceReverts() (gas: 18035)
+BurnMintERC677_approve:testApproveSuccess() (gas: 55248)
+BurnMintERC677_approve:testInvalidAddressReverts() (gas: 10663)
+BurnMintERC677_burn:testBasicBurnSuccess() (gas: 164342)
+BurnMintERC677_burn:testBurnFromZeroAddressReverts() (gas: 47201)
+BurnMintERC677_burn:testExceedsBalanceReverts() (gas: 21841)
 BurnMintERC677_burn:testSenderNotBurnerReverts() (gas: 13359)
-BurnMintERC677_burnFrom:testBurnFromSuccess() (gas: 54662)
-BurnMintERC677_burnFrom:testExceedsBalanceReverts() (gas: 32873)
-BurnMintERC677_burnFrom:testInsufficientAllowanceReverts() (gas: 18107)
+BurnMintERC677_burnFrom:testBurnFromSuccess() (gas: 57658)
+BurnMintERC677_burnFrom:testExceedsBalanceReverts() (gas: 35864)
+BurnMintERC677_burnFrom:testInsufficientAllowanceReverts() (gas: 21849)
 BurnMintERC677_burnFrom:testSenderNotBurnerReverts() (gas: 13359)
-BurnMintERC677_burnFromAlias:testBurnFromSuccess() (gas: 54688)
-BurnMintERC677_burnFromAlias:testExceedsBalanceReverts() (gas: 32889)
-BurnMintERC677_burnFromAlias:testInsufficientAllowanceReverts() (gas: 18127)
+BurnMintERC677_burnFromAlias:testBurnFromSuccess() (gas: 57684)
+BurnMintERC677_burnFromAlias:testExceedsBalanceReverts() (gas: 35880)
+BurnMintERC677_burnFromAlias:testInsufficientAllowanceReverts() (gas: 21869)
 BurnMintERC677_burnFromAlias:testSenderNotBurnerReverts() (gas: 13379)
 BurnMintERC677_constructor:testConstructorSuccess() (gas: 1669109)
-BurnMintERC677_decreaseApproval:testDecreaseApprovalSuccess() (gas: 28520)
-BurnMintERC677_grantMintAndBurnRoles:testGrantMintAndBurnRolesSuccess() (gas: 120049)
-BurnMintERC677_grantRole:testGrantBurnAccessSuccess() (gas: 52707)
+BurnMintERC677_decreaseApproval:testDecreaseApprovalSuccess() (gas: 28537)
+BurnMintERC677_grantMintAndBurnRoles:testGrantMintAndBurnRolesSuccess() (gas: 120071)
+BurnMintERC677_grantRole:testGrantBurnAccessSuccess() (gas: 52724)
 BurnMintERC677_grantRole:testGrantManySuccess() (gas: 935521)
-BurnMintERC677_grantRole:testGrantMintAccessSuccess() (gas: 93588)
+BurnMintERC677_grantRole:testGrantMintAccessSuccess() (gas: 93605)
 BurnMintERC677_increaseApproval:testIncreaseApprovalSuccess() (gas: 40911)
 BurnMintERC677_mint:testBasicMintSuccess() (gas: 149365)
-BurnMintERC677_mint:testMaxSupplyExceededReverts() (gas: 46627)
+BurnMintERC677_mint:testMaxSupplyExceededReverts() (gas: 50385)
 BurnMintERC677_mint:testSenderNotMinterReverts() (gas: 11195)
 BurnMintERC677_supportsInterface:testConstructorSuccess() (gas: 8685)
-BurnMintERC677_transfer:testInvalidAddressReverts() (gas: 10617)
+BurnMintERC677_transfer:testInvalidAddressReverts() (gas: 10639)
 BurnMintERC677_transfer:testTransferSuccess() (gas: 39462)
 OpStackBurnMintERC677_constructor:testConstructorSuccess() (gas: 1739317)
-OpStackBurnMintERC677_interfaceCompatibility:testBurnCompatibility() (gas: 259440)
-OpStackBurnMintERC677_interfaceCompatibility:testMintCompatibility() (gas: 137935)
+OpStackBurnMintERC677_interfaceCompatibility:testBurnCompatibility() (gas: 263373)
+OpStackBurnMintERC677_interfaceCompatibility:testMintCompatibility() (gas: 137957)
 OpStackBurnMintERC677_interfaceCompatibility:testStaticFunctionsCompatibility() (gas: 10622)
 OpStackBurnMintERC677_supportsInterface:testConstructorSuccess() (gas: 8961)

contracts/gas-snapshots/vrf.gas-snapshot

@@ -1,18 +1,18 @@
 TrustedBlockhashStoreTest:testGenericBHSFunctions() (gas: 53507)
-TrustedBlockhashStoreTest:testTrustedBHSFunctions() (gas: 49536)
+TrustedBlockhashStoreTest:testTrustedBHSFunctions() (gas: 54617)
 VRFCoordinatorV2Plus_Migration:testDeregister() (gas: 101083)
-VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenInvalidCaller() (gas: 29421)
-VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenInvalidCoordinator() (gas: 19855)
-VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenPendingFulfillment() (gas: 237702)
-VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenReentrant() (gas: 354749)
-VRFCoordinatorV2Plus_Migration:testMigration() (gas: 471631)
-VRFCoordinatorV2Plus_Migration:testMigrationNoLink() (gas: 431052)
-VRFV2Plus:testCancelSubWithNoLink() (gas: 160261)
+VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenInvalidCaller() (gas: 33190)
+VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenInvalidCoordinator() (gas: 19877)
+VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenPendingFulfillment() (gas: 237679)
+VRFCoordinatorV2Plus_Migration:testMigrateRevertsWhenReentrant() (gas: 357765)
+VRFCoordinatorV2Plus_Migration:testMigration() (gas: 471605)
+VRFCoordinatorV2Plus_Migration:testMigrationNoLink() (gas: 431075)
+VRFV2Plus:testCancelSubWithNoLink() (gas: 160279)
 VRFV2Plus:testCreateSubscription() (gas: 181127)
-VRFV2Plus:testGetActiveSubscriptionIds() (gas: 3453659)
-VRFV2Plus:testRegisterProvingKey() (gas: 101025)
-VRFV2Plus:testRequestAndFulfillRandomWordsLINK() (gas: 755178)
-VRFV2Plus:testRequestAndFulfillRandomWordsNative() (gas: 705581)
+VRFV2Plus:testGetActiveSubscriptionIds() (gas: 3453681)
+VRFV2Plus:testRegisterProvingKey() (gas: 101085)
+VRFV2Plus:testRequestAndFulfillRandomWordsLINK() (gas: 755144)
+VRFV2Plus:testRequestAndFulfillRandomWordsNative() (gas: 705591)
 VRFV2Plus:testSetConfig() (gas: 73032)
-VRFV2PlusWrapperTest:testRequestAndFulfillRandomWordsLINKWrapper() (gas: 390063)
-VRFV2PlusWrapperTest:testRequestAndFulfillRandomWordsNativeWrapper() (gas: 290612)
+VRFV2PlusWrapperTest:testRequestAndFulfillRandomWordsLINKWrapper() (gas: 393885)
+VRFV2PlusWrapperTest:testRequestAndFulfillRandomWordsNativeWrapper() (gas: 294434)

core/chains/evm/txmgr/evm_tx_store.go

@@ -152,6 +152,7 @@ func toOnchainReceipt(rs []*evmtypes.Receipt) []rawOnchainReceipt {
 // This is exported, as tests and other external code still directly reads DB using this schema.
 type DbEthTx struct {
 	ID             int64
+	IdempotencyKey *string
 	Nonce          *int64
 	FromAddress    common.Address
 	ToAddress      common.Address
@@ -218,6 +219,7 @@ func DbEthTxToEthTx(dbEthTx DbEthTx, evmEthTx *Tx) {
 		n := evmtypes.Nonce(*dbEthTx.Nonce)
 		evmEthTx.Sequence = &n
 	}
+	evmEthTx.IdempotencyKey = dbEthTx.IdempotencyKey
 	evmEthTx.FromAddress = dbEthTx.FromAddress
 	evmEthTx.ToAddress = dbEthTx.ToAddress
 	evmEthTx.EncodedPayload = dbEthTx.EncodedPayload
@@ -906,6 +908,21 @@ func (o *evmTxStore) FindReceiptsPendingConfirmation(ctx context.Context, blockN
 	return
 }
 
+// FindTxWithIdempotencyKey returns any broadcast ethtx with the given idempotencyKey and chainID
+func (o *evmTxStore) FindTxWithIdempotencyKey(idempotencyKey string, chainID *big.Int) (etx *Tx, err error) {
+	var dbEtx DbEthTx
+	err = o.q.Get(&dbEtx, `SELECT * FROM eth_txes WHERE idempotency_key = $1 and evm_chain_id = $2`, idempotencyKey, chainID.String())
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil, nil
+		}
+		return nil, pkgerrors.Wrap(err, "FindTxWithIdempotencyKey failed to load eth_txes")
+	}
+	etx = new(Tx)
+	DbEthTxToEthTx(dbEtx, etx)
+	return
+}
+
 // FindTxWithSequence returns any broadcast ethtx with the given nonce
 func (o *evmTxStore) FindTxWithSequence(fromAddress common.Address, nonce evmtypes.Nonce) (etx *Tx, err error) {
 	etx = new(Tx)
@@ -1501,12 +1518,12 @@ func (o *evmTxStore) CreateTransaction(txRequest TxRequest, chainID *big.Int, qo
 			}
 		}
 		err = tx.Get(&dbEtx, `
-INSERT INTO eth_txes (from_address, to_address, encoded_payload, value, gas_limit, state, created_at, meta, subject, evm_chain_id, min_confirmations, pipeline_task_run_id, transmit_checker)
+INSERT INTO eth_txes (from_address, to_address, encoded_payload, value, gas_limit, state, created_at, meta, subject, evm_chain_id, min_confirmations, pipeline_task_run_id, transmit_checker, idempotency_key)
 VALUES (
-$1,$2,$3,$4,$5,'unstarted',NOW(),$6,$7,$8,$9,$10,$11
+$1,$2,$3,$4,$5,'unstarted',NOW(),$6,$7,$8,$9,$10,$11,$12
 )
 RETURNING "eth_txes".*
-`, txRequest.FromAddress, txRequest.ToAddress, txRequest.EncodedPayload, assets.Eth(txRequest.Value), txRequest.FeeLimit, txRequest.Meta, txRequest.Strategy.Subject(), chainID.String(), txRequest.MinConfirmations, txRequest.PipelineTaskRunID, txRequest.Checker)
+`, txRequest.FromAddress, txRequest.ToAddress, txRequest.EncodedPayload, assets.Eth(txRequest.Value), txRequest.FeeLimit, txRequest.Meta, txRequest.Strategy.Subject(), chainID.String(), txRequest.MinConfirmations, txRequest.PipelineTaskRunID, txRequest.Checker, txRequest.IdempotencyKey)
 		if err != nil {
 			return pkgerrors.Wrap(err, "CreateEthTransaction failed to insert eth_tx")
 		}

core/chains/evm/txmgr/evm_tx_store_test.go

@@ -666,6 +666,35 @@ func TestORM_FindReceiptsPendingConfirmation(t *testing.T) {
 	assert.Equal(t, tr.ID, receiptsPlus[0].ID)
 }
 
+func Test_FindTxWithIdempotencyKey(t *testing.T) {
+	t.Parallel()
+	db := pgtest.NewSqlxDB(t)
+	cfg := newTestChainScopedConfig(t)
+	txStore := cltest.NewTestTxStore(t, db, cfg.Database())
+	ethKeyStore := cltest.NewKeyStore(t, db, cfg.Database()).Eth()
+	_, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore, 0)
+
+	t.Run("returns nil if no results", func(t *testing.T) {
+		idempotencyKey := "777"
+		etx, err := txStore.FindTxWithIdempotencyKey(idempotencyKey, big.NewInt(0))
+		require.NoError(t, err)
+		assert.Nil(t, etx)
+	})
+
+	t.Run("returns transaction if it exists", func(t *testing.T) {
+		idempotencyKey := "777"
+		cfg.EVM().ChainID()
+		etx := cltest.MustCreateUnstartedGeneratedTx(t, txStore, fromAddress, big.NewInt(0),
+			cltest.EvmTxRequestWithIdempotencyKey(idempotencyKey))
+		require.Equal(t, idempotencyKey, *etx.IdempotencyKey)
+
+		res, err := txStore.FindTxWithIdempotencyKey(idempotencyKey, big.NewInt(0))
+		require.NoError(t, err)
+		assert.Equal(t, etx.Sequence, res.Sequence)
+		require.Equal(t, idempotencyKey, *res.IdempotencyKey)
+	})
+}
+
 func TestORM_FindTxWithSequence(t *testing.T) {
 	t.Parallel()