feat: add timeout when waiting for transaction to be minted (#621)

* feat: add timeout when waiting for transaction to be minted

* feat: expose the evm retry timeout as flag

benchmark/main.go

@@ -111,7 +111,7 @@ func DeployContractAndSubmitDataCommitment() error {
 		return err
 	}
 
-	receipt, err := evmClient.WaitForTransaction(ctx, backend, tx)
+	receipt, err := evmClient.WaitForTransaction(ctx, backend, tx, time.Minute)
 	if err == nil && receipt != nil && receipt.Status == 1 {
 		logger.Info("deployed Blobstream contract", "proxy_address", address.Hex(), "tx_hash", tx.Hash().String())
 	}

cmd/blobstream/base/config.go

@@ -76,6 +76,7 @@ const (
 	FlagEVMRPC             = "evm.rpc"
 	FlagEVMGasLimit        = "evm.gas-limit"
 	FlagEVMContractAddress = "evm.contract-address"
+	FlagEVMRetryTimeout    = "evm.retry-timeout"
 
 	FlagCoreGRPC = "core.grpc"
 	FlagCoreRPC  = "core.rpc"
@@ -337,3 +338,20 @@ func GetLogger(level string, format string) (tmlog.Logger, error) {
 
 	return server.ZeroLogWrapper{Logger: zerolog.New(logWriter).Level(logLvl).With().Timestamp().Logger()}, nil
 }
+
+func GetEVMRetryTimeoutFlag(cmd *cobra.Command) (uint64, bool, error) {
+	changed := cmd.Flags().Changed(FlagEVMRetryTimeout)
+	val, err := cmd.Flags().GetUint64(FlagEVMRetryTimeout)
+	if err != nil {
+		return 0, changed, err
+	}
+	return val, changed, nil
+}
+
+func AddEVMRetryTimeoutFlag(cmd *cobra.Command) {
+	cmd.Flags().Uint64(
+		FlagEVMRetryTimeout,
+		15,
+		"The time, in minutes, to wait for transactions to be mined on the target EVM chain before recreating them with a different gas price",
+	)
+}

cmd/blobstream/deploy/cmd.go

@@ -3,6 +3,7 @@ package deploy
 import (
 	"context"
 	"strconv"
+	"time"
 
 	"github.com/celestiaorg/orchestrator-relayer/cmd/blobstream/base"
 
@@ -138,7 +139,7 @@ func Command() *cobra.Command {
 				return err
 			}
 
-			receipt, err := evmClient.WaitForTransaction(cmd.Context(), backend, tx)
+			receipt, err := evmClient.WaitForTransaction(cmd.Context(), backend, tx, 5*time.Minute)
 			if err == nil && receipt != nil && receipt.Status == 1 {
 				logger.Info("deployed Blobstream contract", "proxy_address", address.Hex(), "tx_hash", tx.Hash().String())
 			}

cmd/blobstream/relayer/cmd.go

@@ -201,6 +201,7 @@ func Start() *cobra.Command {
 				logger,
 				retrier,
 				s.SignatureStore,
+				time.Duration(config.EVMRetryTimeout)*time.Minute,
 			)
 
 			// Listen for and trap any OS signal to graceful shutdown and exit

cmd/blobstream/relayer/config.go

@@ -63,6 +63,10 @@ contract-address = "{{ .ContractAddr }}"
 
 # Evm gas limit.
 gas-limit = "{{ .EvmGasLimit }}"
+
+# The time, in minutes, to wait for transactions to be mined
+# on the target EVM chain before recreating them with a different gas price.
+retry-timeout = "{{ .EVMRetryTimeout }}"
 `
 
 func addRelayerStartFlags(cmd *cobra.Command) *cobra.Command {
@@ -85,37 +89,40 @@ func addRelayerStartFlags(cmd *cobra.Command) *cobra.Command {
 	base.AddGRPCInsecureFlag(cmd)
 	base.AddLogLevelFlag(cmd)
 	base.AddLogFormatFlag(cmd)
+	base.AddEVMRetryTimeoutFlag(cmd)
 
 	return cmd
 }
 
 type StartConfig struct {
 	base.Config
-	EvmChainID    uint64 `mapstructure:"evm-chain-id" json:"evm-chain-id"`
-	EvmRPC        string `mapstructure:"evm-rpc" json:"evm-rpc"`
-	CoreGRPC      string `mapstructure:"core-grpc" json:"core-grpc"`
-	CoreRPC       string `mapstructure:"core-rpc" json:"core-rpc"`
-	evmAccAddress string
-	ContractAddr  string `mapstructure:"contract-address" json:"contract-address"`
-	EvmGasLimit   uint64 `mapstructure:"gas-limit" json:"gas-limit"`
-	Bootstrappers string `mapstructure:"bootstrappers" json:"bootstrappers"`
-	P2PListenAddr string `mapstructure:"listen-addr" json:"listen-addr"`
-	p2pNickname   string
-	GrpcInsecure  bool `mapstructure:"grpc-insecure" json:"grpc-insecure"`
-	LogLevel      string
-	LogFormat     string
+	EvmChainID      uint64 `mapstructure:"evm-chain-id" json:"evm-chain-id"`
+	EvmRPC          string `mapstructure:"evm-rpc" json:"evm-rpc"`
+	CoreGRPC        string `mapstructure:"core-grpc" json:"core-grpc"`
+	CoreRPC         string `mapstructure:"core-rpc" json:"core-rpc"`
+	evmAccAddress   string
+	ContractAddr    string `mapstructure:"contract-address" json:"contract-address"`
+	EvmGasLimit     uint64 `mapstructure:"gas-limit" json:"gas-limit"`
+	Bootstrappers   string `mapstructure:"bootstrappers" json:"bootstrappers"`
+	P2PListenAddr   string `mapstructure:"listen-addr" json:"listen-addr"`
+	p2pNickname     string
+	GrpcInsecure    bool `mapstructure:"grpc-insecure" json:"grpc-insecure"`
+	LogLevel        string
+	LogFormat       string
+	EVMRetryTimeout uint64 `mapstructure:"retry-timeout" json:"retry-timeout"`
 }
 
 func DefaultStartConfig() *StartConfig {
 	return &StartConfig{
-		CoreRPC:       "tcp://localhost:26657",
-		CoreGRPC:      "localhost:9090",
-		Bootstrappers: "",
-		P2PListenAddr: "/ip4/0.0.0.0/tcp/30000",
-		GrpcInsecure:  true,
-		EvmChainID:    5,
-		EvmRPC:        "http://localhost:8545",
-		EvmGasLimit:   2500000,
+		CoreRPC:         "tcp://localhost:26657",
+		CoreGRPC:        "localhost:9090",
+		Bootstrappers:   "",
+		P2PListenAddr:   "/ip4/0.0.0.0/tcp/30000",
+		GrpcInsecure:    true,
+		EvmChainID:      5,
+		EvmRPC:          "http://localhost:8545",
+		EvmGasLimit:     2500000,
+		EVMRetryTimeout: 15,
 	}
 }
 
@@ -241,6 +248,14 @@ func parseRelayerStartFlags(cmd *cobra.Command, fileConfig *StartConfig) (StartC
 	}
 	fileConfig.LogFormat = logFormat
 
+	retryTimeout, changed, err := base.GetEVMRetryTimeoutFlag(cmd)
+	if err != nil {
+		return StartConfig{}, err
+	}
+	if changed {
+		fileConfig.EVMRetryTimeout = retryTimeout
+	}
+
 	return *fileConfig, nil
 }
 

evm/evm_client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -228,9 +229,13 @@ func (ec *Client) WaitForTransaction(
 	ctx context.Context,
 	backend bind.DeployBackend,
 	tx *coregethtypes.Transaction,
+	timeout time.Duration,
 ) (*coregethtypes.Receipt, error) {
 	ec.logger.Debug("waiting for transaction to be confirmed", "hash", tx.Hash().String())
 
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
 	receipt, err := bind.WaitMined(ctx, backend, tx)
 	if err == nil && receipt != nil && receipt.Status == 1 {
 		ec.logger.Info("transaction confirmed", "hash", tx.Hash().String(), "block", receipt.BlockNumber.Uint64())

relayer/relayer.go

@@ -38,6 +38,7 @@ type Relayer struct {
 	logger         tmlog.Logger
 	Retrier        *helpers.Retrier
 	SignatureStore *badger.Datastore
+	RetryTimeout   time.Duration
 }
 
 func NewRelayer(
@@ -48,6 +49,7 @@ func NewRelayer(
 	logger tmlog.Logger,
 	retrier *helpers.Retrier,
 	sigStore *badger.Datastore,
+	retryTimeout time.Duration,
 ) *Relayer {
 	return &Relayer{
 		TmQuerier:      tmQuerier,
@@ -57,6 +59,7 @@ func NewRelayer(
 		logger:         logger,
 		Retrier:        retrier,
 		SignatureStore: sigStore,
+		RetryTimeout:   retryTimeout,
 	}
 }
 
@@ -111,7 +114,7 @@ func (r *Relayer) Start(ctx context.Context) error {
 				}
 
 				// wait for transaction to be mined
-				_, err = r.EVMClient.WaitForTransaction(ctx, ethClient, tx)
+				_, err = r.EVMClient.WaitForTransaction(ctx, ethClient, tx, r.RetryTimeout)
 				if err != nil {
 					return err
 				}

relayer/relayer_test.go

@@ -40,7 +40,7 @@ func (s *RelayerTestSuite) TestProcessAttestation() {
 
 	tx, err := s.Relayer.ProcessAttestation(ctx, s.Node.EVMChain.Auth, att)
 	require.NoError(t, err)
-	receipt, err := s.Relayer.EVMClient.WaitForTransaction(ctx, s.Node.EVMChain.Backend, tx)
+	receipt, err := s.Relayer.EVMClient.WaitForTransaction(ctx, s.Node.EVMChain.Backend, tx, 20*time.Second)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(1), receipt.Status)
 
@@ -113,7 +113,7 @@ func TestUseValsetFromP2P(t *testing.T) {
 	tx, err := relayer.ProcessAttestation(ctx, node.EVMChain.Auth, att)
 	require.NoError(t, err)
 
-	receipt, err := relayer.EVMClient.WaitForTransaction(ctx, node.EVMChain.Backend, tx)
+	receipt, err := relayer.EVMClient.WaitForTransaction(ctx, node.EVMChain.Backend, tx, 20*time.Second)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(1), receipt.Status)
 

testing/blobstream.go

@@ -49,7 +49,7 @@ func NewRelayer(
 	tempDir := t.TempDir()
 	sigStore, err := badger.NewDatastore(tempDir, store.DefaultBadgerOptions(tempDir))
 	require.NoError(t, err)
-	r := relayer.NewRelayer(tmQuerier, appQuerier, p2pQuerier, evmClient, logger, retrier, sigStore)
+	r := relayer.NewRelayer(tmQuerier, appQuerier, p2pQuerier, evmClient, logger, retrier, sigStore, 30*time.Second)
 	return r
 }