Add new getTransactions endpoint and database backend for transactions

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 
 ## Unreleased
-n/a
+
+### Added
+* Transactions will now be stored in a database rather than in memory ([#174](https://github.com/stellar/soroban-rpc/pull/174)).
+
+You can opt-in to longer transaction retention by setting `--transaction-retention-window` / `TRANSACTION_RETENTION_WINDOW` to a higher number of ledgers. This will also retain corresponding number of ledgers in the database. Keep in mind, of course, that this will cause an increase in disk usage for the growing database.
 
 
 ## [v21.2.0](https://github.com/stellar/soroban-rpc/compare/v21.1.0...v21.2.0)

cmd/soroban-rpc/internal/config/config.go

@@ -25,6 +25,7 @@ type Config struct {
 	CheckpointFrequency                         uint32
 	CoreRequestTimeout                          time.Duration
 	DefaultEventsLimit                          uint
+	DefaultTransactionsLimit                    uint
 	EventLedgerRetentionWindow                  uint32
 	FriendbotURL                                string
 	HistoryArchiveURLs                          []string
@@ -33,6 +34,7 @@ type Config struct {
 	LogFormat                                   LogFormat
 	LogLevel                                    logrus.Level
 	MaxEventsLimit                              uint
+	MaxTransactionsLimit                        uint
 	MaxHealthyLedgerLatency                     time.Duration
 	NetworkPassphrase                           string
 	PreflightWorkerCount                        uint
@@ -48,6 +50,7 @@ type Config struct {
 	RequestBacklogGetLatestLedgerQueueLimit     uint
 	RequestBacklogGetLedgerEntriesQueueLimit    uint
 	RequestBacklogGetTransactionQueueLimit      uint
+	RequestBacklogGetTransactionsQueueLimit     uint
 	RequestBacklogSendTransactionQueueLimit     uint
 	RequestBacklogSimulateTransactionQueueLimit uint
 	RequestExecutionWarningThreshold            time.Duration
@@ -59,6 +62,7 @@ type Config struct {
 	MaxGetLatestLedgerExecutionDuration         time.Duration
 	MaxGetLedgerEntriesExecutionDuration        time.Duration
 	MaxGetTransactionExecutionDuration          time.Duration
+	MaxGetTransactionsExecutionDuration         time.Duration
 	MaxSendTransactionExecutionDuration         time.Duration
 	MaxSimulateTransactionExecutionDuration     time.Duration
 

cmd/soroban-rpc/internal/config/options.go

@@ -211,18 +211,22 @@ func (cfg *Config) options() ConfigOptions {
 		},
 		{
 			Name: "event-retention-window",
-			Usage: fmt.Sprintf("configures the event retention window expressed in number of ledgers,"+
-				" the default value is %d which corresponds to about 24 hours of history", ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
+			Usage: fmt.Sprintf(
+				"configures the event retention window expressed in number of ledgers,"+
+					" the default value is %d which corresponds to about 24 hours of history",
+				ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
 			ConfigKey:    &cfg.EventLedgerRetentionWindow,
 			DefaultValue: uint32(ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
 			Validate:     positive,
 		},
 		{
 			Name: "transaction-retention-window",
-			Usage: "configures the transaction retention window expressed in number of ledgers," +
-				" the default value is 1440 which corresponds to about 2 hours of history",
+			Usage: fmt.Sprintf(
+				"configures the transaction retention window expressed in number of ledgers,"+
+					" the default value is %d which corresponds to about 24 hours of history",
+				ledgerbucketwindow.OneDayOfLedgers),
 			ConfigKey:    &cfg.TransactionLedgerRetentionWindow,
-			DefaultValue: uint32(1440),
+			DefaultValue: uint32(ledgerbucketwindow.OneDayOfLedgers),
 			Validate:     positive,
 		},
 		{
@@ -247,6 +251,28 @@ func (cfg *Config) options() ConfigOptions {
 				return nil
 			},
 		},
+		{
+			Name:         "max-transactions-limit",
+			Usage:        "Maximum amount of transactions allowed in a single getTransactions response",
+			ConfigKey:    &cfg.MaxTransactionsLimit,
+			DefaultValue: uint(200),
+		},
+		{
+			Name:         "default-transactions-limit",
+			Usage:        "Default cap on the amount of transactions included in a single getTransactions response",
+			ConfigKey:    &cfg.DefaultTransactionsLimit,
+			DefaultValue: uint(50),
+			Validate: func(co *ConfigOption) error {
+				if cfg.DefaultTransactionsLimit > cfg.MaxTransactionsLimit {
+					return fmt.Errorf(
+						"default-transactions-limit (%v) cannot exceed max-transactions-limit (%v)",
+						cfg.DefaultTransactionsLimit,
+						cfg.MaxTransactionsLimit,
+					)
+				}
+				return nil
+			},
+		},
 		{
 			Name: "max-healthy-ledger-latency",
 			Usage: "maximum ledger latency (i.e. time elapsed since the last known ledger closing time) considered to be healthy" +
@@ -330,6 +356,13 @@ func (cfg *Config) options() ConfigOptions {
 			DefaultValue: uint(1000),
 			Validate:     positive,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("request-backlog-get-transactions-queue-limit"),
+			Usage:        "Maximum number of outstanding GetTransactions requests",
+			ConfigKey:    &cfg.RequestBacklogGetTransactionsQueueLimit,
+			DefaultValue: uint(1000),
+			Validate:     positive,
+		},
 		{
 			TomlKey:      strutils.KebabToConstantCase("request-backlog-send-transaction-queue-limit"),
 			Usage:        "Maximum number of outstanding SendTransaction requests",
@@ -398,6 +431,12 @@ func (cfg *Config) options() ConfigOptions {
 			ConfigKey:    &cfg.MaxGetTransactionExecutionDuration,
 			DefaultValue: 5 * time.Second,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("max-get-transactions-execution-duration"),
+			Usage:        "The maximum duration of time allowed for processing a getTransactions request. When that time elapses, the rpc server would return -32001 and abort the request's execution",
+			ConfigKey:    &cfg.MaxGetTransactionsExecutionDuration,
+			DefaultValue: 5 * time.Second,
+		},
 		{
 			TomlKey:      strutils.KebabToConstantCase("max-send-transaction-execution-duration"),
 			Usage:        "The maximum duration of time allowed for processing a sendTransaction request. When that time elapses, the rpc server would return -32001 and abort the request's execution",

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -19,6 +19,7 @@ import (
 	"github.com/stellar/go/ingest/ledgerbackend"
 	supporthttp "github.com/stellar/go/support/http"
 	supportlog "github.com/stellar/go/support/log"
+	"github.com/stellar/go/support/ordered"
 	"github.com/stellar/go/support/storage"
 	"github.com/stellar/go/xdr"
 
@@ -29,7 +30,6 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util"
 )
 
@@ -134,7 +134,6 @@ func newCaptiveCore(cfg *config.Config, logger *supportlog.Entry) (*ledgerbacken
 func MustNew(cfg *config.Config) *Daemon {
 	logger := supportlog.New()
 	logger.SetLevel(cfg.LogLevel)
-
 	if cfg.LogFormat == config.LogFormatJSON {
 		logger.UseJSONFormatter()
 	}
@@ -150,7 +149,7 @@ func MustNew(cfg *config.Config) *Daemon {
 	}
 
 	if len(cfg.HistoryArchiveURLs) == 0 {
-		logger.Fatal("no history archives url were provided")
+		logger.Fatal("no history archives URLs were provided")
 	}
 
 	historyArchive, err := historyarchive.NewArchivePool(
@@ -191,11 +190,6 @@ func MustNew(cfg *config.Config) *Daemon {
 		cfg.NetworkPassphrase,
 		cfg.EventLedgerRetentionWindow,
 	)
-	transactionStore := transactions.NewMemoryStore(
-		daemon,
-		cfg.NetworkPassphrase,
-		cfg.TransactionLedgerRetentionWindow,
-	)
 
 	// initialize the stores using what was on the DB
 	readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
@@ -220,9 +214,6 @@ func MustNew(cfg *config.Config) *Daemon {
 		if err := eventStore.IngestEvents(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize event memory store")
 		}
-		if err := transactionStore.IngestTransactions(txmeta); err != nil {
-			logger.WithError(err).Fatal("could not initialize transaction memory store")
-		}
 		return nil
 	})
 	if err != nil {
@@ -237,17 +228,27 @@ func MustNew(cfg *config.Config) *Daemon {
 	onIngestionRetry := func(err error, dur time.Duration) {
 		logger.WithError(err).Error("could not run ingestion. Retrying")
 	}
-	maxRetentionWindow := cfg.EventLedgerRetentionWindow
-	if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow {
-		maxRetentionWindow = cfg.TransactionLedgerRetentionWindow
-	} else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow {
-		maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow
+
+	// Take the larger of (event retention, tx retention) and then the smaller
+	// of (tx retention, default event retention) if event retention wasn't
+	// specified, for some reason...?
+	maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow)
+	if cfg.EventLedgerRetentionWindow <= 0 {
+		maxRetentionWindow = ordered.Min(
+			maxRetentionWindow,
+			ledgerbucketwindow.DefaultEventLedgerRetentionWindow)
 	}
 	ingestService := ingest.NewService(ingest.Config{
-		Logger:            logger,
-		DB:                db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow),
+		Logger: logger,
+		DB: db.NewReadWriter(
+			logger,
+			dbConn,
+			daemon,
+			maxLedgerEntryWriteBatchSize,
+			maxRetentionWindow,
+			cfg.NetworkPassphrase,
+		),
 		EventStore:        eventStore,
-		TransactionStore:  transactionStore,
 		NetworkPassPhrase: cfg.NetworkPassphrase,
 		Archive:           historyArchive,
 		LedgerBackend:     core,
@@ -270,10 +271,10 @@ func MustNew(cfg *config.Config) *Daemon {
 	jsonRPCHandler := internal.NewJSONRPCHandler(cfg, internal.HandlerParams{
 		Daemon:            daemon,
 		EventStore:        eventStore,
-		TransactionStore:  transactionStore,
 		Logger:            logger,
 		LedgerReader:      db.NewLedgerReader(dbConn),
 		LedgerEntryReader: db.NewLedgerEntryReader(dbConn),
+		TransactionReader: db.NewTransactionReader(logger, dbConn, cfg.NetworkPassphrase),
 		PreflightGetter:   preflightWorkerPool,
 	})
 

cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go

@@ -24,7 +24,7 @@ func MakeNoOpDeamon() *noOpDaemon {
 }
 
 func (d *noOpDaemon) MetricsRegistry() *prometheus.Registry {
-	return d.metricsRegistry
+	return prometheus.NewRegistry() // so that you can register metrics many times
 }
 
 func (d *noOpDaemon) MetricsNamespace() string {