Add new getTransactions endpoint and database backend for transactions

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 
 ## Unreleased
-n/a
+
+### Added
+* Transactions will now be stored in a database rather than in memory ([#174](https://github.com/stellar/soroban-rpc/pull/174)).
+
+You can opt-in to longer transaction retention by setting `--transaction-retention-window` / `TRANSACTION_RETENTION_WINDOW` to a higher number of ledgers. This will also retain corresponding number of ledgers in the database. Keep in mind, of course, that this will cause an increase in disk usage for the growing database.
 
 
 ## [v21.2.0](https://github.com/stellar/soroban-rpc/compare/v21.1.0...v21.2.0)

cmd/soroban-rpc/internal/config/config.go

@@ -25,6 +25,7 @@ type Config struct {
 	CheckpointFrequency                            uint32
 	CoreRequestTimeout                             time.Duration
 	DefaultEventsLimit                             uint
+	DefaultTransactionsLimit                       uint
 	EventLedgerRetentionWindow                     uint32
 	FriendbotURL                                   string
 	HistoryArchiveURLs                             []string
@@ -33,6 +34,7 @@ type Config struct {
 	LogFormat                                      LogFormat
 	LogLevel                                       logrus.Level
 	MaxEventsLimit                                 uint
+	MaxTransactionsLimit                           uint
 	MaxHealthyLedgerLatency                        time.Duration
 	NetworkPassphrase                              string
 	PreflightWorkerCount                           uint
@@ -50,6 +52,7 @@ type Config struct {
 	RequestBacklogGetLatestLedgerQueueLimit        uint
 	RequestBacklogGetLedgerEntriesQueueLimit       uint
 	RequestBacklogGetTransactionQueueLimit         uint
+	RequestBacklogGetTransactionsQueueLimit        uint
 	RequestBacklogSendTransactionQueueLimit        uint
 	RequestBacklogSimulateTransactionQueueLimit    uint
 	RequestBacklogGetFeeStatsTransactionQueueLimit uint
@@ -62,6 +65,7 @@ type Config struct {
 	MaxGetLatestLedgerExecutionDuration            time.Duration
 	MaxGetLedgerEntriesExecutionDuration           time.Duration
 	MaxGetTransactionExecutionDuration             time.Duration
+	MaxGetTransactionsExecutionDuration            time.Duration
 	MaxSendTransactionExecutionDuration            time.Duration
 	MaxSimulateTransactionExecutionDuration        time.Duration
 	MaxGetFeeStatsExecutionDuration                time.Duration

cmd/soroban-rpc/internal/config/options.go

@@ -211,18 +211,22 @@ func (cfg *Config) options() ConfigOptions {
 		},
 		{
 			Name: "event-retention-window",
-			Usage: fmt.Sprintf("configures the event retention window expressed in number of ledgers,"+
-				" the default value is %d which corresponds to about 24 hours of history", ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
+			Usage: fmt.Sprintf(
+				"configures the event retention window expressed in number of ledgers,"+
+					" the default value is %d which corresponds to about 24 hours of history",
+				ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
 			ConfigKey:    &cfg.EventLedgerRetentionWindow,
 			DefaultValue: uint32(ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
 			Validate:     positive,
 		},
 		{
 			Name: "transaction-retention-window",
-			Usage: "configures the transaction retention window expressed in number of ledgers," +
-				" the default value is 1440 which corresponds to about 2 hours of history",
+			Usage: fmt.Sprintf(
+				"configures the transaction retention window expressed in number of ledgers,"+
+					" the default value is %d which corresponds to about 24 hours of history",
+				ledgerbucketwindow.OneDayOfLedgers),
 			ConfigKey:    &cfg.TransactionLedgerRetentionWindow,
-			DefaultValue: uint32(1440),
+			DefaultValue: uint32(ledgerbucketwindow.OneDayOfLedgers),
 			Validate:     positive,
 		},
 		{
@@ -261,6 +265,28 @@ func (cfg *Config) options() ConfigOptions {
 				return nil
 			},
 		},
+		{
+			Name:         "max-transactions-limit",
+			Usage:        "Maximum amount of transactions allowed in a single getTransactions response",
+			ConfigKey:    &cfg.MaxTransactionsLimit,
+			DefaultValue: uint(200),
+		},
+		{
+			Name:         "default-transactions-limit",
+			Usage:        "Default cap on the amount of transactions included in a single getTransactions response",
+			ConfigKey:    &cfg.DefaultTransactionsLimit,
+			DefaultValue: uint(50),
+			Validate: func(co *ConfigOption) error {
+				if cfg.DefaultTransactionsLimit > cfg.MaxTransactionsLimit {
+					return fmt.Errorf(
+						"default-transactions-limit (%v) cannot exceed max-transactions-limit (%v)",
+						cfg.DefaultTransactionsLimit,
+						cfg.MaxTransactionsLimit,
+					)
+				}
+				return nil
+			},
+		},
 		{
 			Name: "max-healthy-ledger-latency",
 			Usage: "maximum ledger latency (i.e. time elapsed since the last known ledger closing time) considered to be healthy" +
@@ -344,6 +370,13 @@ func (cfg *Config) options() ConfigOptions {
 			DefaultValue: uint(1000),
 			Validate:     positive,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("request-backlog-get-transactions-queue-limit"),
+			Usage:        "Maximum number of outstanding GetTransactions requests",
+			ConfigKey:    &cfg.RequestBacklogGetTransactionsQueueLimit,
+			DefaultValue: uint(1000),
+			Validate:     positive,
+		},
 		{
 			TomlKey:      strutils.KebabToConstantCase("request-backlog-send-transaction-queue-limit"),
 			Usage:        "Maximum number of outstanding SendTransaction requests",
@@ -419,6 +452,12 @@ func (cfg *Config) options() ConfigOptions {
 			ConfigKey:    &cfg.MaxGetTransactionExecutionDuration,
 			DefaultValue: 5 * time.Second,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("max-get-transactions-execution-duration"),
+			Usage:        "The maximum duration of time allowed for processing a getTransactions request. When that time elapses, the rpc server would return -32001 and abort the request's execution",
+			ConfigKey:    &cfg.MaxGetTransactionsExecutionDuration,
+			DefaultValue: 5 * time.Second,
+		},
 		{
 			TomlKey:      strutils.KebabToConstantCase("max-send-transaction-execution-duration"),
 			Usage:        "The maximum duration of time allowed for processing a sendTransaction request. When that time elapses, the rpc server would return -32001 and abort the request's execution",

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -19,6 +19,7 @@ import (
 	"github.com/stellar/go/ingest/ledgerbackend"
 	supporthttp "github.com/stellar/go/support/http"
 	supportlog "github.com/stellar/go/support/log"
+	"github.com/stellar/go/support/ordered"
 	"github.com/stellar/go/support/storage"
 	"github.com/stellar/go/xdr"
 
@@ -28,8 +29,8 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest"
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util"
 )
 
@@ -135,7 +136,6 @@ func newCaptiveCore(cfg *config.Config, logger *supportlog.Entry) (*ledgerbacken
 func MustNew(cfg *config.Config) *Daemon {
 	logger := supportlog.New()
 	logger.SetLevel(cfg.LogLevel)
-
 	if cfg.LogFormat == config.LogFormatJSON {
 		logger.UseJSONFormatter()
 	}
@@ -151,7 +151,7 @@ func MustNew(cfg *config.Config) *Daemon {
 	}
 
 	if len(cfg.HistoryArchiveURLs) == 0 {
-		logger.Fatal("no history archives url were provided")
+		logger.Fatal("no history archives URLs were provided")
 	}
 
 	historyArchive, err := historyarchive.NewArchivePool(
@@ -192,11 +192,6 @@ func MustNew(cfg *config.Config) *Daemon {
 		cfg.NetworkPassphrase,
 		cfg.EventLedgerRetentionWindow,
 	)
-	transactionStore := transactions.NewMemoryStore(
-		daemon,
-		cfg.NetworkPassphrase,
-		cfg.TransactionLedgerRetentionWindow,
-	)
 	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
 
 	// initialize the stores using what was on the DB
@@ -222,9 +217,6 @@ func MustNew(cfg *config.Config) *Daemon {
 		if err := eventStore.IngestEvents(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize event memory store")
 		}
-		if err := transactionStore.IngestTransactions(txmeta); err != nil {
-			logger.WithError(err).Fatal("could not initialize transaction memory store")
-		}
 		if err := feewindows.IngestFees(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize fee stats")
 		}
@@ -242,12 +234,27 @@ func MustNew(cfg *config.Config) *Daemon {
 	onIngestionRetry := func(err error, dur time.Duration) {
 		logger.WithError(err).Error("could not run ingestion. Retrying")
 	}
-	maxRetentionWindow := max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow, cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow)
+
+	// Take the larger of (event retention, tx retention) and then the smaller
+	// of (tx retention, default event retention) if event retention wasn't
+	// specified, for some reason...?
+	maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow)
+	if cfg.EventLedgerRetentionWindow <= 0 {
+		maxRetentionWindow = ordered.Min(
+			maxRetentionWindow,
+			ledgerbucketwindow.DefaultEventLedgerRetentionWindow)
+	}
 	ingestService := ingest.NewService(ingest.Config{
-		Logger:            logger,
-		DB:                db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow),
+		Logger: logger,
+		DB: db.NewReadWriter(
+			logger,
+			dbConn,
+			daemon,
+			maxLedgerEntryWriteBatchSize,
+			maxRetentionWindow,
+			cfg.NetworkPassphrase,
+		),
 		EventStore:        eventStore,
-		TransactionStore:  transactionStore,
 		NetworkPassPhrase: cfg.NetworkPassphrase,
 		Archive:           historyArchive,
 		LedgerBackend:     core,
@@ -271,11 +278,11 @@ func MustNew(cfg *config.Config) *Daemon {
 	jsonRPCHandler := internal.NewJSONRPCHandler(cfg, internal.HandlerParams{
 		Daemon:            daemon,
 		EventStore:        eventStore,
-		TransactionStore:  transactionStore,
 		FeeStatWindows:    feewindows,
 		Logger:            logger,
 		LedgerReader:      db.NewLedgerReader(dbConn),
 		LedgerEntryReader: db.NewLedgerEntryReader(dbConn),
+		TransactionReader: db.NewTransactionReader(logger, dbConn, cfg.NetworkPassphrase),
 		PreflightGetter:   preflightWorkerPool,
 	})
 

cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go

@@ -24,7 +24,7 @@ func MakeNoOpDeamon() *noOpDaemon {
 }
 
 func (d *noOpDaemon) MetricsRegistry() *prometheus.Registry {
-	return d.metricsRegistry
+	return prometheus.NewRegistry() // so that you can register metrics many times
 }
 
 func (d *noOpDaemon) MetricsNamespace() string {