Add getFeeStats method (#172)

cmd/soroban-rpc/internal/config/config.go

@@ -20,47 +20,51 @@ type Config struct {
 	CaptiveCoreConfigPath  string
 	CaptiveCoreHTTPPort    uint
 
-	Endpoint                                    string
-	AdminEndpoint                               string
-	CheckpointFrequency                         uint32
-	CoreRequestTimeout                          time.Duration
-	DefaultEventsLimit                          uint
-	EventLedgerRetentionWindow                  uint32
-	FriendbotURL                                string
-	HistoryArchiveURLs                          []string
-	HistoryArchiveUserAgent                     string
-	IngestionTimeout                            time.Duration
-	LogFormat                                   LogFormat
-	LogLevel                                    logrus.Level
-	MaxEventsLimit                              uint
-	MaxHealthyLedgerLatency                     time.Duration
-	NetworkPassphrase                           string
-	PreflightWorkerCount                        uint
-	PreflightWorkerQueueSize                    uint
-	PreflightEnableDebug                        bool
-	SQLiteDBPath                                string
-	TransactionLedgerRetentionWindow            uint32
-	RequestBacklogGlobalQueueLimit              uint
-	RequestBacklogGetHealthQueueLimit           uint
-	RequestBacklogGetEventsQueueLimit           uint
-	RequestBacklogGetNetworkQueueLimit          uint
-	RequestBacklogGetVersionInfoQueueLimit      uint
-	RequestBacklogGetLatestLedgerQueueLimit     uint
-	RequestBacklogGetLedgerEntriesQueueLimit    uint
-	RequestBacklogGetTransactionQueueLimit      uint
-	RequestBacklogSendTransactionQueueLimit     uint
-	RequestBacklogSimulateTransactionQueueLimit uint
-	RequestExecutionWarningThreshold            time.Duration
-	MaxRequestExecutionDuration                 time.Duration
-	MaxGetHealthExecutionDuration               time.Duration
-	MaxGetEventsExecutionDuration               time.Duration
-	MaxGetNetworkExecutionDuration              time.Duration
-	MaxGetVersionInfoExecutionDuration          time.Duration
-	MaxGetLatestLedgerExecutionDuration         time.Duration
-	MaxGetLedgerEntriesExecutionDuration        time.Duration
-	MaxGetTransactionExecutionDuration          time.Duration
-	MaxSendTransactionExecutionDuration         time.Duration
-	MaxSimulateTransactionExecutionDuration     time.Duration
+	Endpoint                                       string
+	AdminEndpoint                                  string
+	CheckpointFrequency                            uint32
+	CoreRequestTimeout                             time.Duration
+	DefaultEventsLimit                             uint
+	EventLedgerRetentionWindow                     uint32
+	FriendbotURL                                   string
+	HistoryArchiveURLs                             []string
+	HistoryArchiveUserAgent                        string
+	IngestionTimeout                               time.Duration
+	LogFormat                                      LogFormat
+	LogLevel                                       logrus.Level
+	MaxEventsLimit                                 uint
+	MaxHealthyLedgerLatency                        time.Duration
+	NetworkPassphrase                              string
+	PreflightWorkerCount                           uint
+	PreflightWorkerQueueSize                       uint
+	PreflightEnableDebug                           bool
+	SQLiteDBPath                                   string
+	TransactionLedgerRetentionWindow               uint32
+	SorobanFeeStatsLedgerRetentionWindow           uint32
+	ClassicFeeStatsLedgerRetentionWindow           uint32
+	RequestBacklogGlobalQueueLimit                 uint
+	RequestBacklogGetHealthQueueLimit              uint
+	RequestBacklogGetEventsQueueLimit              uint
+	RequestBacklogGetNetworkQueueLimit             uint
+	RequestBacklogGetVersionInfoQueueLimit         uint
+	RequestBacklogGetLatestLedgerQueueLimit        uint
+	RequestBacklogGetLedgerEntriesQueueLimit       uint
+	RequestBacklogGetTransactionQueueLimit         uint
+	RequestBacklogSendTransactionQueueLimit        uint
+	RequestBacklogSimulateTransactionQueueLimit    uint
+	RequestBacklogGetFeeStatsTransactionQueueLimit uint
+	RequestExecutionWarningThreshold               time.Duration
+	MaxRequestExecutionDuration                    time.Duration
+	MaxGetHealthExecutionDuration                  time.Duration
+	MaxGetEventsExecutionDuration                  time.Duration
+	MaxGetNetworkExecutionDuration                 time.Duration
+	MaxGetVersionInfoExecutionDuration             time.Duration
+	MaxGetLatestLedgerExecutionDuration            time.Duration
+	MaxGetLedgerEntriesExecutionDuration           time.Duration
+	MaxGetTransactionExecutionDuration             time.Duration
+	MaxSendTransactionExecutionDuration            time.Duration
+	MaxSimulateTransactionExecutionDuration        time.Duration
+	MaxGetFeeStatsExecutionDuration                time.Duration
 
 	// We memoize these, so they bind to pflags correctly
 	optionsCache *ConfigOptions

cmd/soroban-rpc/internal/config/options.go

@@ -225,6 +225,20 @@ func (cfg *Config) options() ConfigOptions {
 			DefaultValue: uint32(1440),
 			Validate:     positive,
 		},
+		{
+			Name:         "classic-fee-stats-retention-window",
+			Usage:        "configures classic fee stats retention window expressed in number of ledgers",
+			ConfigKey:    &cfg.ClassicFeeStatsLedgerRetentionWindow,
+			DefaultValue: uint32(10),
+			Validate:     positive,
+		},
+		{
+			Name:         "soroban-fee-stats-retention-window",
+			Usage:        "configures soroban inclusion fee stats retention window expressed in number of ledgers",
+			ConfigKey:    &cfg.SorobanFeeStatsLedgerRetentionWindow,
+			DefaultValue: uint32(50),
+			Validate:     positive,
+		},
 		{
 			Name:         "max-events-limit",
 			Usage:        "Maximum amount of events allowed in a single getEvents response",
@@ -344,6 +358,13 @@ func (cfg *Config) options() ConfigOptions {
 			DefaultValue: uint(100),
 			Validate:     positive,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("request-backlog-get-fee-stats-queue-limit"),
+			Usage:        "Maximum number of outstanding GetFeeStats requests",
+			ConfigKey:    &cfg.RequestBacklogGetFeeStatsTransactionQueueLimit,
+			DefaultValue: uint(100),
+			Validate:     positive,
+		},
 		{
 			TomlKey:      strutils.KebabToConstantCase("request-execution-warning-threshold"),
 			Usage:        "The request execution warning threshold is the predetermined maximum duration of time that a request can take to be processed before a warning would be generated",
@@ -410,6 +431,12 @@ func (cfg *Config) options() ConfigOptions {
 			ConfigKey:    &cfg.MaxSimulateTransactionExecutionDuration,
 			DefaultValue: 15 * time.Second,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("max-get-fee-stats-execution-duration"),
+			Usage:        "The maximum duration of time allowed for processing a getFeeStats request. When that time elapses, the rpc server would return -32001 and abort the request's execution",
+			ConfigKey:    &cfg.MaxGetFeeStatsExecutionDuration,
+			DefaultValue: 5 * time.Second,
+		},
 	}
 	return *cfg.optionsCache
 }

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -26,8 +26,8 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events"
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util"
@@ -103,13 +103,14 @@ func (d *Daemon) Close() error {
 // newCaptiveCore creates a new captive core backend instance and returns it.
 func newCaptiveCore(cfg *config.Config, logger *supportlog.Entry) (*ledgerbackend.CaptiveStellarCore, error) {
 	captiveCoreTomlParams := ledgerbackend.CaptiveCoreTomlParams{
-		HTTPPort:                       &cfg.CaptiveCoreHTTPPort,
-		HistoryArchiveURLs:             cfg.HistoryArchiveURLs,
-		NetworkPassphrase:              cfg.NetworkPassphrase,
-		Strict:                         true,
-		UseDB:                          true,
-		EnforceSorobanDiagnosticEvents: true,
-		CoreBinaryPath:                 cfg.StellarCoreBinaryPath,
+		HTTPPort:                           &cfg.CaptiveCoreHTTPPort,
+		HistoryArchiveURLs:                 cfg.HistoryArchiveURLs,
+		NetworkPassphrase:                  cfg.NetworkPassphrase,
+		Strict:                             true,
+		UseDB:                              true,
+		EnforceSorobanDiagnosticEvents:     true,
+		EnforceSorobanTransactionMetaExtV1: true,
+		CoreBinaryPath:                     cfg.StellarCoreBinaryPath,
 	}
 	captiveCoreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(cfg.CaptiveCoreConfigPath, captiveCoreTomlParams)
 	if err != nil {
@@ -196,12 +197,13 @@ func MustNew(cfg *config.Config) *Daemon {
 		cfg.NetworkPassphrase,
 		cfg.TransactionLedgerRetentionWindow,
 	)
+	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
 
 	// initialize the stores using what was on the DB
 	readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
 	defer cancelReadTxMeta()
 	// NOTE: We could optimize this to avoid unnecessary ingestion calls
-	//       (the range of txmetads can be larger than the store retention windows)
+	//       (the range of txmetas can be larger than the individual store retention windows)
 	//       but it's probably not worth the pain.
 	var initialSeq uint32
 	var currentSeq uint32
@@ -223,6 +225,9 @@ func MustNew(cfg *config.Config) *Daemon {
 		if err := transactionStore.IngestTransactions(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize transaction memory store")
 		}
+		if err := feewindows.IngestFees(txmeta); err != nil {
+			logger.WithError(err).Fatal("could not initialize fee stats")
+		}
 		return nil
 	})
 	if err != nil {
@@ -237,12 +242,7 @@ func MustNew(cfg *config.Config) *Daemon {
 	onIngestionRetry := func(err error, dur time.Duration) {
 		logger.WithError(err).Error("could not run ingestion. Retrying")
 	}
-	maxRetentionWindow := cfg.EventLedgerRetentionWindow
-	if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow {
-		maxRetentionWindow = cfg.TransactionLedgerRetentionWindow
-	} else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow {
-		maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow
-	}
+	maxRetentionWindow := max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow, cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow)
 	ingestService := ingest.NewService(ingest.Config{
 		Logger:            logger,
 		DB:                db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow),
@@ -254,6 +254,7 @@ func MustNew(cfg *config.Config) *Daemon {
 		Timeout:           cfg.IngestionTimeout,
 		OnIngestionRetry:  onIngestionRetry,
 		Daemon:            daemon,
+		FeeWindows:        feewindows,
 	})
 
 	ledgerEntryReader := db.NewLedgerEntryReader(dbConn)
@@ -271,6 +272,7 @@ func MustNew(cfg *config.Config) *Daemon {
 		Daemon:            daemon,
 		EventStore:        eventStore,
 		TransactionStore:  transactionStore,
+		FeeStatWindows:    feewindows,
 		Logger:            logger,
 		LedgerReader:      db.NewLedgerReader(dbConn),
 		LedgerEntryReader: db.NewLedgerEntryReader(dbConn),