From 160a0c33b85fc07793b194fe8c2e4269e5505d16 Mon Sep 17 00:00:00 2001 From: pritsheth Date: Tue, 16 Apr 2024 04:51:21 +0530 Subject: [PATCH 1/5] Add Ledger Retention Window of 7 days --- cmd/soroban-rpc/internal/config/config.go | 1 + cmd/soroban-rpc/internal/config/options.go | 9 +++++++++ cmd/soroban-rpc/internal/daemon/daemon.go | 11 +++-------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/config.go b/cmd/soroban-rpc/internal/config/config.go index 15e69f6b..639d9e53 100644 --- a/cmd/soroban-rpc/internal/config/config.go +++ b/cmd/soroban-rpc/internal/config/config.go @@ -26,6 +26,7 @@ type Config struct { CoreRequestTimeout time.Duration DefaultEventsLimit uint EventLedgerRetentionWindow uint32 + LedgerRetentionWindow uint32 FriendbotURL string HistoryArchiveURLs []string HistoryArchiveUserAgent string diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index df503200..e131cb09 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -209,6 +209,15 @@ func (cfg *Config) options() ConfigOptions { ConfigKey: &cfg.CheckpointFrequency, DefaultValue: uint32(64), }, + { + Name: "ledger-retention-window", + Usage: fmt.Sprintf("configures the ledger retention window expressed in number of ledgers,"+ + " the default value is %d which corresponds to about 24 hours of history", ledgerbucketwindow.DefaultEventLedgerRetentionWindow), + ConfigKey: &cfg.LedgerRetentionWindow, + DefaultValue: uint32(120960), + Validate: positive, + }, + { Name: "event-retention-window", Usage: fmt.Sprintf("configures the event retention window expressed in number of ledgers,"+ diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index c2c224de..c28e4726 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -27,7 +27,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util" @@ -236,15 +235,11 @@ func MustNew(cfg *config.Config) *Daemon { onIngestionRetry := func(err error, dur time.Duration) { logger.WithError(err).Error("could not run ingestion. Retrying") } - maxRetentionWindow := cfg.EventLedgerRetentionWindow - if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow { - maxRetentionWindow = cfg.TransactionLedgerRetentionWindow - } else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow { - maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow - } + ledgerRetentionWindow := cfg.LedgerRetentionWindow + ingestService := ingest.NewService(ingest.Config{ Logger: logger, - DB: db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow), + DB: db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, ledgerRetentionWindow), EventStore: eventStore, TransactionStore: transactionStore, NetworkPassPhrase: cfg.NetworkPassphrase, From 5be08c6de44b3ce31ab51db68bb0d8916c9bdd58 Mon Sep 17 00:00:00 2001 From: pritsheth Date: Tue, 16 Apr 2024 05:00:41 +0530 Subject: [PATCH 2/5] update usage comment --- cmd/soroban-rpc/internal/config/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index e131cb09..41fc6c6d 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -211,8 +211,8 @@ func (cfg *Config) options() ConfigOptions { }, { Name: "ledger-retention-window", - Usage: fmt.Sprintf("configures the ledger retention window expressed in number of ledgers,"+ - " the default value is %d which corresponds to about 24 hours of history", ledgerbucketwindow.DefaultEventLedgerRetentionWindow), + Usage: fmt.Sprintf("configures the ledger retention window expressed in number of ledgers," + + " the default value is 120960 which corresponds to about 7 days of history"), ConfigKey: &cfg.LedgerRetentionWindow, DefaultValue: uint32(120960), Validate: positive, From 626cfc42b2a4544ef462839fe00cddd787982def Mon Sep 17 00:00:00 2001 From: pritsheth Date: Mon, 13 May 2024 16:34:18 -0700 Subject: [PATCH 3/5] Add backward compatibility and fix review comments. --- cmd/soroban-rpc/internal/config/config.go | 2 +- cmd/soroban-rpc/internal/config/options.go | 10 +++++----- cmd/soroban-rpc/internal/daemon/daemon.go | 23 ++++++++++++++-------- cmd/soroban-rpc/internal/jsonrpc.go | 8 +++++++- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/config.go b/cmd/soroban-rpc/internal/config/config.go index f0dca82a..1bb28321 100644 --- a/cmd/soroban-rpc/internal/config/config.go +++ b/cmd/soroban-rpc/internal/config/config.go @@ -27,7 +27,7 @@ type Config struct { DefaultEventsLimit uint DefaultTransactionsLimit uint EventLedgerRetentionWindow uint32 - LedgerRetentionWindow uint32 + HistoryRetentionWindow uint32 FriendbotURL string HistoryArchiveURLs []string HistoryArchiveUserAgent string diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index 2a0e59bc..f135d768 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -210,10 +210,10 @@ func (cfg *Config) options() ConfigOptions { DefaultValue: uint32(64), }, { - Name: "ledger-retention-window", - Usage: fmt.Sprintf("configures the ledger retention window expressed in number of ledgers," + + Name: "history-retention-window", + Usage: fmt.Sprintf("configures the history retention window expressed in number of ledgers," + " the default value is 120960 which corresponds to about 7 days of history"), - ConfigKey: &cfg.LedgerRetentionWindow, + ConfigKey: &cfg.HistoryRetentionWindow, DefaultValue: uint32(120960), Validate: positive, }, @@ -232,8 +232,8 @@ func (cfg *Config) options() ConfigOptions { Name: "transaction-retention-window", Usage: fmt.Sprintf( "configures the transaction retention window expressed in number of ledgers,"+ - " the default value is %d which corresponds to about 24 hours of history", - ledgerbucketwindow.OneDayOfLedgers), + " the default value is %d which corresponds to about 24 hours of history"+ + "\n\n DEPRECATED: Please use the history-retention-window instead", ledgerbucketwindow.OneDayOfLedgers), ConfigKey: &cfg.TransactionLedgerRetentionWindow, DefaultValue: uint32(ledgerbucketwindow.OneDayOfLedgers), Validate: positive, diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index a8f5d2d8..fb1784cf 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -228,14 +228,21 @@ func MustNew(cfg *config.Config) *Daemon { logger.WithError(err).Error("could not run ingestion. Retrying") } - // Take the larger of (event retention, tx retention) and then the smaller - // of (tx retention, default event retention) if event retention wasn't - // specified, for some reason...? - maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow) - if cfg.EventLedgerRetentionWindow <= 0 { - maxRetentionWindow = ordered.Min( - maxRetentionWindow, - ledgerbucketwindow.DefaultEventLedgerRetentionWindow) + var maxRetentionWindow uint32 + + if cfg.HistoryRetentionWindow > 0 { + maxRetentionWindow = cfg.HistoryRetentionWindow + } else { + // TODO: Discard this logic once we fully move from in-memory to on-disk storage + // Take the larger of (event retention, tx retention) and then the smaller + // of (tx retention, default event retention) if event retention wasn't + // specified, for some reason...? + maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow) + if cfg.EventLedgerRetentionWindow <= 0 { + maxRetentionWindow = ordered.Min( + maxRetentionWindow, + ledgerbucketwindow.DefaultEventLedgerRetentionWindow) + } } ingestService := ingest.NewService(ingest.Config{ Logger: logger, diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 6529e79b..c7c2698c 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -137,7 +137,13 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { // While we transition from in-memory to database-oriented history storage, // the on-disk (transaction) retention window will always be larger than the // in-memory (events) one. - var retentionWindow = cfg.TransactionLedgerRetentionWindow + var retentionWindow uint32 + + if cfg.HistoryRetentionWindow > 0 { + retentionWindow = cfg.HistoryRetentionWindow + } else { + retentionWindow = cfg.TransactionLedgerRetentionWindow + } handlers := []struct { methodName string From d08ab2880ce46b583e871e5a05e1eabba84e1b67 Mon Sep 17 00:00:00 2001 From: pritsheth Date: Tue, 14 May 2024 13:00:32 -0700 Subject: [PATCH 4/5] Fix tests and update deprecation message --- cmd/soroban-rpc/internal/config/options.go | 11 +++++------ cmd/soroban-rpc/internal/daemon/daemon.go | 5 +++++ .../internal/ledgerbucketwindow/ledgerbucketwindow.go | 1 + cmd/soroban-rpc/internal/test/health_test.go | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index f135d768..cd1e425d 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -211,10 +211,10 @@ func (cfg *Config) options() ConfigOptions { }, { Name: "history-retention-window", - Usage: fmt.Sprintf("configures the history retention window expressed in number of ledgers," + - " the default value is 120960 which corresponds to about 7 days of history"), + Usage: fmt.Sprintf("configures the history retention window expressed in number of ledgers,"+ + " the default value is %d which corresponds to about 7 days of history", ledgerbucketwindow.SevenDaysOfLedgers), ConfigKey: &cfg.HistoryRetentionWindow, - DefaultValue: uint32(120960), + DefaultValue: uint32(ledgerbucketwindow.SevenDaysOfLedgers), Validate: positive, }, @@ -231,9 +231,8 @@ func (cfg *Config) options() ConfigOptions { { Name: "transaction-retention-window", Usage: fmt.Sprintf( - "configures the transaction retention window expressed in number of ledgers,"+ - " the default value is %d which corresponds to about 24 hours of history"+ - "\n\n DEPRECATED: Please use the history-retention-window instead", ledgerbucketwindow.OneDayOfLedgers), + "DEPRECATED - The usage of the flag transaction-retention-window has been deprecated. ," + + "RPC now uses history-retention-window by default and this flag will soon be removed in future. "), ConfigKey: &cfg.TransactionLedgerRetentionWindow, DefaultValue: uint32(ledgerbucketwindow.OneDayOfLedgers), Validate: positive, diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index fb1784cf..fba5fa4c 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -137,6 +137,11 @@ func MustNew(cfg *config.Config) *Daemon { logger.UseJSONFormatter() } + if cfg.TransactionLedgerRetentionWindow > 0 { + logger.Info("DEPRECATION WARNING: transaction-retention-window has been deprecated" + + " Please use history-retention-window instead") + } + logger.WithFields(supportlog.F{ "version": config.Version, "commit": config.CommitHash, diff --git a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go index ee91427c..ec185a2f 100644 --- a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go +++ b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go @@ -22,6 +22,7 @@ type LedgerBucket[T any] struct { // OneDayOfLedgers is (roughly) a 24 hour window of ledgers. const OneDayOfLedgers = 17280 +const SevenDaysOfLedgers = 120960 // DefaultEventLedgerRetentionWindow represents the max number of ledgers we // would like to keep an incoming event in memory. diff --git a/cmd/soroban-rpc/internal/test/health_test.go b/cmd/soroban-rpc/internal/test/health_test.go index 0840959c..11d264f4 100644 --- a/cmd/soroban-rpc/internal/test/health_test.go +++ b/cmd/soroban-rpc/internal/test/health_test.go @@ -23,7 +23,7 @@ func TestHealth(t *testing.T) { t.Fatalf("rpc call failed: %v", err) } assert.Equal(t, "healthy", result.Status) - assert.Equal(t, uint32(ledgerbucketwindow.OneDayOfLedgers), result.LedgerRetentionWindow) + assert.Equal(t, uint32(ledgerbucketwindow.SevenDaysOfLedgers), result.LedgerRetentionWindow) assert.Greater(t, result.OldestLedger, uint32(0)) assert.Greater(t, result.LatestLedger, uint32(0)) assert.GreaterOrEqual(t, result.LatestLedger, result.OldestLedger) From 201da8143b2a259c31edd7e3824a38cca7f6dc6d Mon Sep 17 00:00:00 2001 From: pritsheth Date: Wed, 15 May 2024 12:20:27 -0700 Subject: [PATCH 5/5] Fix review comments --- cmd/soroban-rpc/internal/config/options.go | 6 ++---- .../internal/ledgerbucketwindow/ledgerbucketwindow.go | 2 +- cmd/soroban-rpc/internal/test/health_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index cd1e425d..d2ed8c80 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -212,10 +212,8 @@ func (cfg *Config) options() ConfigOptions { { Name: "history-retention-window", Usage: fmt.Sprintf("configures the history retention window expressed in number of ledgers,"+ - " the default value is %d which corresponds to about 7 days of history", ledgerbucketwindow.SevenDaysOfLedgers), - ConfigKey: &cfg.HistoryRetentionWindow, - DefaultValue: uint32(ledgerbucketwindow.SevenDaysOfLedgers), - Validate: positive, + " the maximum value can be %d which corresponds to about 7 days of history", ledgerbucketwindow.SevenDaysOfLedgers), + ConfigKey: &cfg.HistoryRetentionWindow, }, { diff --git a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go index ec185a2f..f58579fe 100644 --- a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go +++ b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go @@ -22,7 +22,7 @@ type LedgerBucket[T any] struct { // OneDayOfLedgers is (roughly) a 24 hour window of ledgers. const OneDayOfLedgers = 17280 -const SevenDaysOfLedgers = 120960 +const SevenDaysOfLedgers = 7 * OneDayOfLedgers // DefaultEventLedgerRetentionWindow represents the max number of ledgers we // would like to keep an incoming event in memory. diff --git a/cmd/soroban-rpc/internal/test/health_test.go b/cmd/soroban-rpc/internal/test/health_test.go index 11d264f4..0840959c 100644 --- a/cmd/soroban-rpc/internal/test/health_test.go +++ b/cmd/soroban-rpc/internal/test/health_test.go @@ -23,7 +23,7 @@ func TestHealth(t *testing.T) { t.Fatalf("rpc call failed: %v", err) } assert.Equal(t, "healthy", result.Status) - assert.Equal(t, uint32(ledgerbucketwindow.SevenDaysOfLedgers), result.LedgerRetentionWindow) + assert.Equal(t, uint32(ledgerbucketwindow.OneDayOfLedgers), result.LedgerRetentionWindow) assert.Greater(t, result.OldestLedger, uint32(0)) assert.Greater(t, result.LatestLedger, uint32(0)) assert.GreaterOrEqual(t, result.LatestLedger, result.OldestLedger)