From f15ad4a4a29e917a4abf84462cfaaa25ad0be0b3 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 17 May 2024 18:42:03 +0200 Subject: [PATCH] Add getFeeStats method (#172) --- cmd/soroban-rpc/internal/config/config.go | 86 +++--- cmd/soroban-rpc/internal/config/options.go | 27 ++ cmd/soroban-rpc/internal/daemon/daemon.go | 32 +- .../internal/feewindow/feewindow.go | 191 ++++++++++++ .../internal/feewindow/feewindow_test.go | 291 ++++++++++++++++++ cmd/soroban-rpc/internal/ingest/service.go | 9 + .../internal/ingest/service_test.go | 2 + cmd/soroban-rpc/internal/jsonrpc.go | 9 + .../internal/methods/get_events.go | 4 +- .../internal/methods/get_events_test.go | 2 +- .../internal/methods/get_fee_stats.go | 68 ++++ .../internal/test/get_fee_stats_test.go | 122 ++++++++ cmd/soroban-rpc/internal/test/integration.go | 1 - go.mod | 1 + go.sum | 2 + 15 files changed, 787 insertions(+), 60 deletions(-) create mode 100644 cmd/soroban-rpc/internal/feewindow/feewindow.go create mode 100644 cmd/soroban-rpc/internal/feewindow/feewindow_test.go create mode 100644 cmd/soroban-rpc/internal/methods/get_fee_stats.go create mode 100644 cmd/soroban-rpc/internal/test/get_fee_stats_test.go diff --git a/cmd/soroban-rpc/internal/config/config.go b/cmd/soroban-rpc/internal/config/config.go index 5d753b45..139144b0 100644 --- a/cmd/soroban-rpc/internal/config/config.go +++ b/cmd/soroban-rpc/internal/config/config.go @@ -20,47 +20,51 @@ type Config struct { CaptiveCoreConfigPath string CaptiveCoreHTTPPort uint - Endpoint string - AdminEndpoint string - CheckpointFrequency uint32 - CoreRequestTimeout time.Duration - DefaultEventsLimit uint - EventLedgerRetentionWindow uint32 - FriendbotURL string - HistoryArchiveURLs []string - HistoryArchiveUserAgent string - IngestionTimeout time.Duration - LogFormat LogFormat - LogLevel logrus.Level - MaxEventsLimit uint - MaxHealthyLedgerLatency time.Duration - NetworkPassphrase string - PreflightWorkerCount uint - PreflightWorkerQueueSize uint - PreflightEnableDebug bool - SQLiteDBPath string - TransactionLedgerRetentionWindow uint32 - RequestBacklogGlobalQueueLimit uint - RequestBacklogGetHealthQueueLimit uint - RequestBacklogGetEventsQueueLimit uint - RequestBacklogGetNetworkQueueLimit uint - RequestBacklogGetVersionInfoQueueLimit uint - RequestBacklogGetLatestLedgerQueueLimit uint - RequestBacklogGetLedgerEntriesQueueLimit uint - RequestBacklogGetTransactionQueueLimit uint - RequestBacklogSendTransactionQueueLimit uint - RequestBacklogSimulateTransactionQueueLimit uint - RequestExecutionWarningThreshold time.Duration - MaxRequestExecutionDuration time.Duration - MaxGetHealthExecutionDuration time.Duration - MaxGetEventsExecutionDuration time.Duration - MaxGetNetworkExecutionDuration time.Duration - MaxGetVersionInfoExecutionDuration time.Duration - MaxGetLatestLedgerExecutionDuration time.Duration - MaxGetLedgerEntriesExecutionDuration time.Duration - MaxGetTransactionExecutionDuration time.Duration - MaxSendTransactionExecutionDuration time.Duration - MaxSimulateTransactionExecutionDuration time.Duration + Endpoint string + AdminEndpoint string + CheckpointFrequency uint32 + CoreRequestTimeout time.Duration + DefaultEventsLimit uint + EventLedgerRetentionWindow uint32 + FriendbotURL string + HistoryArchiveURLs []string + HistoryArchiveUserAgent string + IngestionTimeout time.Duration + LogFormat LogFormat + LogLevel logrus.Level + MaxEventsLimit uint + MaxHealthyLedgerLatency time.Duration + NetworkPassphrase string + PreflightWorkerCount uint + PreflightWorkerQueueSize uint + PreflightEnableDebug bool + SQLiteDBPath string + TransactionLedgerRetentionWindow uint32 + SorobanFeeStatsLedgerRetentionWindow uint32 + ClassicFeeStatsLedgerRetentionWindow uint32 + RequestBacklogGlobalQueueLimit uint + RequestBacklogGetHealthQueueLimit uint + RequestBacklogGetEventsQueueLimit uint + RequestBacklogGetNetworkQueueLimit uint + RequestBacklogGetVersionInfoQueueLimit uint + RequestBacklogGetLatestLedgerQueueLimit uint + RequestBacklogGetLedgerEntriesQueueLimit uint + RequestBacklogGetTransactionQueueLimit uint + RequestBacklogSendTransactionQueueLimit uint + RequestBacklogSimulateTransactionQueueLimit uint + RequestBacklogGetFeeStatsTransactionQueueLimit uint + RequestExecutionWarningThreshold time.Duration + MaxRequestExecutionDuration time.Duration + MaxGetHealthExecutionDuration time.Duration + MaxGetEventsExecutionDuration time.Duration + MaxGetNetworkExecutionDuration time.Duration + MaxGetVersionInfoExecutionDuration time.Duration + MaxGetLatestLedgerExecutionDuration time.Duration + MaxGetLedgerEntriesExecutionDuration time.Duration + MaxGetTransactionExecutionDuration time.Duration + MaxSendTransactionExecutionDuration time.Duration + MaxSimulateTransactionExecutionDuration time.Duration + MaxGetFeeStatsExecutionDuration time.Duration // We memoize these, so they bind to pflags correctly optionsCache *ConfigOptions diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index 5be1d31a..80d5068e 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -225,6 +225,20 @@ func (cfg *Config) options() ConfigOptions { DefaultValue: uint32(1440), Validate: positive, }, + { + Name: "classic-fee-stats-retention-window", + Usage: "configures classic fee stats retention window expressed in number of ledgers", + ConfigKey: &cfg.ClassicFeeStatsLedgerRetentionWindow, + DefaultValue: uint32(10), + Validate: positive, + }, + { + Name: "soroban-fee-stats-retention-window", + Usage: "configures soroban inclusion fee stats retention window expressed in number of ledgers", + ConfigKey: &cfg.SorobanFeeStatsLedgerRetentionWindow, + DefaultValue: uint32(50), + Validate: positive, + }, { Name: "max-events-limit", Usage: "Maximum amount of events allowed in a single getEvents response", @@ -344,6 +358,13 @@ func (cfg *Config) options() ConfigOptions { DefaultValue: uint(100), Validate: positive, }, + { + TomlKey: strutils.KebabToConstantCase("request-backlog-get-fee-stats-queue-limit"), + Usage: "Maximum number of outstanding GetFeeStats requests", + ConfigKey: &cfg.RequestBacklogGetFeeStatsTransactionQueueLimit, + DefaultValue: uint(100), + Validate: positive, + }, { TomlKey: strutils.KebabToConstantCase("request-execution-warning-threshold"), Usage: "The request execution warning threshold is the predetermined maximum duration of time that a request can take to be processed before a warning would be generated", @@ -410,6 +431,12 @@ func (cfg *Config) options() ConfigOptions { ConfigKey: &cfg.MaxSimulateTransactionExecutionDuration, DefaultValue: 15 * time.Second, }, + { + TomlKey: strutils.KebabToConstantCase("max-get-fee-stats-execution-duration"), + Usage: "The maximum duration of time allowed for processing a getFeeStats request. When that time elapses, the rpc server would return -32001 and abort the request's execution", + ConfigKey: &cfg.MaxGetFeeStatsExecutionDuration, + DefaultValue: 5 * time.Second, + }, } return *cfg.optionsCache } diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 0e273f69..d3432391 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -26,8 +26,8 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util" @@ -103,13 +103,14 @@ func (d *Daemon) Close() error { // newCaptiveCore creates a new captive core backend instance and returns it. func newCaptiveCore(cfg *config.Config, logger *supportlog.Entry) (*ledgerbackend.CaptiveStellarCore, error) { captiveCoreTomlParams := ledgerbackend.CaptiveCoreTomlParams{ - HTTPPort: &cfg.CaptiveCoreHTTPPort, - HistoryArchiveURLs: cfg.HistoryArchiveURLs, - NetworkPassphrase: cfg.NetworkPassphrase, - Strict: true, - UseDB: true, - EnforceSorobanDiagnosticEvents: true, - CoreBinaryPath: cfg.StellarCoreBinaryPath, + HTTPPort: &cfg.CaptiveCoreHTTPPort, + HistoryArchiveURLs: cfg.HistoryArchiveURLs, + NetworkPassphrase: cfg.NetworkPassphrase, + Strict: true, + UseDB: true, + EnforceSorobanDiagnosticEvents: true, + EnforceSorobanTransactionMetaExtV1: true, + CoreBinaryPath: cfg.StellarCoreBinaryPath, } captiveCoreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(cfg.CaptiveCoreConfigPath, captiveCoreTomlParams) if err != nil { @@ -196,12 +197,13 @@ func MustNew(cfg *config.Config) *Daemon { cfg.NetworkPassphrase, cfg.TransactionLedgerRetentionWindow, ) + feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase) // initialize the stores using what was on the DB readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout) defer cancelReadTxMeta() // NOTE: We could optimize this to avoid unnecessary ingestion calls - // (the range of txmetads can be larger than the store retention windows) + // (the range of txmetas can be larger than the individual store retention windows) // but it's probably not worth the pain. var initialSeq uint32 var currentSeq uint32 @@ -223,6 +225,9 @@ func MustNew(cfg *config.Config) *Daemon { if err := transactionStore.IngestTransactions(txmeta); err != nil { logger.WithError(err).Fatal("could not initialize transaction memory store") } + if err := feewindows.IngestFees(txmeta); err != nil { + logger.WithError(err).Fatal("could not initialize fee stats") + } return nil }) if err != nil { @@ -237,12 +242,7 @@ func MustNew(cfg *config.Config) *Daemon { onIngestionRetry := func(err error, dur time.Duration) { logger.WithError(err).Error("could not run ingestion. Retrying") } - maxRetentionWindow := cfg.EventLedgerRetentionWindow - if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow { - maxRetentionWindow = cfg.TransactionLedgerRetentionWindow - } else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow { - maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow - } + maxRetentionWindow := max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow, cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow) ingestService := ingest.NewService(ingest.Config{ Logger: logger, DB: db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow), @@ -254,6 +254,7 @@ func MustNew(cfg *config.Config) *Daemon { Timeout: cfg.IngestionTimeout, OnIngestionRetry: onIngestionRetry, Daemon: daemon, + FeeWindows: feewindows, }) ledgerEntryReader := db.NewLedgerEntryReader(dbConn) @@ -271,6 +272,7 @@ func MustNew(cfg *config.Config) *Daemon { Daemon: daemon, EventStore: eventStore, TransactionStore: transactionStore, + FeeStatWindows: feewindows, Logger: logger, LedgerReader: db.NewLedgerReader(dbConn), LedgerEntryReader: db.NewLedgerEntryReader(dbConn), diff --git a/cmd/soroban-rpc/internal/feewindow/feewindow.go b/cmd/soroban-rpc/internal/feewindow/feewindow.go new file mode 100644 index 00000000..4adc4880 --- /dev/null +++ b/cmd/soroban-rpc/internal/feewindow/feewindow.go @@ -0,0 +1,191 @@ +package feewindow + +import ( + "io" + "slices" + "sync" + + "github.com/stellar/go/ingest" + "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" +) + +type FeeDistribution struct { + Max uint64 + Min uint64 + Mode uint64 + P10 uint64 + P20 uint64 + P30 uint64 + P40 uint64 + P50 uint64 + P60 uint64 + P70 uint64 + P80 uint64 + P90 uint64 + P95 uint64 + P99 uint64 + FeeCount uint32 + LedgerCount uint32 +} + +type FeeWindow struct { + lock sync.RWMutex + feesPerLedger *ledgerbucketwindow.LedgerBucketWindow[[]uint64] + distribution FeeDistribution +} + +func NewFeeWindow(retentionWindow uint32) *FeeWindow { + window := ledgerbucketwindow.NewLedgerBucketWindow[[]uint64](retentionWindow) + return &FeeWindow{ + feesPerLedger: window, + } +} + +func (fw *FeeWindow) AppendLedgerFees(fees ledgerbucketwindow.LedgerBucket[[]uint64]) error { + fw.lock.Lock() + defer fw.lock.Unlock() + _, err := fw.feesPerLedger.Append(fees) + if err != nil { + return err + } + + var allFees []uint64 + for i := uint32(0); i < fw.feesPerLedger.Len(); i++ { + allFees = append(allFees, fw.feesPerLedger.Get(i).BucketContent...) + } + fw.distribution = computeFeeDistribution(allFees, fw.feesPerLedger.Len()) + + return nil +} + +func computeFeeDistribution(fees []uint64, ledgerCount uint32) FeeDistribution { + if len(fees) == 0 { + return FeeDistribution{} + } + slices.Sort(fees) + mode := fees[0] + lastVal := fees[0] + maxRepetitions := 0 + localRepetitions := 0 + for i := 1; i < len(fees); i++ { + if fees[i] == lastVal { + localRepetitions += 1 + continue + } + + // new cluster of values + + if localRepetitions > maxRepetitions { + maxRepetitions = localRepetitions + mode = lastVal + } + lastVal = fees[i] + localRepetitions = 0 + } + + if localRepetitions > maxRepetitions { + // the last cluster of values was the longest + mode = fees[len(fees)-1] + } + + count := len(fees) + // nearest-rank percentile + percentile := func(p uint64) uint64 { + // ceiling(p*count/100) + kth := ((p * uint64(count)) + 100 - 1) / 100 + return fees[kth-1] + } + return FeeDistribution{ + Max: fees[len(fees)-1], + Min: fees[0], + Mode: mode, + P10: percentile(10), + P20: percentile(20), + P30: percentile(30), + P40: percentile(40), + P50: percentile(50), + P60: percentile(60), + P70: percentile(70), + P80: percentile(80), + P90: percentile(90), + P95: percentile(95), + P99: percentile(99), + FeeCount: uint32(count), + LedgerCount: ledgerCount, + } +} + +func (fw *FeeWindow) GetFeeDistribution() FeeDistribution { + fw.lock.RLock() + defer fw.lock.RUnlock() + return fw.distribution +} + +type FeeWindows struct { + SorobanInclusionFeeWindow *FeeWindow + ClassicFeeWindow *FeeWindow + networkPassPhrase string +} + +func NewFeeWindows(classicRetention uint32, sorobanRetetion uint32, networkPassPhrase string) *FeeWindows { + return &FeeWindows{ + SorobanInclusionFeeWindow: NewFeeWindow(sorobanRetetion), + ClassicFeeWindow: NewFeeWindow(classicRetention), + networkPassPhrase: networkPassPhrase, + } +} + +func (fw *FeeWindows) IngestFees(meta xdr.LedgerCloseMeta) error { + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(fw.networkPassPhrase, meta) + if err != nil { + return err + } + var sorobanInclusionFees []uint64 + var classicFees []uint64 + for { + tx, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return err + } + feeCharged := uint64(tx.Result.Result.FeeCharged) + ops := tx.Envelope.Operations() + if len(ops) == 0 { + // should not happen + continue + } + if len(ops) == 1 { + switch ops[0].Body.Type { + case xdr.OperationTypeInvokeHostFunction, xdr.OperationTypeExtendFootprintTtl, xdr.OperationTypeRestoreFootprint: + if tx.UnsafeMeta.V != 3 || tx.UnsafeMeta.V3.SorobanMeta == nil || tx.UnsafeMeta.V3.SorobanMeta.Ext.V != 1 { + continue + } + sorobanFees := tx.UnsafeMeta.V3.SorobanMeta.Ext.V1 + resourceFeeCharged := sorobanFees.TotalNonRefundableResourceFeeCharged + sorobanFees.TotalRefundableResourceFeeCharged + inclusionFee := feeCharged - uint64(resourceFeeCharged) + sorobanInclusionFees = append(sorobanInclusionFees, inclusionFee) + continue + } + } + feePerOp := feeCharged / uint64(len(ops)) + classicFees = append(classicFees, feePerOp) + + } + bucket := ledgerbucketwindow.LedgerBucket[[]uint64]{ + LedgerSeq: meta.LedgerSequence(), + LedgerCloseTimestamp: meta.LedgerCloseTime(), + BucketContent: classicFees, + } + if err := fw.ClassicFeeWindow.AppendLedgerFees(bucket); err != nil { + return err + } + bucket.BucketContent = sorobanInclusionFees + if err := fw.SorobanInclusionFeeWindow.AppendLedgerFees(bucket); err != nil { + return err + } + return nil +} diff --git a/cmd/soroban-rpc/internal/feewindow/feewindow_test.go b/cmd/soroban-rpc/internal/feewindow/feewindow_test.go new file mode 100644 index 00000000..53969ff2 --- /dev/null +++ b/cmd/soroban-rpc/internal/feewindow/feewindow_test.go @@ -0,0 +1,291 @@ +package feewindow + +import ( + "fmt" + "math/rand" + "slices" + "testing" + + "github.com/montanaflynn/stats" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBasicComputeFeeDistribution(t *testing.T) { + for _, testCase := range []struct { + name string + input []uint64 + output FeeDistribution + }{ + {"nil", nil, FeeDistribution{}}, + {"empty", []uint64{}, FeeDistribution{}}, + {"one", + []uint64{100}, + FeeDistribution{ + Max: 100, + Min: 100, + Mode: 100, + P10: 100, + P20: 100, + P30: 100, + P40: 100, + P50: 100, + P60: 100, + P70: 100, + P80: 100, + P90: 100, + P95: 100, + P99: 100, + FeeCount: 1, + }, + }, + {"even number of elements: four 100s and six 1000s", + []uint64{100, 100, 100, 1000, 100, 1000, 1000, 1000, 1000, 1000}, + FeeDistribution{ + Max: 1000, + Min: 100, + Mode: 1000, + P10: 100, + P20: 100, + P30: 100, + P40: 100, + P50: 1000, + P60: 1000, + P70: 1000, + P80: 1000, + P90: 1000, + P95: 1000, + P99: 1000, + FeeCount: 10, + }, + }, + {"odd number of elements: five 100s and six 1000s", + []uint64{100, 100, 100, 1000, 100, 1000, 1000, 1000, 1000, 1000, 100}, + FeeDistribution{ + Max: 1000, + Min: 100, + Mode: 1000, + P10: 100, + P20: 100, + P30: 100, + P40: 100, + P50: 1000, + P60: 1000, + P70: 1000, + P80: 1000, + P90: 1000, + P95: 1000, + P99: 1000, + FeeCount: 11, + }, + }, + {"mutiple modes favors the smallest value", + []uint64{100, 1000}, + FeeDistribution{ + Max: 1000, + Min: 100, + Mode: 100, + P10: 100, + P20: 100, + P30: 100, + P40: 100, + P50: 100, + P60: 1000, + P70: 1000, + P80: 1000, + P90: 1000, + P95: 1000, + P99: 1000, + FeeCount: 2, + }, + }, + {"random distribution with a repetition", + []uint64{515, 245, 245, 530, 221, 262, 927}, + FeeDistribution{ + Max: 927, + Min: 221, + Mode: 245, + P10: 221, + P20: 245, + P30: 245, + P40: 245, + P50: 262, + P60: 515, + P70: 515, + P80: 530, + P90: 927, + P95: 927, + P99: 927, + FeeCount: 7, + }, + }, + {"random distribution with a repetition of its largest value", + []uint64{515, 245, 530, 221, 262, 927, 927}, + FeeDistribution{ + Max: 927, + Min: 221, + Mode: 927, + P10: 221, + P20: 245, + P30: 262, + P40: 262, + P50: 515, + P60: 530, + P70: 530, + P80: 927, + P90: 927, + P95: 927, + P99: 927, + FeeCount: 7, + }, + }, + } { + assert.Equal(t, computeFeeDistribution(testCase.input, 0), testCase.output, testCase.name) + } +} + +func TestComputeFeeDistributionAgainstAlternative(t *testing.T) { + + for i := 0; i < 100_000; i++ { + fees := generateFees(nil) + feesCopy1 := make([]uint64, len(fees)) + feesCopy2 := make([]uint64, len(fees)) + for i := 0; i < len(fees); i++ { + feesCopy1[i] = fees[i] + feesCopy2[i] = fees[i] + } + actual := computeFeeDistribution(feesCopy2, 0) + expected, err := alternativeComputeFeeDistribution(feesCopy2, 0) + require.NoError(t, err) + assert.Equal(t, expected, actual, fmt.Sprintf("input fees: %v", fees)) + } +} + +func generateFees(l *int) []uint64 { + var length int + if l != nil { + length = *l + } else { + // Generate sequences with a length between 0 and 1000 + length = rand.Intn(100) + } + result := make([]uint64, length) + lastFee := uint64(0) + for i := 0; i < length; i++ { + if lastFee != 0 && rand.Intn(100) <= 25 { + // To test the Mode correctly, generate a repetition with a chance of 25% + result[i] = lastFee + } else { + // generate fees between 100 and 1000 + lastFee = uint64(rand.Intn(900) + 100) + result[i] = lastFee + } + } + return result +} + +func BenchmarkComputeFeeDistribution(b *testing.B) { + length := 5000 + fees := generateFees(&length) + b.Run("computeFeeDistribution", func(b *testing.B) { + for i := 0; i < b.N; i++ { + computeFeeDistribution(fees, 0) + } + }) + b.Run("alternativeComputeFeeDistribution", func(b *testing.B) { + for i := 0; i < b.N; i++ { + alternativeComputeFeeDistribution(fees, 0) + } + }) +} + +func alternativeComputeFeeDistribution(fees []uint64, ledgerCount uint32) (FeeDistribution, error) { + if len(fees) == 0 { + return FeeDistribution{}, nil + } + input := stats.LoadRawData(fees) + max, err := input.Max() + if err != nil { + return FeeDistribution{}, err + } + min, err := input.Min() + if err != nil { + return FeeDistribution{}, err + } + modeSeq, err := input.Mode() + if err != nil { + return FeeDistribution{}, err + } + var mode uint64 + if len(modeSeq) == 0 { + // mode can have length 0 if no value is repeated more than the rest + slices.Sort(fees) + mode = fees[0] + } else { + mode = uint64(modeSeq[0]) + } + p10, err := input.PercentileNearestRank(float64(10)) + if err != nil { + return FeeDistribution{}, err + } + p20, err := input.PercentileNearestRank(float64(20)) + if err != nil { + return FeeDistribution{}, err + } + p30, err := input.PercentileNearestRank(float64(30)) + if err != nil { + return FeeDistribution{}, err + } + p40, err := input.PercentileNearestRank(float64(40)) + if err != nil { + return FeeDistribution{}, err + } + p50, err := input.PercentileNearestRank(float64(50)) + if err != nil { + return FeeDistribution{}, err + } + p60, err := input.PercentileNearestRank(float64(60)) + if err != nil { + return FeeDistribution{}, err + } + p70, err := input.PercentileNearestRank(float64(70)) + if err != nil { + return FeeDistribution{}, err + } + p80, err := input.PercentileNearestRank(float64(80)) + if err != nil { + return FeeDistribution{}, err + } + p90, err := input.PercentileNearestRank(float64(90)) + if err != nil { + return FeeDistribution{}, err + } + p95, err := input.PercentileNearestRank(float64(95)) + if err != nil { + return FeeDistribution{}, err + } + p99, err := input.PercentileNearestRank(float64(99)) + if err != nil { + return FeeDistribution{}, err + } + + result := FeeDistribution{ + Max: uint64(max), + Min: uint64(min), + Mode: mode, + P10: uint64(p10), + P20: uint64(p20), + P30: uint64(p30), + P40: uint64(p40), + P50: uint64(p50), + P60: uint64(p60), + P70: uint64(p70), + P80: uint64(p80), + P90: uint64(p90), + P95: uint64(p95), + P99: uint64(p99), + FeeCount: uint32(len(fees)), + LedgerCount: ledgerCount, + } + return result, nil +} diff --git a/cmd/soroban-rpc/internal/ingest/service.go b/cmd/soroban-rpc/internal/ingest/service.go index 9979db3b..41116a59 100644 --- a/cmd/soroban-rpc/internal/ingest/service.go +++ b/cmd/soroban-rpc/internal/ingest/service.go @@ -18,6 +18,7 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" @@ -35,6 +36,7 @@ type Config struct { DB db.ReadWriter EventStore *events.MemoryStore TransactionStore *transactions.MemoryStore + FeeWindows *feewindow.FeeWindows NetworkPassPhrase string Archive historyarchive.ArchiveInterface LedgerBackend backends.LedgerBackend @@ -83,6 +85,7 @@ func newService(cfg Config) *Service { db: cfg.DB, eventStore: cfg.EventStore, transactionStore: cfg.TransactionStore, + feeWindows: cfg.FeeWindows, ledgerBackend: cfg.LedgerBackend, networkPassPhrase: cfg.NetworkPassPhrase, timeout: cfg.Timeout, @@ -135,6 +138,7 @@ type Service struct { db db.ReadWriter eventStore *events.MemoryStore transactionStore *transactions.MemoryStore + feeWindows *feewindow.FeeWindows ledgerBackend backends.LedgerBackend timeout time.Duration networkPassPhrase string @@ -332,5 +336,10 @@ func (s *Service) ingestLedgerCloseMeta(tx db.WriteTx, ledgerCloseMeta xdr.Ledge if err := s.transactionStore.IngestTransactions(ledgerCloseMeta); err != nil { return err } + + if err := s.feeWindows.IngestFees(ledgerCloseMeta); err != nil { + return err + } + return nil } diff --git a/cmd/soroban-rpc/internal/ingest/service_test.go b/cmd/soroban-rpc/internal/ingest/service_test.go index 1fb233c0..d5b86879 100644 --- a/cmd/soroban-rpc/internal/ingest/service_test.go +++ b/cmd/soroban-rpc/internal/ingest/service_test.go @@ -17,6 +17,7 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions" ) @@ -72,6 +73,7 @@ func TestIngestion(t *testing.T) { DB: mockDB, EventStore: events.NewMemoryStore(daemon, network.TestNetworkPassphrase, 1), TransactionStore: transactions.NewMemoryStore(daemon, network.TestNetworkPassphrase, 1), + FeeWindows: feewindow.NewFeeWindows(1, 1, network.TestNetworkPassphrase), LedgerBackend: mockLedgerBackend, Daemon: daemon, NetworkPassPhrase: network.TestNetworkPassphrase, diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 3b302aa7..9151d629 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -20,6 +20,7 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/network" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions" @@ -48,6 +49,7 @@ func (h Handler) Close() { type HandlerParams struct { EventStore *events.MemoryStore TransactionStore *transactions.MemoryStore + FeeStatWindows *feewindow.FeeWindows LedgerEntryReader db.LedgerEntryReader LedgerReader db.LedgerReader Logger *log.Entry @@ -220,6 +222,13 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { queueLimit: cfg.RequestBacklogSimulateTransactionQueueLimit, requestDurationLimit: cfg.MaxSimulateTransactionExecutionDuration, }, + { + methodName: "getFeeStats", + underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, ledgerRangeGetter), + longName: "get_fee_stats", + queueLimit: cfg.RequestBacklogGetFeeStatsTransactionQueueLimit, + requestDurationLimit: cfg.MaxGetFeeStatsExecutionDuration, + }, } handlersMap := handler.Map{} for _, handler := range handlers { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 8b4a9439..841ec192 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -295,7 +295,7 @@ type PaginationOptions struct { type GetEventsResponse struct { Events []EventInfo `json:"events"` - LatestLedger int64 `json:"latestLedger"` + LatestLedger uint32 `json:"latestLedger"` } type eventScanner interface { @@ -372,7 +372,7 @@ func (h eventsRPCHandler) getEvents(request GetEventsRequest) (GetEventsResponse results = append(results, info) } return GetEventsResponse{ - LatestLedger: int64(latestLedger), + LatestLedger: uint32(latestLedger), Events: results, }, nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 087a3ffd..1960ad0d 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -669,7 +669,7 @@ func TestGetEvents(t *testing.T) { }, }) assert.NoError(t, err) - assert.Equal(t, int64(1), results.LatestLedger) + assert.Equal(t, uint32(1), results.LatestLedger) expectedIds := []string{ events.Cursor{Ledger: 1, Tx: 1, Op: 0, Event: 0}.String(), diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go new file mode 100644 index 00000000..e1f1182b --- /dev/null +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -0,0 +1,68 @@ +package methods + +import ( + "context" + + "github.com/creachadair/jrpc2" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" +) + +type FeeDistribution struct { + Max uint64 `json:"max,string"` + Min uint64 `json:"min,string"` + Mode uint64 `json:"mode,string"` + P10 uint64 `json:"p10,string"` + P20 uint64 `json:"p20,string"` + P30 uint64 `json:"p30,string"` + P40 uint64 `json:"p40,string"` + P50 uint64 `json:"p50,string"` + P60 uint64 `json:"p60,string"` + P70 uint64 `json:"p70,string"` + P80 uint64 `json:"p80,string"` + P90 uint64 `json:"p90,string"` + P95 uint64 `json:"p95,string"` + P99 uint64 `json:"p99,string"` + TransactionCount uint32 `json:"transactionCount,string"` + LedgerCount uint32 `json:"ledgerCount"` +} + +func convertFeeDistribution(distribution feewindow.FeeDistribution) FeeDistribution { + return FeeDistribution{ + Max: distribution.Max, + Min: distribution.Min, + Mode: distribution.Mode, + P10: distribution.P10, + P20: distribution.P20, + P30: distribution.P30, + P40: distribution.P40, + P50: distribution.P50, + P60: distribution.P60, + P70: distribution.P70, + P80: distribution.P80, + P90: distribution.P90, + P95: distribution.P95, + P99: distribution.P99, + TransactionCount: distribution.FeeCount, + LedgerCount: distribution.LedgerCount, + } + +} + +type GetFeeStatsResult struct { + SorobanInclusionFee FeeDistribution `json:"sorobanInclusionFee"` + InclusionFee FeeDistribution `json:"inclusionFee"` + LatestLedger uint32 `json:"latestLedger"` +} + +// NewGetFeeStatsHandler returns a handler obtaining fee statistics +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, ledgerRangeGetter LedgerRangeGetter) jrpc2.Handler { + return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { + result := GetFeeStatsResult{ + SorobanInclusionFee: convertFeeDistribution(windows.SorobanInclusionFeeWindow.GetFeeDistribution()), + InclusionFee: convertFeeDistribution(windows.ClassicFeeWindow.GetFeeDistribution()), + LatestLedger: ledgerRangeGetter.GetLedgerRange().LastLedger.Sequence, + } + return result, nil + }) +} diff --git a/cmd/soroban-rpc/internal/test/get_fee_stats_test.go b/cmd/soroban-rpc/internal/test/get_fee_stats_test.go new file mode 100644 index 00000000..b1de24e7 --- /dev/null +++ b/cmd/soroban-rpc/internal/test/get_fee_stats_test.go @@ -0,0 +1,122 @@ +package test + +import ( + "context" + "testing" + + "github.com/creachadair/jrpc2" + "github.com/creachadair/jrpc2/jhttp" + "github.com/stellar/go/keypair" + "github.com/stellar/go/txnbuild" + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods" +) + +func TestGetFeeStats(t *testing.T) { + test := NewTest(t, nil) + + ch := jhttp.NewChannel(test.sorobanRPCURL(), nil) + client := jrpc2.NewClient(ch, nil) + + sourceAccount := keypair.Root(StandaloneNetworkPassphrase) + address := sourceAccount.Address() + account := txnbuild.NewSimpleAccount(address, 0) + + // Submit soroban transaction + contractBinary := getHelloWorldContract(t) + params := preflightTransactionParams(t, client, txnbuild.TransactionParams{ + SourceAccount: &account, + IncrementSequenceNum: true, + Operations: []txnbuild.Operation{ + createInstallContractCodeOperation(account.AccountID, contractBinary), + }, + BaseFee: txnbuild.MinBaseFee, + Preconditions: txnbuild.Preconditions{ + TimeBounds: txnbuild.NewInfiniteTimeout(), + }, + }) + tx, err := txnbuild.NewTransaction(params) + assert.NoError(t, err) + sorobanTxResponse := sendSuccessfulTransaction(t, client, sourceAccount, tx) + var sorobanTxResult xdr.TransactionResult + require.NoError(t, xdr.SafeUnmarshalBase64(sorobanTxResponse.ResultXdr, &sorobanTxResult)) + sorobanTotalFee := sorobanTxResult.FeeCharged + var sorobanTxMeta xdr.TransactionMeta + require.NoError(t, xdr.SafeUnmarshalBase64(sorobanTxResponse.ResultMetaXdr, &sorobanTxMeta)) + sorobanFees := sorobanTxMeta.MustV3().SorobanMeta.Ext.MustV1() + sorobanResourceFeeCharged := sorobanFees.TotalRefundableResourceFeeCharged + sorobanFees.TotalNonRefundableResourceFeeCharged + sorobanInclusionFee := uint64(sorobanTotalFee - sorobanResourceFeeCharged) + + // Submit classic transaction + params = txnbuild.TransactionParams{ + SourceAccount: &account, + IncrementSequenceNum: true, + Operations: []txnbuild.Operation{ + &txnbuild.BumpSequence{BumpTo: account.Sequence + 100}, + }, + BaseFee: txnbuild.MinBaseFee, + Memo: nil, + Preconditions: txnbuild.Preconditions{ + TimeBounds: txnbuild.NewInfiniteTimeout(), + }, + } + tx, err = txnbuild.NewTransaction(params) + assert.NoError(t, err) + classicTxResponse := sendSuccessfulTransaction(t, client, sourceAccount, tx) + var classicTxResult xdr.TransactionResult + require.NoError(t, xdr.SafeUnmarshalBase64(classicTxResponse.ResultXdr, &classicTxResult)) + classicFee := uint64(classicTxResult.FeeCharged) + + var result methods.GetFeeStatsResult + if err := client.CallResult(context.Background(), "getFeeStats", nil, &result); err != nil { + t.Fatalf("rpc call failed: %v", err) + } + expectedResult := methods.GetFeeStatsResult{ + SorobanInclusionFee: methods.FeeDistribution{ + Max: sorobanInclusionFee, + Min: sorobanInclusionFee, + Mode: sorobanInclusionFee, + P10: sorobanInclusionFee, + P20: sorobanInclusionFee, + P30: sorobanInclusionFee, + P40: sorobanInclusionFee, + P50: sorobanInclusionFee, + P60: sorobanInclusionFee, + P70: sorobanInclusionFee, + P80: sorobanInclusionFee, + P90: sorobanInclusionFee, + P95: sorobanInclusionFee, + P99: sorobanInclusionFee, + TransactionCount: 1, + LedgerCount: result.SorobanInclusionFee.LedgerCount, + }, + InclusionFee: methods.FeeDistribution{ + Max: classicFee, + Min: classicFee, + Mode: classicFee, + P10: classicFee, + P20: classicFee, + P30: classicFee, + P40: classicFee, + P50: classicFee, + P60: classicFee, + P70: classicFee, + P80: classicFee, + P90: classicFee, + P95: classicFee, + P99: classicFee, + TransactionCount: 1, + LedgerCount: result.InclusionFee.LedgerCount, + }, + LatestLedger: result.LatestLedger, + } + assert.Equal(t, expectedResult, result) + + // check ledgers separately + assert.Greater(t, result.InclusionFee.LedgerCount, uint32(0)) + assert.Greater(t, result.SorobanInclusionFee.LedgerCount, uint32(0)) + assert.Greater(t, result.LatestLedger, uint32(0)) +} diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go index 882e2250..fc22c986 100644 --- a/cmd/soroban-rpc/internal/test/integration.go +++ b/cmd/soroban-rpc/internal/test/integration.go @@ -36,7 +36,6 @@ const ( stellarCorePort = 11626 stellarCoreArchiveHost = "localhost:1570" goModFile = "go.mod" - goMonorepoGithubPath = "github.com/stellar/go" friendbotURL = "http://localhost:8000/friendbot" // Needed when Core is run with ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true diff --git a/go.mod b/go.mod index 1fda2cb3..b946b5ad 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/creachadair/jrpc2 v1.2.0 github.com/go-chi/chi v4.1.2+incompatible github.com/mattn/go-sqlite3 v1.14.17 + github.com/montanaflynn/stats v0.7.1 github.com/pelletier/go-toml v1.9.5 github.com/prometheus/client_golang v1.17.0 github.com/rs/cors v1.10.1 diff --git a/go.sum b/go.sum index 187ad20e..63129b95 100644 --- a/go.sum +++ b/go.sum @@ -277,6 +277,8 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvls github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db h1:eZgFHVkk9uOTaOQLC6tgjkzdp7Ays8eEVecBcfHZlJQ= github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=