diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index dc865935..820f2f0a 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -5,6 +5,10 @@ on: branches: [ main, release/** ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + defaults: run: shell: bash diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f14121e9..fb492f0c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,6 +5,10 @@ on: branches: [ main, release/** ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + jobs: integration: name: System tests diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index 21db63bb..f7696412 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -4,6 +4,10 @@ on: branches: [ main, release/** ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + permissions: contents: read # Optional: allow read access to pull request. Use with `only-new-issues` option. diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6b1cb36a..487ee752 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -5,6 +5,10 @@ on: branches: [ main, release/** ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + defaults: run: shell: bash diff --git a/.github/workflows/soroban-rpc.yml b/.github/workflows/soroban-rpc.yml index 7edfe2e3..c9e3dc1f 100644 --- a/.github/workflows/soroban-rpc.yml +++ b/.github/workflows/soroban-rpc.yml @@ -9,6 +9,10 @@ on: branches: [ main, release/** ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + jobs: test: name: Unit tests @@ -100,10 +104,10 @@ jobs: SOROBAN_RPC_INTEGRATION_TESTS_ENABLED: true SOROBAN_RPC_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core - PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.1-1897.dfd3dbff1.focal - PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.1-1897.dfd3dbff1.focal - PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.1-1897.dfd3dbff1.focal - PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.1-1897.dfd3dbff1.focal + PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.1.0-1909.rc1.b3aeb14cc.focal + PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21.1.0-1909.rc1.b3aeb14cc.focal + PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.1.0-1909.rc1.b3aeb14cc.focal + PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.1.0-1909.rc1.b3aeb14cc.focal steps: - uses: actions/checkout@v4 with: diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index 9d14084a..9fb80996 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -121,11 +121,9 @@ func (cfg *Config) options() Options { case nil: return nil case string: - return fmt.Errorf( - "could not parse %s: %w", - option.Name, - cfg.LogFormat.UnmarshalText([]byte(v)), - ) + if err := cfg.LogFormat.UnmarshalText([]byte(v)); err != nil { + return fmt.Errorf("could not parse %s: %w", option.Name, err) + } case LogFormat: cfg.LogFormat = v case *LogFormat: diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index ab8ec8a0..c6d287c6 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -296,7 +296,11 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow cfg.NetworkPassphrase, cfg.HistoryRetentionWindow, ) - feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase) + feewindows := feewindow.NewFeeWindows( + cfg.ClassicFeeStatsLedgerRetentionWindow, + cfg.SorobanFeeStatsLedgerRetentionWindow, + cfg.NetworkPassphrase, + ) readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout) defer cancelReadTxMeta() @@ -315,7 +319,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow initialSeq = currentSeq d.logger.WithFields(supportlog.F{ "seq": currentSeq, - }).Info("initializing in-memory store") + }).Info("initializing in-memory store and applying DB data migrations") } else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 { d.logger.WithFields(supportlog.F{ "seq": currentSeq, @@ -346,7 +350,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow if currentSeq != 0 { d.logger.WithFields(supportlog.F{ "seq": currentSeq, - }).Info("finished initializing in-memory store") + }).Info("finished initializing in-memory store and applying DB data migrations") } return feewindows, eventStore diff --git a/cmd/soroban-rpc/internal/daemon/interfaces/interfaces.go b/cmd/soroban-rpc/internal/daemon/interfaces/interfaces.go index b97cb376..7cb08168 100644 --- a/cmd/soroban-rpc/internal/daemon/interfaces/interfaces.go +++ b/cmd/soroban-rpc/internal/daemon/interfaces/interfaces.go @@ -5,6 +5,7 @@ import ( "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/ingest/ledgerbackend" proto "github.com/stellar/go/protocols/stellarcore" ) @@ -15,6 +16,7 @@ type Daemon interface { MetricsRegistry() *prometheus.Registry MetricsNamespace() string CoreClient() CoreClient + GetCore() *ledgerbackend.CaptiveStellarCore } type CoreClient interface { diff --git a/cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go b/cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go index db8d513d..a5ba0db3 100644 --- a/cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go +++ b/cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go @@ -5,6 +5,7 @@ import ( "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/ingest/ledgerbackend" proto "github.com/stellar/go/protocols/stellarcore" ) @@ -14,6 +15,7 @@ type NoOpDaemon struct { metricsRegistry *prometheus.Registry metricsNamespace string coreClient noOpCoreClient + core *ledgerbackend.CaptiveStellarCore } func MakeNoOpDeamon() *NoOpDaemon { @@ -36,6 +38,10 @@ func (d *NoOpDaemon) CoreClient() CoreClient { return d.coreClient } +func (d *NoOpDaemon) GetCore() *ledgerbackend.CaptiveStellarCore { + return d.core +} + type noOpCoreClient struct{} func (s noOpCoreClient) Info(context.Context) (*proto.InfoResponse, error) { diff --git a/cmd/soroban-rpc/internal/daemon/metrics.go b/cmd/soroban-rpc/internal/daemon/metrics.go index 2e409cf4..f283a3b7 100644 --- a/cmd/soroban-rpc/internal/daemon/metrics.go +++ b/cmd/soroban-rpc/internal/daemon/metrics.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/stellar/go/clients/stellarcore" + "github.com/stellar/go/ingest/ledgerbackend" proto "github.com/stellar/go/protocols/stellarcore" supportlog "github.com/stellar/go/support/log" "github.com/stellar/go/support/logmetrics" @@ -106,6 +107,10 @@ func (d *Daemon) CoreClient() interfaces.CoreClient { return d.coreClient } +func (d *Daemon) GetCore() *ledgerbackend.CaptiveStellarCore { + return d.core +} + func (d *Daemon) Logger() *supportlog.Entry { return d.logger } diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 7c547693..39e43b83 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -5,6 +5,7 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" + "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index dbe07ab5..552cc98d 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -115,9 +115,13 @@ type guardedMigration struct { db *DB migration MigrationApplier alreadyMigrated bool + logger *log.Entry + applyLogged bool } -func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, factory migrationApplierFactory, db *DB) (Migration, error) { +func newGuardedDataMigration( + ctx context.Context, uniqueMigrationName string, logger *log.Entry, factory migrationApplierFactory, db *DB, +) (Migration, error) { migrationDB := &DB{ cache: db.cache, SessionInterface: db.SessionInterface.Clone(), @@ -132,7 +136,7 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa return nil, err } latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx) - if err != nil && err != ErrEmptyDB { + if err != nil && !errors.Is(err, ErrEmptyDB) { err = errors.Join(err, migrationDB.Rollback()) return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err) } @@ -146,6 +150,7 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa db: migrationDB, migration: applier, alreadyMigrated: previouslyMigrated, + logger: logger, } return guardedMigration, nil } @@ -156,6 +161,10 @@ func (g *guardedMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) // but, just in case. return nil } + if !g.applyLogged { + g.logger.WithField("ledger", meta.LedgerSequence()).Info("applying migration") + g.applyLogged = true + } return g.migration.Apply(ctx, meta) } @@ -177,19 +186,20 @@ func (g *guardedMigration) Commit(ctx context.Context) error { return g.db.Commit() } -func (g *guardedMigration) Rollback(ctx context.Context) error { +func (g *guardedMigration) Rollback(_ context.Context) error { return g.db.Rollback() } func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) { migrationName := "TransactionsTable" + logger = logger.WithField("migration", migrationName) factory := newTransactionTableMigration( ctx, - logger.WithField("migration", migrationName), + logger, cfg.HistoryRetentionWindow, cfg.NetworkPassphrase, ) - m, err := newGuardedDataMigration(ctx, migrationName, factory, db) + m, err := newGuardedDataMigration(ctx, migrationName, logger, factory, db) if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) } diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 0a7c85a9..8bccc58e 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -111,7 +111,7 @@ func (txn *transactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error _, err = query.RunWith(txn.stmtCache).Exec() L.WithField("duration", time.Since(start)). - Infof("Ingested %d transaction lookups", len(transactions)) + Debugf("Ingested %d transaction lookups", len(transactions)) return err } diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/docker/docker-compose.yml b/cmd/soroban-rpc/internal/integrationtest/infrastructure/docker/docker-compose.yml index 579cf67b..de3bcc49 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/docker/docker-compose.yml +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/docker/docker-compose.yml @@ -13,7 +13,7 @@ services: # Note: Please keep the image pinned to an immutable tag matching the Captive Core version. # This avoids implicit updates which break compatibility between # the Core container and captive core. - image: ${CORE_IMAGE:-stellar/unsafe-stellar-core:21.0.1-1897.dfd3dbff1.focal} + image: ${CORE_IMAGE:-stellar/stellar-core:21.1.0-1909.rc1.b3aeb14cc.focal} depends_on: - core-postgres environment: diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index a6329456..677b7c46 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -168,8 +168,13 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { requestDurationLimit: cfg.MaxGetEventsExecutionDuration, }, { - methodName: "getNetwork", - underlyingHandler: methods.NewGetNetworkHandler(params.Daemon, cfg.NetworkPassphrase, cfg.FriendbotURL), + methodName: "getNetwork", + underlyingHandler: methods.NewGetNetworkHandler( + cfg.NetworkPassphrase, + cfg.FriendbotURL, + params.LedgerEntryReader, + params.LedgerReader, + ), longName: "get_network", queueLimit: cfg.RequestBacklogGetNetworkQueueLimit, requestDurationLimit: cfg.MaxGetNetworkExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_network.go b/cmd/soroban-rpc/internal/methods/get_network.go index f706e7c6..16b2b650 100644 --- a/cmd/soroban-rpc/internal/methods/get_network.go +++ b/cmd/soroban-rpc/internal/methods/get_network.go @@ -5,7 +5,7 @@ import ( "github.com/creachadair/jrpc2" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" ) type GetNetworkRequest struct{} @@ -17,20 +17,25 @@ type GetNetworkResponse struct { } // NewGetNetworkHandler returns a json rpc handler to for the getNetwork method -func NewGetNetworkHandler(daemon interfaces.Daemon, networkPassphrase, friendbotURL string) jrpc2.Handler { - coreClient := daemon.CoreClient() +func NewGetNetworkHandler( + networkPassphrase string, + friendbotURL string, + ledgerEntryReader db.LedgerEntryReader, + ledgerReader db.LedgerReader, +) jrpc2.Handler { return NewHandler(func(ctx context.Context, request GetNetworkRequest) (GetNetworkResponse, error) { - info, err := coreClient.Info(ctx) + protocolVersion, err := getProtocolVersion(ctx, ledgerEntryReader, ledgerReader) if err != nil { - return GetNetworkResponse{}, (&jrpc2.Error{ + return GetNetworkResponse{}, &jrpc2.Error{ Code: jrpc2.InternalError, Message: err.Error(), - }) + } } + return GetNetworkResponse{ FriendbotURL: friendbotURL, Passphrase: networkPassphrase, - ProtocolVersion: info.Info.ProtocolVersion, + ProtocolVersion: int(protocolVersion), }, nil }) } diff --git a/cmd/soroban-rpc/internal/methods/get_transactions.go b/cmd/soroban-rpc/internal/methods/get_transactions.go index 1d16f259..13ff359f 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions.go @@ -14,6 +14,7 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/support/log" "github.com/stellar/go/toid" + "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" @@ -93,38 +94,19 @@ type transactionsRPCHandler struct { networkPassphrase string } -// getTransactionsByLedgerSequence fetches transactions between the start and end ledgers, inclusive of both. -// The number of ledgers returned can be tuned using the pagination options - cursor and limit. -func (h transactionsRPCHandler) getTransactionsByLedgerSequence(ctx context.Context, request GetTransactionsRequest) (GetTransactionsResponse, error) { - ledgerRange, err := h.ledgerReader.GetLedgerRange(ctx) - if err != nil { - return GetTransactionsResponse{}, &jrpc2.Error{ - Code: jrpc2.InternalError, - Message: err.Error(), - } - } - - err = request.isValid(h.maxLimit, ledgerRange) - if err != nil { - return GetTransactionsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidRequest, - Message: err.Error(), - } - } - - // Move start to pagination cursor +// initializePagination sets the pagination limit and cursor +func (h transactionsRPCHandler) initializePagination(request GetTransactionsRequest) (toid.ID, uint, error) { start := toid.New(int32(request.StartLedger), 1, 1) limit := h.defaultLimit if request.Pagination != nil { if request.Pagination.Cursor != "" { cursorInt, err := strconv.ParseInt(request.Pagination.Cursor, 10, 64) if err != nil { - return GetTransactionsResponse{}, &jrpc2.Error{ + return toid.ID{}, 0, &jrpc2.Error{ Code: jrpc2.InvalidParams, Message: err.Error(), } } - *start = toid.Parse(cursorInt) // increment tx index because, when paginating, // we start with the item right after the cursor @@ -134,92 +116,142 @@ func (h transactionsRPCHandler) getTransactionsByLedgerSequence(ctx context.Cont limit = request.Pagination.Limit } } + return *start, limit, nil +} - // Iterate through each ledger and its transactions until limit or end range is reached. - // The latest ledger acts as the end ledger range for the request. - var txns []TransactionInfo - cursor := toid.New(0, 0, 0) -LedgerLoop: - for ledgerSeq := start.LedgerSequence; ledgerSeq <= int32(ledgerRange.LastLedger.Sequence); ledgerSeq++ { - // Get ledger close meta from db - ledger, found, err := h.ledgerReader.GetLedger(ctx, uint32(ledgerSeq)) - if err != nil { - return GetTransactionsResponse{}, &jrpc2.Error{ +// fetchLedgerData calls the meta table to fetch the corresponding ledger data. +func (h transactionsRPCHandler) fetchLedgerData(ctx context.Context, ledgerSeq uint32) (xdr.LedgerCloseMeta, error) { + ledger, found, err := h.ledgerReader.GetLedger(ctx, ledgerSeq) + if err != nil { + return ledger, &jrpc2.Error{ + Code: jrpc2.InternalError, + Message: err.Error(), + } + } else if !found { + return ledger, &jrpc2.Error{ + Code: jrpc2.InvalidParams, + Message: fmt.Sprintf("database does not contain metadata for ledger: %d", ledgerSeq), + } + } + return ledger, nil +} + +// processTransactionsInLedger cycles through all the transactions in a ledger, extracts the transaction info +// and builds the list of transactions. +func (h transactionsRPCHandler) processTransactionsInLedger(ledger xdr.LedgerCloseMeta, start toid.ID, + txns *[]TransactionInfo, limit uint, +) (*toid.ID, bool, error) { + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(h.networkPassphrase, ledger) + if err != nil { + return nil, false, &jrpc2.Error{ + Code: jrpc2.InternalError, + Message: err.Error(), + } + } + + startTxIdx := 1 + ledgerSeq := ledger.LedgerSequence() + if int32(ledgerSeq) == start.LedgerSequence { + startTxIdx = int(start.TransactionOrder) + if ierr := reader.Seek(startTxIdx - 1); ierr != nil && !errors.Is(ierr, io.EOF) { + return nil, false, &jrpc2.Error{ Code: jrpc2.InternalError, - Message: err.Error(), + Message: ierr.Error(), + } + } + } + + txCount := ledger.CountTransactions() + cursor := toid.New(int32(ledgerSeq), 0, 1) + for i := startTxIdx; i <= txCount; i++ { + cursor.TransactionOrder = int32(i) + + ingestTx, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break } - } else if !found { - return GetTransactionsResponse{}, &jrpc2.Error{ + return nil, false, &jrpc2.Error{ Code: jrpc2.InvalidParams, - Message: fmt.Sprintf("ledger close meta not found: %d", ledgerSeq), + Message: err.Error(), } } - // Initialize tx reader. - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(h.networkPassphrase, ledger) + tx, err := db.ParseTransaction(ledger, ingestTx) if err != nil { - return GetTransactionsResponse{}, &jrpc2.Error{ + return nil, false, &jrpc2.Error{ Code: jrpc2.InternalError, Message: err.Error(), } } - // Move the reader to specific tx idx - startTxIdx := 1 - if ledgerSeq == start.LedgerSequence { - startTxIdx = int(start.TransactionOrder) - if ierr := reader.Seek(startTxIdx - 1); ierr != nil && ierr != io.EOF { - return GetTransactionsResponse{}, &jrpc2.Error{ - Code: jrpc2.InternalError, - Message: ierr.Error(), - } - } + txInfo := TransactionInfo{ + ApplicationOrder: tx.ApplicationOrder, + FeeBump: tx.FeeBump, + ResultXdr: base64.StdEncoding.EncodeToString(tx.Result), + ResultMetaXdr: base64.StdEncoding.EncodeToString(tx.Meta), + EnvelopeXdr: base64.StdEncoding.EncodeToString(tx.Envelope), + DiagnosticEventsXDR: base64EncodeSlice(tx.Events), + Ledger: tx.Ledger.Sequence, + LedgerCloseTime: tx.Ledger.CloseTime, + } + txInfo.Status = TransactionStatusFailed + if tx.Successful { + txInfo.Status = TransactionStatusSuccess } - // Decode transaction info from ledger meta - txCount := ledger.CountTransactions() - for i := startTxIdx; i <= txCount; i++ { - cursor = toid.New(int32(ledger.LedgerSequence()), int32(i), 1) + *txns = append(*txns, txInfo) + if len(*txns) >= int(limit) { + return cursor, true, nil + } + } - ingestTx, err := reader.Read() - if err != nil { - if errors.Is(err, io.EOF) { - // No more transactions to read. Start from next ledger - break - } - return GetTransactionsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidParams, - Message: err.Error(), - } - } + return cursor, false, nil +} - tx, err := db.ParseTransaction(ledger, ingestTx) - if err != nil { - return GetTransactionsResponse{}, &jrpc2.Error{ - Code: jrpc2.InternalError, - Message: err.Error(), - } - } +// getTransactionsByLedgerSequence fetches transactions between the start and end ledgers, inclusive of both. +// The number of ledgers returned can be tuned using the pagination options - cursor and limit. +func (h transactionsRPCHandler) getTransactionsByLedgerSequence(ctx context.Context, + request GetTransactionsRequest, +) (GetTransactionsResponse, error) { + ledgerRange, err := h.ledgerReader.GetLedgerRange(ctx) + if err != nil { + return GetTransactionsResponse{}, &jrpc2.Error{ + Code: jrpc2.InternalError, + Message: err.Error(), + } + } - txInfo := TransactionInfo{ - ApplicationOrder: tx.ApplicationOrder, - FeeBump: tx.FeeBump, - ResultXdr: base64.StdEncoding.EncodeToString(tx.Result), - ResultMetaXdr: base64.StdEncoding.EncodeToString(tx.Meta), - EnvelopeXdr: base64.StdEncoding.EncodeToString(tx.Envelope), - DiagnosticEventsXDR: base64EncodeSlice(tx.Events), - Ledger: tx.Ledger.Sequence, - LedgerCloseTime: tx.Ledger.CloseTime, - } - txInfo.Status = TransactionStatusFailed - if tx.Successful { - txInfo.Status = TransactionStatusSuccess - } + err = request.isValid(h.maxLimit, ledgerRange) + if err != nil { + return GetTransactionsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidRequest, + Message: err.Error(), + } + } - txns = append(txns, txInfo) - if len(txns) >= int(limit) { - break LedgerLoop - } + start, limit, err := h.initializePagination(request) + if err != nil { + return GetTransactionsResponse{}, err + } + + // Iterate through each ledger and its transactions until limit or end range is reached. + // The latest ledger acts as the end ledger range for the request. + var txns []TransactionInfo + var done bool + cursor := toid.New(0, 0, 0) + for ledgerSeq := start.LedgerSequence; ledgerSeq <= int32(ledgerRange.LastLedger.Sequence); ledgerSeq++ { + ledger, err := h.fetchLedgerData(ctx, uint32(ledgerSeq)) + if err != nil { + return GetTransactionsResponse{}, err + } + + cursor, done, err = h.processTransactionsInLedger(ledger, start, &txns, limit) + if err != nil { + return GetTransactionsResponse{}, err + } + if done { + break } } diff --git a/cmd/soroban-rpc/internal/methods/get_transactions_test.go b/cmd/soroban-rpc/internal/methods/get_transactions_test.go index 76f833c4..ef695ffb 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions_test.go @@ -238,7 +238,7 @@ func TestGetTransactions_LedgerNotFound(t *testing.T) { } response, err := handler.getTransactionsByLedgerSequence(context.TODO(), request) - expectedErr := fmt.Errorf("[%d] ledger close meta not found: 2", jrpc2.InvalidParams) + expectedErr := fmt.Errorf("[%d] database does not contain metadata for ledger: 2", jrpc2.InvalidParams) assert.Equal(t, expectedErr.Error(), err.Error()) assert.Nil(t, response.Transactions) } diff --git a/cmd/soroban-rpc/internal/methods/get_version_info.go b/cmd/soroban-rpc/internal/methods/get_version_info.go index 6c3053a3..1d868ef2 100644 --- a/cmd/soroban-rpc/internal/methods/get_version_info.go +++ b/cmd/soroban-rpc/internal/methods/get_version_info.go @@ -22,35 +22,19 @@ type GetVersionInfoResponse struct { ProtocolVersion uint32 `json:"protocol_version"` //nolint:tagliatelle } -func NewGetVersionInfoHandler(logger *log.Entry, ledgerEntryReader db.LedgerEntryReader, ledgerReader db.LedgerReader, daemon interfaces.Daemon) jrpc2.Handler { - coreClient := daemon.CoreClient() - return handler.New(func(ctx context.Context) (GetVersionInfoResponse, error) { - var captiveCoreVersion string - info, err := coreClient.Info(ctx) - if err != nil { - logger.WithError(err).Info("error occurred while calling Info endpoint of core") - } else { - captiveCoreVersion = info.Info.Build - } - - // Fetch Protocol version - var protocolVersion uint32 - readTx, err := ledgerEntryReader.NewCachedTx(ctx) - if err != nil { - logger.WithError(err).Info("Cannot create read transaction") - } - defer func() { - _ = readTx.Done() - }() +func NewGetVersionInfoHandler( + logger *log.Entry, + ledgerEntryReader db.LedgerEntryReader, + ledgerReader db.LedgerReader, + daemon interfaces.Daemon, +) jrpc2.Handler { + core := daemon.GetCore() - latestLedger, err := readTx.GetLatestLedgerSequence() - if err != nil { - logger.WithError(err).Info("error occurred while getting latest ledger") - } - - _, protocolVersion, err = getBucketListSizeAndProtocolVersion(ctx, ledgerReader, latestLedger) + return handler.New(func(ctx context.Context) (GetVersionInfoResponse, error) { + captiveCoreVersion := core.GetCoreVersion() + protocolVersion, err := getProtocolVersion(ctx, ledgerEntryReader, ledgerReader) if err != nil { - logger.WithError(err).Info("error occurred while fetching protocol version") + logger.WithError(err).Error("failed to fetch protocol version") } return GetVersionInfoResponse{ diff --git a/cmd/soroban-rpc/internal/methods/simulate_transaction.go b/cmd/soroban-rpc/internal/methods/simulate_transaction.go index 264f386b..dfe5b9d2 100644 --- a/cmd/soroban-rpc/internal/methods/simulate_transaction.go +++ b/cmd/soroban-rpc/internal/methods/simulate_transaction.go @@ -286,7 +286,11 @@ func base64EncodeSlice(in [][]byte) []string { return result } -func getBucketListSizeAndProtocolVersion(ctx context.Context, ledgerReader db.LedgerReader, latestLedger uint32) (uint64, uint32, error) { +func getBucketListSizeAndProtocolVersion( + ctx context.Context, + ledgerReader db.LedgerReader, + latestLedger uint32, +) (uint64, uint32, error) { // obtain bucket size closeMeta, ok, err := ledgerReader.GetLedger(ctx, latestLedger) if err != nil { diff --git a/cmd/soroban-rpc/internal/methods/util.go b/cmd/soroban-rpc/internal/methods/util.go new file mode 100644 index 00000000..eafad857 --- /dev/null +++ b/cmd/soroban-rpc/internal/methods/util.go @@ -0,0 +1,32 @@ +package methods + +import ( + "context" + "fmt" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" +) + +func getProtocolVersion( + ctx context.Context, + ledgerEntryReader db.LedgerEntryReader, + ledgerReader db.LedgerReader, +) (uint32, error) { + latestLedger, err := ledgerEntryReader.GetLatestLedgerSequence(ctx) + if err != nil { + return 0, err + } + + // obtain bucket size + closeMeta, ok, err := ledgerReader.GetLedger(ctx, latestLedger) + if err != nil { + return 0, err + } + if !ok { + return 0, fmt.Errorf("missing meta for latest ledger (%d)", latestLedger) + } + if closeMeta.V != 1 { + return 0, fmt.Errorf("latest ledger (%d) meta has unexpected verion (%d)", latestLedger, closeMeta.V) + } + return uint32(closeMeta.V1.LedgerHeader.Header.LedgerVersion), nil +} diff --git a/cmd/soroban-rpc/internal/methods/util_test.go b/cmd/soroban-rpc/internal/methods/util_test.go new file mode 100644 index 00000000..520a1bae --- /dev/null +++ b/cmd/soroban-rpc/internal/methods/util_test.go @@ -0,0 +1,92 @@ +package methods + +import ( + "context" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" +) + +func BenchmarkGetProtocolVersion(b *testing.B) { + dbx := NewTestDB(b) + daemon := interfaces.MakeNoOpDeamon() + + ledgerReader := db.NewLedgerReader(dbx) + _, exists, err := ledgerReader.GetLedger(context.Background(), 1) + require.NoError(b, err) + assert.False(b, exists) + + ledgerSequence := uint32(1) + tx, err := db.NewReadWriter(log.DefaultLogger, dbx, daemon, 150, 15, "passphrase").NewTx(context.Background()) + require.NoError(b, err) + require.NoError(b, tx.LedgerWriter().InsertLedger(createMockLedgerCloseMeta(ledgerSequence))) + require.NoError(b, tx.Commit(ledgerSequence)) + + ledgerEntryReader := db.NewLedgerEntryReader(dbx) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := getProtocolVersion(context.TODO(), ledgerEntryReader, ledgerReader) + if err != nil { + b.Fatalf("getProtocolVersion failed: %v", err) + } + } +} + +func TestGetProtocolVersion(t *testing.T) { + dbx := NewTestDB(t) + daemon := interfaces.MakeNoOpDeamon() + + ledgerReader := db.NewLedgerReader(dbx) + _, exists, err := ledgerReader.GetLedger(context.Background(), 1) + require.NoError(t, err) + assert.False(t, exists) + + ledgerSequence := uint32(1) + tx, err := db.NewReadWriter(log.DefaultLogger, dbx, daemon, 150, 15, "passphrase").NewTx(context.Background()) + require.NoError(t, err) + require.NoError(t, tx.LedgerWriter().InsertLedger(createMockLedgerCloseMeta(ledgerSequence))) + require.NoError(t, tx.Commit(ledgerSequence)) + + ledgerEntryReader := db.NewLedgerEntryReader(dbx) + protocolVersion, err := getProtocolVersion(context.TODO(), ledgerEntryReader, ledgerReader) + require.NoError(t, err) + require.Equal(t, uint32(20), protocolVersion) +} + +func createMockLedgerCloseMeta(ledgerSequence uint32) xdr.LedgerCloseMeta { + return xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Hash: xdr.Hash{}, + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(ledgerSequence), + LedgerVersion: xdr.Uint32(20), + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{}, + }, + }, + } +} + +func NewTestDB(tb testing.TB) *db.DB { + tmp := tb.TempDir() + dbPath := path.Join(tmp, "db.sqlite") + db, err := db.OpenSQLiteDB(dbPath) + require.NoError(tb, err) + tb.Cleanup(func() { + require.NoError(tb, db.Close()) + }) + return db +} diff --git a/cmd/soroban-rpc/lib/preflight/src/lib.rs b/cmd/soroban-rpc/lib/preflight/src/lib.rs index e84bb5cf..79fec314 100644 --- a/cmd/soroban-rpc/lib/preflight/src/lib.rs +++ b/cmd/soroban-rpc/lib/preflight/src/lib.rs @@ -189,14 +189,14 @@ impl CPreflightResult { } fn new_from_transaction_data( - transaction_data: &Option, - restore_preamble: &Option, + transaction_data: Option<&SorobanTransactionData>, + restore_preamble: Option<&RestoreOpSimulationResult>, error: String, ) -> Self { - let min_fee = transaction_data.as_ref().map_or(0, |d| d.resource_fee); + let min_fee = transaction_data.map_or(0, |d| d.resource_fee); let mut result = Self { error: string_to_c(error), - transaction_data: option_xdr_to_c(transaction_data.as_ref()), + transaction_data: option_xdr_to_c(transaction_data), min_fee, ..Default::default() }; @@ -361,10 +361,10 @@ fn preflight_extend_ttl_op( Err(e) => (None, Err(e)), }; - let error_str = extract_error_string(&maybe_restore_result, go_storage.as_ref()); + let error_str = extract_error_string(&maybe_restore_result, go_storage); Ok(CPreflightResult::new_from_transaction_data( - &maybe_transaction_data, - &maybe_restore_result.unwrap_or(None), + maybe_transaction_data.as_ref(), + maybe_restore_result.ok().flatten().as_ref(), error_str, )) } @@ -384,10 +384,8 @@ fn preflight_restore_op( ); let error_str = extract_error_string(&simulation_result, go_storage.as_ref()); CPreflightResult::new_from_transaction_data( - &simulation_result - .map(|r| Some(r.transaction_data)) - .unwrap_or(None), - &None, + simulation_result.ok().map(|r| r.transaction_data).as_ref(), + None, error_str, ) } @@ -655,10 +653,9 @@ impl SnapshotSourceWithArchive for GoLedgerStorage { fn extract_error_string(simulation_result: &Result, go_storage: &GoLedgerStorage) -> String { match simulation_result { Ok(_) => String::new(), - Err(e) => - // Override any simulation result with a storage error (if any). Simulation does not propagate the storage - // errors, but these provide more exact information on the root cause. - { + Err(e) => { + // Override any simulation result with a storage error (if any). Simulation does not propagate the storage + // errors, but these provide more exact information on the root cause. if let Some(e) = go_storage.internal_error.borrow().as_ref() { format!("{e:?}") } else {