Merge branch 'main' into events-db-backend

.github/workflows/dependency-check.yml

@@ -5,6 +5,10 @@ on:
     branches: [ main, release/** ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 defaults:
   run:
     shell: bash

.github/workflows/e2e.yml

@@ -5,6 +5,10 @@ on:
     branches: [ main, release/** ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   integration:
     name: System tests

.github/workflows/golang.yml

@@ -4,6 +4,10 @@ on:
     branches: [ main, release/** ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
   # Optional: allow read access to pull request. Use with `only-new-issues` option.

.github/workflows/rust.yml

@@ -5,6 +5,10 @@ on:
     branches: [ main, release/** ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 defaults:
   run:
     shell: bash

.github/workflows/soroban-rpc.yml

@@ -9,6 +9,10 @@ on:
     branches: [ main, release/** ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Unit tests
@@ -100,10 +104,10 @@ jobs:
       SOROBAN_RPC_INTEGRATION_TESTS_ENABLED: true
       SOROBAN_RPC_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
       SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
-      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.1-1897.dfd3dbff1.focal
-      PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.1-1897.dfd3dbff1.focal
-      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.1-1897.dfd3dbff1.focal
-      PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.1-1897.dfd3dbff1.focal
+      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.1.0-1909.rc1.b3aeb14cc.focal
+      PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21.1.0-1909.rc1.b3aeb14cc.focal
+      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.1.0-1909.rc1.b3aeb14cc.focal
+      PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.1.0-1909.rc1.b3aeb14cc.focal
     steps:
       - uses: actions/checkout@v4
         with:

cmd/soroban-rpc/internal/config/options.go

@@ -121,11 +121,9 @@ func (cfg *Config) options() Options {
 				case nil:
 					return nil
 				case string:
-					return fmt.Errorf(
-						"could not parse %s: %w",
-						option.Name,
-						cfg.LogFormat.UnmarshalText([]byte(v)),
-					)
+					if err := cfg.LogFormat.UnmarshalText([]byte(v)); err != nil {
+						return fmt.Errorf("could not parse %s: %w", option.Name, err)
+					}
 				case LogFormat:
 					cfg.LogFormat = v
 				case *LogFormat:

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -296,7 +296,11 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 		cfg.NetworkPassphrase,
 		cfg.HistoryRetentionWindow,
 	)
-	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
+	feewindows := feewindow.NewFeeWindows(
+		cfg.ClassicFeeStatsLedgerRetentionWindow,
+		cfg.SorobanFeeStatsLedgerRetentionWindow,
+		cfg.NetworkPassphrase,
+	)
 
 	readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
 	defer cancelReadTxMeta()
@@ -315,7 +319,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 			initialSeq = currentSeq
 			d.logger.WithFields(supportlog.F{
 				"seq": currentSeq,
-			}).Info("initializing in-memory store")
+			}).Info("initializing in-memory store and applying DB data migrations")
 		} else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 {
 			d.logger.WithFields(supportlog.F{
 				"seq": currentSeq,
@@ -346,7 +350,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 	if currentSeq != 0 {
 		d.logger.WithFields(supportlog.F{
 			"seq": currentSeq,
-		}).Info("finished initializing in-memory store")
+		}).Info("finished initializing in-memory store and applying DB data migrations")
 	}
 
 	return feewindows, eventStore

cmd/soroban-rpc/internal/daemon/interfaces/interfaces.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/stellar/go/ingest/ledgerbackend"
 	proto "github.com/stellar/go/protocols/stellarcore"
 )
 
@@ -15,6 +16,7 @@ type Daemon interface {
 	MetricsRegistry() *prometheus.Registry
 	MetricsNamespace() string
 	CoreClient() CoreClient
+	GetCore() *ledgerbackend.CaptiveStellarCore
 }
 
 type CoreClient interface {

cmd/soroban-rpc/internal/daemon/interfaces/noOpDaemon.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/stellar/go/ingest/ledgerbackend"
 	proto "github.com/stellar/go/protocols/stellarcore"
 )
 
@@ -14,6 +15,7 @@ type NoOpDaemon struct {
 	metricsRegistry  *prometheus.Registry
 	metricsNamespace string
 	coreClient       noOpCoreClient
+	core             *ledgerbackend.CaptiveStellarCore
 }
 
 func MakeNoOpDeamon() *NoOpDaemon {
@@ -36,6 +38,10 @@ func (d *NoOpDaemon) CoreClient() CoreClient {
 	return d.coreClient
 }
 
+func (d *NoOpDaemon) GetCore() *ledgerbackend.CaptiveStellarCore {
+	return d.core
+}
+
 type noOpCoreClient struct{}
 
 func (s noOpCoreClient) Info(context.Context) (*proto.InfoResponse, error) {

cmd/soroban-rpc/internal/daemon/metrics.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/collectors"
 
 	"github.com/stellar/go/clients/stellarcore"
+	"github.com/stellar/go/ingest/ledgerbackend"
 	proto "github.com/stellar/go/protocols/stellarcore"
 	supportlog "github.com/stellar/go/support/log"
 	"github.com/stellar/go/support/logmetrics"
@@ -106,6 +107,10 @@ func (d *Daemon) CoreClient() interfaces.CoreClient {
 	return d.coreClient
 }
 
+func (d *Daemon) GetCore() *ledgerbackend.CaptiveStellarCore {
+	return d.core
+}
+
 func (d *Daemon) Logger() *supportlog.Entry {
 	return d.logger
 }

cmd/soroban-rpc/internal/db/ledger.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	sq "github.com/Masterminds/squirrel"
+
 	"github.com/stellar/go/xdr"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"

cmd/soroban-rpc/internal/db/migration.go

@@ -115,9 +115,13 @@ type guardedMigration struct {
 	db              *DB
 	migration       MigrationApplier
 	alreadyMigrated bool
+	logger          *log.Entry
+	applyLogged     bool
 }
 
-func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, factory migrationApplierFactory, db *DB) (Migration, error) {
+func newGuardedDataMigration(
+	ctx context.Context, uniqueMigrationName string, logger *log.Entry, factory migrationApplierFactory, db *DB,
+) (Migration, error) {
 	migrationDB := &DB{
 		cache:            db.cache,
 		SessionInterface: db.SessionInterface.Clone(),
@@ -132,7 +136,7 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa
 		return nil, err
 	}
 	latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx)
-	if err != nil && err != ErrEmptyDB {
+	if err != nil && !errors.Is(err, ErrEmptyDB) {
 		err = errors.Join(err, migrationDB.Rollback())
 		return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err)
 	}
@@ -146,6 +150,7 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa
 		db:              migrationDB,
 		migration:       applier,
 		alreadyMigrated: previouslyMigrated,
+		logger:          logger,
 	}
 	return guardedMigration, nil
 }
@@ -156,6 +161,10 @@ func (g *guardedMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta)
 		// but, just in case.
 		return nil
 	}
+	if !g.applyLogged {
+		g.logger.WithField("ledger", meta.LedgerSequence()).Info("applying migration")
+		g.applyLogged = true
+	}
 	return g.migration.Apply(ctx, meta)
 }
 
@@ -177,19 +186,20 @@ func (g *guardedMigration) Commit(ctx context.Context) error {
 	return g.db.Commit()
 }
 
-func (g *guardedMigration) Rollback(ctx context.Context) error {
+func (g *guardedMigration) Rollback(_ context.Context) error {
 	return g.db.Rollback()
 }
 
 func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) {
 	migrationName := "TransactionsTable"
+	logger = logger.WithField("migration", migrationName)
 	factory := newTransactionTableMigration(
 		ctx,
-		logger.WithField("migration", migrationName),
+		logger,
 		cfg.HistoryRetentionWindow,
 		cfg.NetworkPassphrase,
 	)
-	m, err := newGuardedDataMigration(ctx, migrationName, factory, db)
+	m, err := newGuardedDataMigration(ctx, migrationName, logger, factory, db)
 	if err != nil {
 		return nil, fmt.Errorf("creating guarded transaction migration: %w", err)
 	}

cmd/soroban-rpc/internal/db/transaction.go

@@ -111,7 +111,7 @@ func (txn *transactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error
 	_, err = query.RunWith(txn.stmtCache).Exec()
 
 	L.WithField("duration", time.Since(start)).
-		Infof("Ingested %d transaction lookups", len(transactions))
+		Debugf("Ingested %d transaction lookups", len(transactions))
 
 	return err
 }

cmd/soroban-rpc/internal/integrationtest/infrastructure/docker/docker-compose.yml

@@ -13,7 +13,7 @@ services:
     # Note: Please keep the image pinned to an immutable tag matching the Captive Core version.
     #       This avoids implicit updates which break compatibility between
     #       the Core container and captive core.
-    image: ${CORE_IMAGE:-stellar/unsafe-stellar-core:21.0.1-1897.dfd3dbff1.focal}
+    image: ${CORE_IMAGE:-stellar/stellar-core:21.1.0-1909.rc1.b3aeb14cc.focal}
     depends_on:
       - core-postgres
     environment:

cmd/soroban-rpc/internal/jsonrpc.go

@@ -168,8 +168,13 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler {
 			requestDurationLimit: cfg.MaxGetEventsExecutionDuration,
 		},
 		{
-			methodName:           "getNetwork",
-			underlyingHandler:    methods.NewGetNetworkHandler(params.Daemon, cfg.NetworkPassphrase, cfg.FriendbotURL),
+			methodName: "getNetwork",
+			underlyingHandler: methods.NewGetNetworkHandler(
+				cfg.NetworkPassphrase,
+				cfg.FriendbotURL,
+				params.LedgerEntryReader,
+				params.LedgerReader,
+			),
 			longName:             "get_network",
 			queueLimit:           cfg.RequestBacklogGetNetworkQueueLimit,
 			requestDurationLimit: cfg.MaxGetNetworkExecutionDuration,

cmd/soroban-rpc/internal/methods/get_network.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/creachadair/jrpc2"
 
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces"
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db"
 )
 
 type GetNetworkRequest struct{}
@@ -17,20 +17,25 @@ type GetNetworkResponse struct {
 }
 
 // NewGetNetworkHandler returns a json rpc handler to for the getNetwork method
-func NewGetNetworkHandler(daemon interfaces.Daemon, networkPassphrase, friendbotURL string) jrpc2.Handler {
-	coreClient := daemon.CoreClient()
+func NewGetNetworkHandler(
+	networkPassphrase string,
+	friendbotURL string,
+	ledgerEntryReader db.LedgerEntryReader,
+	ledgerReader db.LedgerReader,
+) jrpc2.Handler {
 	return NewHandler(func(ctx context.Context, request GetNetworkRequest) (GetNetworkResponse, error) {
-		info, err := coreClient.Info(ctx)
+		protocolVersion, err := getProtocolVersion(ctx, ledgerEntryReader, ledgerReader)
 		if err != nil {
-			return GetNetworkResponse{}, (&jrpc2.Error{
+			return GetNetworkResponse{}, &jrpc2.Error{
 				Code:    jrpc2.InternalError,
 				Message: err.Error(),
-			})
+			}
 		}
+
 		return GetNetworkResponse{
 			FriendbotURL:    friendbotURL,
 			Passphrase:      networkPassphrase,
-			ProtocolVersion: info.Info.ProtocolVersion,
+			ProtocolVersion: int(protocolVersion),
 		}, nil
 	})
 }