From b3f414c2e14d327fae7fc696526e591505a8c929 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Mon, 10 Jun 2024 19:37:20 +0200
Subject: [PATCH 01/36] First stab

---
 cmd/soroban-rpc/internal/daemon/daemon.go  | 37 ++++++++++++++++++++++
 cmd/soroban-rpc/internal/db/db.go          | 36 +++++++++++++++++----
 cmd/soroban-rpc/internal/db/transaction.go | 11 ++++++-
 3 files changed, 77 insertions(+), 7 deletions(-)

diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index c38ccde9..84b07eae 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -40,6 +40,7 @@ const (
 	defaultReadTimeout                    = 5 * time.Second
 	defaultShutdownGracePeriod            = 10 * time.Second
 	inMemoryInitializationLedgerLogPeriod = 1_000_000
+	transactionsTableMigrationDoneMetaKey = "TransactionsTableMigrationDone"
 )
 
 type Daemon struct {
@@ -202,6 +203,36 @@ func MustNew(cfg *config.Config) *Daemon {
 	//       but it's probably not worth the pain.
 	var initialSeq uint32
 	var currentSeq uint32
+	// We should do the migration somewhere else
+	migrationSession := dbConn.Clone()
+	if err = migrationSession.Begin(readTxMetaCtx); err != nil {
+		logger.WithError(err).Fatal("could not start migration session")
+	}
+	migrationFunc := func(txmeta xdr.LedgerCloseMeta) error { return nil }
+	migrationDoneFunc := func() {}
+	val, err := db.GetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
+	if err == db.ErrEmptyDB || val == false {
+		logger.Info("migrating transaction to new backend")
+		writer := db.NewTransactionWriter(logger, migrationSession, cfg.NetworkPassphrase)
+		migrationFunc = func(txmeta xdr.LedgerCloseMeta) error {
+			return writer.InsertTransactions(txmeta)
+		}
+		migrationDoneFunc = func() {
+			err := db.SetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
+			if err != nil {
+				logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not set metadata")
+				migrationSession.Rollback()
+				return
+			}
+			// TODO: rollback wherever necessary
+			if err = migrationSession.Commit(); err != nil {
+				logger.WithError(err).Error("could not commit migration session")
+			}
+		}
+	} else if err != nil {
+		logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not get metadata")
+	}
+
 	err = db.NewLedgerReader(dbConn).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
 		currentSeq = txmeta.LedgerSequence()
 		if initialSeq == 0 {
@@ -220,11 +251,17 @@ func MustNew(cfg *config.Config) *Daemon {
 		if err := feewindows.IngestFees(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize fee stats")
 		}
+		if err := migrationFunc(txmeta); err != nil {
+			// TODO: we should only migrate the transaction range
+			logger.WithError(err).Fatal("could not run migration")
+		}
 		return nil
 	})
 	if err != nil {
 		logger.WithError(err).Fatal("could not obtain txmeta cache from the database")
 	}
+	migrationDoneFunc()
+
 	if currentSeq != 0 {
 		logger.WithFields(supportlog.F{
 			"seq": currentSeq,
diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go
index 227e2115..73d4713a 100644
--- a/cmd/soroban-rpc/internal/db/db.go
+++ b/cmd/soroban-rpc/internal/db/db.go
@@ -17,6 +17,7 @@ import (
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/support/log"
 	"github.com/stellar/go/xdr"
+
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces"
 )
 
@@ -100,21 +101,43 @@ func OpenSQLiteDB(dbFilePath string) (*DB, error) {
 	return &result, nil
 }
 
-func getLatestLedgerSequence(ctx context.Context, q db.SessionInterface, cache *dbCache) (uint32, error) {
-	sql := sq.Select("value").From(metaTableName).Where(sq.Eq{"key": latestLedgerSequenceMetaKey})
+func GetMetaBool(ctx context.Context, q db.SessionInterface, key string) (bool, error) {
+	valueStr, err := getMetaValue(ctx, q, key)
+	if err != nil {
+		return false, err
+	}
+	return strconv.ParseBool(valueStr)
+}
+
+func SetMetaBool(ctx context.Context, q db.SessionInterface, key string) error {
+	_, err := sq.Replace(metaTableName).
+		Values(latestLedgerSequenceMetaKey, "true").
+		Exec()
+	return err
+}
+
+func getMetaValue(ctx context.Context, q db.SessionInterface, key string) (string, error) {
+	sql := sq.Select("value").From(metaTableName).Where(sq.Eq{"key": key})
 	var results []string
 	if err := q.Select(ctx, &results, sql); err != nil {
-		return 0, err
+		return "", err
 	}
 	switch len(results) {
 	case 0:
-		return 0, ErrEmptyDB
+		return "", ErrEmptyDB
 	case 1:
 		// expected length on an initialized DB
 	default:
-		return 0, fmt.Errorf("multiple entries (%d) for key %q in table %q", len(results), latestLedgerSequenceMetaKey, metaTableName)
+		return "", fmt.Errorf("multiple entries (%d) for key %q in table %q", len(results), latestLedgerSequenceMetaKey, metaTableName)
+	}
+	return results[0], nil
+}
+
+func getLatestLedgerSequence(ctx context.Context, q db.SessionInterface, cache *dbCache) (uint32, error) {
+	latestLedgerStr, err := getMetaValue(ctx, q, latestLedgerSequenceMetaKey)
+	if err != nil {
+		return 0, err
 	}
-	latestLedgerStr := results[0]
 	latestLedger, err := strconv.ParseUint(latestLedgerStr, 10, 32)
 	if err != nil {
 		return 0, err
@@ -206,6 +229,7 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
 	writer := writeTx{
 		globalCache: &db.cache,
 		postCommit: func() error {
+			// TODO: this is sqlite-only, it shouldn't be here
 			_, err := db.ExecRaw(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
 			return err
 		},
diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index c703c39e..817719ed 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -39,12 +39,21 @@ type TransactionWriter interface {
 	RegisterMetrics(ingest, count prometheus.Observer)
 }
 
-// TransactionReader provides all of the public ways to read from the DB.
+// TransactionReader provides all the public ways to read from the DB.
 type TransactionReader interface {
 	GetTransaction(ctx context.Context, hash xdr.Hash) (Transaction, ledgerbucketwindow.LedgerRange, error)
 	GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error)
 }
 
+func NewTransactionWriter(log *log.Entry, db db.SessionInterface, networkPassphrase string) TransactionWriter {
+	return &transactionHandler{
+		log:        log,
+		db:         db,
+		stmtCache:  sq.NewStmtCache(db.GetTx()),
+		passphrase: networkPassphrase,
+	}
+}
+
 type transactionHandler struct {
 	log        *log.Entry
 	db         db.SessionInterface

From daf4ca8d15aa2d0caee7cbb928ec6d366478ac8a Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Mon, 10 Jun 2024 21:35:08 +0200
Subject: [PATCH 02/36] Fix query

---
 cmd/soroban-rpc/internal/db/db.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go
index 73d4713a..126b0e20 100644
--- a/cmd/soroban-rpc/internal/db/db.go
+++ b/cmd/soroban-rpc/internal/db/db.go
@@ -110,9 +110,9 @@ func GetMetaBool(ctx context.Context, q db.SessionInterface, key string) (bool,
 }
 
 func SetMetaBool(ctx context.Context, q db.SessionInterface, key string) error {
-	_, err := sq.Replace(metaTableName).
-		Values(latestLedgerSequenceMetaKey, "true").
-		Exec()
+	query := sq.Replace(metaTableName).
+		Values(key, "true")
+	_, err := q.Exec(ctx, query)
 	return err
 }
 

From ce0fe5e726e6935e763ab74f55d0a7c839e52ae2 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 03:01:17 +0200
Subject: [PATCH 03/36] Cleanup storage initialization

---
 cmd/soroban-rpc/internal/daemon/daemon.go | 175 ++++++++++++----------
 1 file changed, 96 insertions(+), 79 deletions(-)

diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index 84b07eae..450d05b9 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -188,85 +188,7 @@ func MustNew(cfg *config.Config) *Daemon {
 		}, metricsRegistry),
 	}
 
-	eventStore := events.NewMemoryStore(
-		daemon,
-		cfg.NetworkPassphrase,
-		cfg.EventLedgerRetentionWindow,
-	)
-	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
-
-	// initialize the stores using what was on the DB
-	readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
-	defer cancelReadTxMeta()
-	// NOTE: We could optimize this to avoid unnecessary ingestion calls
-	//       (the range of txmetas can be larger than the individual store retention windows)
-	//       but it's probably not worth the pain.
-	var initialSeq uint32
-	var currentSeq uint32
-	// We should do the migration somewhere else
-	migrationSession := dbConn.Clone()
-	if err = migrationSession.Begin(readTxMetaCtx); err != nil {
-		logger.WithError(err).Fatal("could not start migration session")
-	}
-	migrationFunc := func(txmeta xdr.LedgerCloseMeta) error { return nil }
-	migrationDoneFunc := func() {}
-	val, err := db.GetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
-	if err == db.ErrEmptyDB || val == false {
-		logger.Info("migrating transaction to new backend")
-		writer := db.NewTransactionWriter(logger, migrationSession, cfg.NetworkPassphrase)
-		migrationFunc = func(txmeta xdr.LedgerCloseMeta) error {
-			return writer.InsertTransactions(txmeta)
-		}
-		migrationDoneFunc = func() {
-			err := db.SetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
-			if err != nil {
-				logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not set metadata")
-				migrationSession.Rollback()
-				return
-			}
-			// TODO: rollback wherever necessary
-			if err = migrationSession.Commit(); err != nil {
-				logger.WithError(err).Error("could not commit migration session")
-			}
-		}
-	} else if err != nil {
-		logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not get metadata")
-	}
-
-	err = db.NewLedgerReader(dbConn).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
-		currentSeq = txmeta.LedgerSequence()
-		if initialSeq == 0 {
-			initialSeq = currentSeq
-			logger.WithFields(supportlog.F{
-				"seq": currentSeq,
-			}).Info("initializing in-memory store")
-		} else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 {
-			logger.WithFields(supportlog.F{
-				"seq": currentSeq,
-			}).Debug("still initializing in-memory store")
-		}
-		if err := eventStore.IngestEvents(txmeta); err != nil {
-			logger.WithError(err).Fatal("could not initialize event memory store")
-		}
-		if err := feewindows.IngestFees(txmeta); err != nil {
-			logger.WithError(err).Fatal("could not initialize fee stats")
-		}
-		if err := migrationFunc(txmeta); err != nil {
-			// TODO: we should only migrate the transaction range
-			logger.WithError(err).Fatal("could not run migration")
-		}
-		return nil
-	})
-	if err != nil {
-		logger.WithError(err).Fatal("could not obtain txmeta cache from the database")
-	}
-	migrationDoneFunc()
-
-	if currentSeq != 0 {
-		logger.WithFields(supportlog.F{
-			"seq": currentSeq,
-		}).Info("finished initializing in-memory store")
-	}
+	feewindows, eventStore := daemon.mustInitializeStorage(cfg)
 
 	onIngestionRetry := func(err error, dur time.Duration) {
 		logger.WithError(err).Error("could not run ingestion. Retrying")
@@ -354,6 +276,101 @@ func MustNew(cfg *config.Config) *Daemon {
 	return daemon
 }
 
+// mustInitializeStorage initializes the storage using what was on the DB
+// TODO: This function is horrendous, cleanup once we remove the in-memory storage
+func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindows, *events.MemoryStore) {
+	eventStore := events.NewMemoryStore(
+		d,
+		cfg.NetworkPassphrase,
+		cfg.EventLedgerRetentionWindow,
+	)
+	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
+
+	readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
+	defer cancelReadTxMeta()
+	var initialSeq uint32
+	var currentSeq uint32
+	// We should do the migration somewhere else
+	migrationSession := d.db.Clone()
+	if err := migrationSession.Begin(readTxMetaCtx); err != nil {
+		d.logger.WithError(err).Fatal("could not start migration session")
+	}
+	migrationFunc := func(txmeta xdr.LedgerCloseMeta) error { return nil }
+	migrationDoneFunc := func() {}
+	val, err := db.GetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
+	if err == db.ErrEmptyDB || val == false {
+		d.logger.Info("migrating transaction to new backend")
+		writer := db.NewTransactionWriter(d.logger, migrationSession, cfg.NetworkPassphrase)
+		latestLedger, err := db.NewLedgerEntryReader(d.db).GetLatestLedgerSequence(readTxMetaCtx)
+		if err != nil || err != db.ErrEmptyDB {
+			d.logger.WithError(err).Fatal("cannot read latest ledger")
+		}
+		firstLedgerToMigrate := uint32(2)
+		if latestLedger > cfg.TransactionLedgerRetentionWindow {
+			firstLedgerToMigrate = latestLedger - cfg.TransactionLedgerRetentionWindow
+		}
+		migrationFunc = func(txmeta xdr.LedgerCloseMeta) error {
+			if txmeta.LedgerSequence() < firstLedgerToMigrate {
+				return nil
+			}
+			return writer.InsertTransactions(txmeta)
+		}
+		migrationDoneFunc = func() {
+			err := db.SetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
+			if err != nil {
+				d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not set metadata")
+				migrationSession.Rollback()
+				return
+			}
+			// TODO: rollback wherever necessary
+			if err = migrationSession.Commit(); err != nil {
+				d.logger.WithError(err).Error("could not commit migration session")
+			}
+		}
+	} else if err != nil {
+		d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not get metadata")
+	}
+	// NOTE: We could optimize this to avoid unnecessary ingestion calls
+	//       (the range of txmetas can be larger than the individual store retention windows)
+	//       but it's probably not worth the pain.
+	err = db.NewLedgerReader(d.db).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
+		currentSeq = txmeta.LedgerSequence()
+		if initialSeq == 0 {
+			initialSeq = currentSeq
+			d.logger.WithFields(supportlog.F{
+				"seq": currentSeq,
+			}).Info("initializing in-memory store")
+		} else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 {
+			d.logger.WithFields(supportlog.F{
+				"seq": currentSeq,
+			}).Debug("still initializing in-memory store")
+		}
+		if err := eventStore.IngestEvents(txmeta); err != nil {
+			d.logger.WithError(err).Fatal("could not initialize event memory store")
+		}
+		if err := feewindows.IngestFees(txmeta); err != nil {
+			d.logger.WithError(err).Fatal("could not initialize fee stats")
+		}
+		if err := migrationFunc(txmeta); err != nil {
+			// TODO: we should only migrate the transaction range
+			d.logger.WithError(err).Fatal("could not run migration")
+		}
+		return nil
+	})
+	if err != nil {
+		d.logger.WithError(err).Fatal("could not obtain txmeta cache from the database")
+	}
+	migrationDoneFunc()
+
+	if currentSeq != 0 {
+		d.logger.WithFields(supportlog.F{
+			"seq": currentSeq,
+		}).Info("finished initializing in-memory store")
+	}
+	
+	return feewindows, eventStore
+}
+
 func (d *Daemon) Run() {
 	d.logger.WithFields(supportlog.F{
 		"addr": d.server.Addr,

From 61dff8bedc1e51aa8f5b728e730e29096749f692 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 03:20:12 +0200
Subject: [PATCH 04/36] Clean up the migration even more

---
 cmd/soroban-rpc/internal/daemon/daemon.go | 102 ++++++++++++----------
 1 file changed, 57 insertions(+), 45 deletions(-)

diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index 450d05b9..d7bb0f54 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -277,7 +277,7 @@ func MustNew(cfg *config.Config) *Daemon {
 }
 
 // mustInitializeStorage initializes the storage using what was on the DB
-// TODO: This function is horrendous, cleanup once we remove the in-memory storage
+// TODO: clean up once we remove the in-memory storage
 func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindows, *events.MemoryStore) {
 	eventStore := events.NewMemoryStore(
 		d,
@@ -290,50 +290,11 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 	defer cancelReadTxMeta()
 	var initialSeq uint32
 	var currentSeq uint32
-	// We should do the migration somewhere else
-	migrationSession := d.db.Clone()
-	if err := migrationSession.Begin(readTxMetaCtx); err != nil {
-		d.logger.WithError(err).Fatal("could not start migration session")
-	}
-	migrationFunc := func(txmeta xdr.LedgerCloseMeta) error { return nil }
-	migrationDoneFunc := func() {}
-	val, err := db.GetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
-	if err == db.ErrEmptyDB || val == false {
-		d.logger.Info("migrating transaction to new backend")
-		writer := db.NewTransactionWriter(d.logger, migrationSession, cfg.NetworkPassphrase)
-		latestLedger, err := db.NewLedgerEntryReader(d.db).GetLatestLedgerSequence(readTxMetaCtx)
-		if err != nil || err != db.ErrEmptyDB {
-			d.logger.WithError(err).Fatal("cannot read latest ledger")
-		}
-		firstLedgerToMigrate := uint32(2)
-		if latestLedger > cfg.TransactionLedgerRetentionWindow {
-			firstLedgerToMigrate = latestLedger - cfg.TransactionLedgerRetentionWindow
-		}
-		migrationFunc = func(txmeta xdr.LedgerCloseMeta) error {
-			if txmeta.LedgerSequence() < firstLedgerToMigrate {
-				return nil
-			}
-			return writer.InsertTransactions(txmeta)
-		}
-		migrationDoneFunc = func() {
-			err := db.SetMetaBool(readTxMetaCtx, migrationSession, transactionsTableMigrationDoneMetaKey)
-			if err != nil {
-				d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not set metadata")
-				migrationSession.Rollback()
-				return
-			}
-			// TODO: rollback wherever necessary
-			if err = migrationSession.Commit(); err != nil {
-				d.logger.WithError(err).Error("could not commit migration session")
-			}
-		}
-	} else if err != nil {
-		d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not get metadata")
-	}
+	migration, migrationDone := d.newTxMigration(readTxMetaCtx, cfg)
 	// NOTE: We could optimize this to avoid unnecessary ingestion calls
 	//       (the range of txmetas can be larger than the individual store retention windows)
 	//       but it's probably not worth the pain.
-	err = db.NewLedgerReader(d.db).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
+	err := db.NewLedgerReader(d.db).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
 		currentSeq = txmeta.LedgerSequence()
 		if initialSeq == 0 {
 			initialSeq = currentSeq
@@ -351,7 +312,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 		if err := feewindows.IngestFees(txmeta); err != nil {
 			d.logger.WithError(err).Fatal("could not initialize fee stats")
 		}
-		if err := migrationFunc(txmeta); err != nil {
+		if err := migration(txmeta); err != nil {
 			// TODO: we should only migrate the transaction range
 			d.logger.WithError(err).Fatal("could not run migration")
 		}
@@ -360,17 +321,68 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 	if err != nil {
 		d.logger.WithError(err).Fatal("could not obtain txmeta cache from the database")
 	}
-	migrationDoneFunc()
+	migrationDone()
 
 	if currentSeq != 0 {
 		d.logger.WithFields(supportlog.F{
 			"seq": currentSeq,
 		}).Info("finished initializing in-memory store")
 	}
-	
+
 	return feewindows, eventStore
 }
 
+// TODO: We should probably implement the migrations somewhere else
+type migrationFunc func(txmeta xdr.LedgerCloseMeta) error
+type migrationDoneFunc func()
+
+func (d *Daemon) newTxMigration(ctx context.Context, cfg *config.Config) (migrationFunc, migrationDoneFunc) {
+	migrationSession := d.db.Clone()
+	if err := migrationSession.Begin(ctx); err != nil {
+		d.logger.WithError(err).Fatal("could not start migration session")
+	}
+	migration := func(txmeta xdr.LedgerCloseMeta) error { return nil }
+	migrationDone := func() {}
+	previouslyMigrated, err := db.GetMetaBool(ctx, migrationSession, transactionsTableMigrationDoneMetaKey)
+	if err != nil {
+		if !errors.Is(err, db.ErrEmptyDB) {
+			d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not get metadata")
+		}
+	} else if previouslyMigrated {
+		migrationSession.Rollback()
+		return migration, migrationDone
+	}
+
+	d.logger.Info("migrating transactions to new backend")
+	writer := db.NewTransactionWriter(d.logger, migrationSession, cfg.NetworkPassphrase)
+	latestLedger, err := db.NewLedgerEntryReader(d.db).GetLatestLedgerSequence(ctx)
+	if err != nil || err != db.ErrEmptyDB {
+		d.logger.WithError(err).Fatal("cannot read latest ledger")
+	}
+	firstLedgerToMigrate := uint32(2)
+	if latestLedger > cfg.TransactionLedgerRetentionWindow {
+		firstLedgerToMigrate = latestLedger - cfg.TransactionLedgerRetentionWindow
+	}
+	migration = func(txmeta xdr.LedgerCloseMeta) error {
+		if txmeta.LedgerSequence() < firstLedgerToMigrate {
+			return nil
+		}
+		return writer.InsertTransactions(txmeta)
+	}
+	migrationDone = func() {
+		err := db.SetMetaBool(ctx, migrationSession, transactionsTableMigrationDoneMetaKey)
+		if err != nil {
+			d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not set metadata")
+			migrationSession.Rollback()
+			return
+		}
+		if err = migrationSession.Commit(); err != nil {
+			d.logger.WithError(err).Error("could not commit migration session")
+		}
+	}
+	return migration, migrationDone
+}
+
 func (d *Daemon) Run() {
 	d.logger.WithFields(supportlog.F{
 		"addr": d.server.Addr,

From 8f3abcaa725523a604ddbd96d560b0535eb0b89d Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 05:14:52 +0200
Subject: [PATCH 05/36] Factor out transaction table migration cleanly

---
 .github/workflows/golangci-lint.yml           |   2 +-
 cmd/soroban-rpc/internal/daemon/daemon.go     |  69 ++-------
 cmd/soroban-rpc/internal/db/db.go             |  41 +++---
 cmd/soroban-rpc/internal/db/ledger_test.go    |   5 +-
 cmd/soroban-rpc/internal/db/ledgerentry.go    |  10 +-
 cmd/soroban-rpc/internal/db/migration.go      | 131 ++++++++++++++++++
 .../{migrations => sqlmigrations}/01_init.sql |   0
 .../02_transactions.sql                       |   0
 cmd/soroban-rpc/internal/db/transaction.go    |  45 ++++--
 .../internal/db/transaction_test.go           |   7 +-
 10 files changed, 215 insertions(+), 95 deletions(-)
 create mode 100644 cmd/soroban-rpc/internal/db/migration.go
 rename cmd/soroban-rpc/internal/db/{migrations => sqlmigrations}/01_init.sql (100%)
 rename cmd/soroban-rpc/internal/db/{migrations => sqlmigrations}/02_transactions.sql (100%)

diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
index 59f7d4d1..2a3dbae1 100644
--- a/.github/workflows/golangci-lint.yml
+++ b/.github/workflows/golangci-lint.yml
@@ -29,7 +29,7 @@ jobs:
           make build-libpreflight
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@537aa1903e5d359d0b27dbc19ddd22c5087f3fbc # version v3.2.0
+        uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # version v6.0.1
         with:
           version: v1.52.2 # this is the golangci-lint version
           args: --issues-exit-code=0 # exit without errors for now - won't fail the build
diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index d7bb0f54..0afe7596 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -241,7 +241,7 @@ func MustNew(cfg *config.Config) *Daemon {
 		Logger:            logger,
 		LedgerReader:      db.NewLedgerReader(dbConn),
 		LedgerEntryReader: db.NewLedgerEntryReader(dbConn),
-		TransactionReader: db.NewTransactionReader(logger, dbConn, cfg.NetworkPassphrase),
+		TransactionReader: db.NewTransactionReader(logger, dbConn.SessionInterface, cfg.NetworkPassphrase),
 		PreflightGetter:   preflightWorkerPool,
 	})
 
@@ -290,11 +290,14 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 	defer cancelReadTxMeta()
 	var initialSeq uint32
 	var currentSeq uint32
-	migration, migrationDone := d.newTxMigration(readTxMetaCtx, cfg)
+	dataMigrations, err := db.BuildMigrations(readTxMetaCtx, d.logger, d.db, cfg)
+	if err != nil {
+		d.logger.WithError(err).Fatal("could not build migrations")
+	}
 	// NOTE: We could optimize this to avoid unnecessary ingestion calls
 	//       (the range of txmetas can be larger than the individual store retention windows)
 	//       but it's probably not worth the pain.
-	err := db.NewLedgerReader(d.db).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
+	err = db.NewLedgerReader(d.db).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
 		currentSeq = txmeta.LedgerSequence()
 		if initialSeq == 0 {
 			initialSeq = currentSeq
@@ -312,16 +315,17 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 		if err := feewindows.IngestFees(txmeta); err != nil {
 			d.logger.WithError(err).Fatal("could not initialize fee stats")
 		}
-		if err := migration(txmeta); err != nil {
-			// TODO: we should only migrate the transaction range
-			d.logger.WithError(err).Fatal("could not run migration")
+		if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil {
+			d.logger.WithError(err).Fatal("could not run migrations")
 		}
 		return nil
 	})
 	if err != nil {
 		d.logger.WithError(err).Fatal("could not obtain txmeta cache from the database")
 	}
-	migrationDone()
+	if err := dataMigrations.Commit(readTxMetaCtx); err != nil {
+		d.logger.WithError(err).Fatal("could not commit data migrations")
+	}
 
 	if currentSeq != 0 {
 		d.logger.WithFields(supportlog.F{
@@ -332,57 +336,6 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 	return feewindows, eventStore
 }
 
-// TODO: We should probably implement the migrations somewhere else
-type migrationFunc func(txmeta xdr.LedgerCloseMeta) error
-type migrationDoneFunc func()
-
-func (d *Daemon) newTxMigration(ctx context.Context, cfg *config.Config) (migrationFunc, migrationDoneFunc) {
-	migrationSession := d.db.Clone()
-	if err := migrationSession.Begin(ctx); err != nil {
-		d.logger.WithError(err).Fatal("could not start migration session")
-	}
-	migration := func(txmeta xdr.LedgerCloseMeta) error { return nil }
-	migrationDone := func() {}
-	previouslyMigrated, err := db.GetMetaBool(ctx, migrationSession, transactionsTableMigrationDoneMetaKey)
-	if err != nil {
-		if !errors.Is(err, db.ErrEmptyDB) {
-			d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not get metadata")
-		}
-	} else if previouslyMigrated {
-		migrationSession.Rollback()
-		return migration, migrationDone
-	}
-
-	d.logger.Info("migrating transactions to new backend")
-	writer := db.NewTransactionWriter(d.logger, migrationSession, cfg.NetworkPassphrase)
-	latestLedger, err := db.NewLedgerEntryReader(d.db).GetLatestLedgerSequence(ctx)
-	if err != nil || err != db.ErrEmptyDB {
-		d.logger.WithError(err).Fatal("cannot read latest ledger")
-	}
-	firstLedgerToMigrate := uint32(2)
-	if latestLedger > cfg.TransactionLedgerRetentionWindow {
-		firstLedgerToMigrate = latestLedger - cfg.TransactionLedgerRetentionWindow
-	}
-	migration = func(txmeta xdr.LedgerCloseMeta) error {
-		if txmeta.LedgerSequence() < firstLedgerToMigrate {
-			return nil
-		}
-		return writer.InsertTransactions(txmeta)
-	}
-	migrationDone = func() {
-		err := db.SetMetaBool(ctx, migrationSession, transactionsTableMigrationDoneMetaKey)
-		if err != nil {
-			d.logger.WithError(err).WithField("key", transactionsTableMigrationDoneMetaKey).Fatal("could not set metadata")
-			migrationSession.Rollback()
-			return
-		}
-		if err = migrationSession.Commit(); err != nil {
-			d.logger.WithError(err).Error("could not commit migration session")
-		}
-	}
-	return migration, migrationDone
-}
-
 func (d *Daemon) Run() {
 	d.logger.WithFields(supportlog.F{
 		"addr": d.server.Addr,
diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go
index 126b0e20..45bbad0b 100644
--- a/cmd/soroban-rpc/internal/db/db.go
+++ b/cmd/soroban-rpc/internal/db/db.go
@@ -21,8 +21,8 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces"
 )
 
-//go:embed migrations/*.sql
-var migrations embed.FS
+//go:embed sqlmigrations/*.sql
+var sqlMigrations embed.FS
 
 var ErrEmptyDB = errors.New("DB is empty")
 
@@ -53,7 +53,14 @@ type dbCache struct {
 
 type DB struct {
 	db.SessionInterface
-	cache dbCache
+	cache *dbCache
+}
+
+func (db *DB) Clone() *DB {
+	return &DB{
+		SessionInterface: db.SessionInterface.Clone(),
+		cache:            db.cache,
+	}
 }
 
 func openSQLiteDB(dbFilePath string) (*db.Session, error) {
@@ -66,9 +73,9 @@ func openSQLiteDB(dbFilePath string) (*db.Session, error) {
 		return nil, errors.Wrap(err, "open failed")
 	}
 
-	if err = runMigrations(session.DB.DB, "sqlite3"); err != nil {
+	if err = runSQLMigrations(session.DB.DB, "sqlite3"); err != nil {
 		_ = session.Close()
-		return nil, errors.Wrap(err, "could not run migrations")
+		return nil, errors.Wrap(err, "could not run SQL migrations")
 	}
 	return session, nil
 }
@@ -80,7 +87,7 @@ func OpenSQLiteDBWithPrometheusMetrics(dbFilePath string, namespace string, sub
 	}
 	result := DB{
 		SessionInterface: db.RegisterMetrics(session, namespace, sub, registry),
-		cache: dbCache{
+		cache: &dbCache{
 			ledgerEntries: newTransactionalCache(),
 		},
 	}
@@ -94,14 +101,14 @@ func OpenSQLiteDB(dbFilePath string) (*DB, error) {
 	}
 	result := DB{
 		SessionInterface: session,
-		cache: dbCache{
+		cache: &dbCache{
 			ledgerEntries: newTransactionalCache(),
 		},
 	}
 	return &result, nil
 }
 
-func GetMetaBool(ctx context.Context, q db.SessionInterface, key string) (bool, error) {
+func getMetaBool(ctx context.Context, q db.SessionInterface, key string) (bool, error) {
 	valueStr, err := getMetaValue(ctx, q, key)
 	if err != nil {
 		return false, err
@@ -109,7 +116,7 @@ func GetMetaBool(ctx context.Context, q db.SessionInterface, key string) (bool,
 	return strconv.ParseBool(valueStr)
 }
 
-func SetMetaBool(ctx context.Context, q db.SessionInterface, key string) error {
+func setMetaBool(ctx context.Context, q db.SessionInterface, key string) error {
 	query := sq.Replace(metaTableName).
 		Values(key, "true")
 	_, err := q.Exec(ctx, query)
@@ -148,7 +155,7 @@ func getLatestLedgerSequence(ctx context.Context, q db.SessionInterface, cache *
 	// Otherwise, the write-through cache won't get updated until the first ingestion commit
 	cache.Lock()
 	if cache.latestLedgerSeq == 0 {
-		// Only update the cache if value is missing (0), otherwise
+		// Only update the cache if the value is missing (0), otherwise
 		// we may end up overwriting the entry with an older version
 		cache.latestLedgerSeq = result
 	}
@@ -215,11 +222,11 @@ func NewReadWriter(
 }
 
 func (rw *readWriter) GetLatestLedgerSequence(ctx context.Context) (uint32, error) {
-	return getLatestLedgerSequence(ctx, rw.db, &rw.db.cache)
+	return getLatestLedgerSequence(ctx, rw.db.SessionInterface, rw.db.cache)
 }
 
 func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
-	txSession := rw.db.Clone()
+	txSession := rw.db.SessionInterface.Clone()
 	if err := txSession.Begin(ctx); err != nil {
 		return nil, err
 	}
@@ -227,7 +234,7 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
 
 	db := rw.db
 	writer := writeTx{
-		globalCache: &db.cache,
+		globalCache: db.cache,
 		postCommit: func() error {
 			// TODO: this is sqlite-only, it shouldn't be here
 			_, err := db.ExecRaw(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
@@ -332,12 +339,12 @@ func (w writeTx) Rollback() error {
 	}
 }
 
-func runMigrations(db *sql.DB, dialect string) error {
+func runSQLMigrations(db *sql.DB, dialect string) error {
 	m := &migrate.AssetMigrationSource{
-		Asset: migrations.ReadFile,
+		Asset: sqlMigrations.ReadFile,
 		AssetDir: func() func(string) ([]string, error) {
 			return func(path string) ([]string, error) {
-				dirEntry, err := migrations.ReadDir(path)
+				dirEntry, err := sqlMigrations.ReadDir(path)
 				if err != nil {
 					return nil, err
 				}
@@ -349,7 +356,7 @@ func runMigrations(db *sql.DB, dialect string) error {
 				return entries, nil
 			}
 		}(),
-		Dir: "migrations",
+		Dir: "sqlmigrations",
 	}
 	_, err := migrate.ExecMax(db, dialect, m, migrate.Up, 0)
 	return err
diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go
index f6ebd70b..9026b784 100644
--- a/cmd/soroban-rpc/internal/db/ledger_test.go
+++ b/cmd/soroban-rpc/internal/db/ledger_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/log"
 	"github.com/stellar/go/xdr"
+
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces"
 )
 
@@ -113,8 +114,8 @@ func NewTestDB(tb testing.TB) *DB {
 		assert.NoError(tb, db.Close())
 	})
 	return &DB{
-		SessionInterface: db,
-		cache: dbCache{
+		SessionInterface: db.SessionInterface,
+		cache: &dbCache{
 			ledgerEntries: newTransactionalCache(),
 		},
 	}
diff --git a/cmd/soroban-rpc/internal/db/ledgerentry.go b/cmd/soroban-rpc/internal/db/ledgerentry.go
index 8286e955..d9d60d37 100644
--- a/cmd/soroban-rpc/internal/db/ledgerentry.go
+++ b/cmd/soroban-rpc/internal/db/ledgerentry.go
@@ -341,12 +341,12 @@ func NewLedgerEntryReader(db *DB) LedgerEntryReader {
 }
 
 func (r ledgerEntryReader) GetLatestLedgerSequence(ctx context.Context) (uint32, error) {
-	return getLatestLedgerSequence(ctx, r.db, &r.db.cache)
+	return getLatestLedgerSequence(ctx, r.db.SessionInterface, r.db.cache)
 }
 
 // NewCachedTx() caches all accessed ledger entries and select statements. If many ledger entries are accessed, it will grow without bounds.
 func (r ledgerEntryReader) NewCachedTx(ctx context.Context) (LedgerEntryReadTx, error) {
-	txSession := r.db.Clone()
+	txSession := r.db.SessionInterface.Clone()
 	// We need to copy the cached ledger entries locally when we start the transaction
 	// since otherwise we would break the consistency between the transaction and the cache.
 
@@ -360,7 +360,7 @@ func (r ledgerEntryReader) NewCachedTx(ctx context.Context) (LedgerEntryReadTx,
 	}
 	cacheReadTx := r.db.cache.ledgerEntries.newReadTx()
 	return &ledgerEntryReadTx{
-		globalCache:            &r.db.cache,
+		globalCache:            r.db.cache,
 		stmtCache:              sq.NewStmtCache(txSession.GetTx()),
 		latestLedgerSeqCache:   r.db.cache.latestLedgerSeq,
 		ledgerEntryCacheReadTx: &cacheReadTx,
@@ -370,14 +370,14 @@ func (r ledgerEntryReader) NewCachedTx(ctx context.Context) (LedgerEntryReadTx,
 }
 
 func (r ledgerEntryReader) NewTx(ctx context.Context) (LedgerEntryReadTx, error) {
-	txSession := r.db.Clone()
+	txSession := r.db.SessionInterface.Clone()
 	if err := txSession.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}); err != nil {
 		return nil, err
 	}
 	r.db.cache.RLock()
 	defer r.db.cache.RUnlock()
 	return &ledgerEntryReadTx{
-		globalCache:          &r.db.cache,
+		globalCache:          r.db.cache,
 		latestLedgerSeqCache: r.db.cache.latestLedgerSeq,
 		tx:                   txSession,
 		buffer:               xdr.NewEncodingBuffer(),
diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go
new file mode 100644
index 00000000..419f34df
--- /dev/null
+++ b/cmd/soroban-rpc/internal/db/migration.go
@@ -0,0 +1,131 @@
+package db
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/stellar/go/support/log"
+	"github.com/stellar/go/xdr"
+
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
+)
+
+type MigrationApplier interface {
+	Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error
+}
+
+type migrationApplierFactory interface {
+	New(db *DB) (MigrationApplier, error)
+}
+
+type migrationApplierFactoryF func(db *DB) (MigrationApplier, error)
+
+func (m migrationApplierFactoryF) New(db *DB) (MigrationApplier, error) {
+	return m(db)
+}
+
+type Migration interface {
+	MigrationApplier
+	Commit(ctx context.Context) error
+	Rollback(ctx context.Context) error
+}
+
+type multiMigration []Migration
+
+func (m multiMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
+	var err error
+	for _, data := range m {
+		if localErr := data.Apply(ctx, meta); localErr != nil {
+			err = errors.Join(err, localErr)
+		}
+	}
+	return err
+}
+
+func (m multiMigration) Commit(ctx context.Context) error {
+	var err error
+	for _, data := range m {
+		if localErr := data.Commit(ctx); localErr != nil {
+			err = errors.Join(err, localErr)
+		}
+	}
+	return err
+}
+
+func (m multiMigration) Rollback(ctx context.Context) error {
+	var err error
+	for _, data := range m {
+		if localErr := data.Rollback(ctx); localErr != nil {
+			err = errors.Join(err, localErr)
+		}
+	}
+	return err
+}
+
+// guardedMigration is a db data migration whose application is guarded by a boolean in the meta table
+// (after the migration is applied the boolean is set to true, so that the migration is not applied again)
+type guardedMigration struct {
+	guardMetaKey    string
+	db              *DB
+	migration       MigrationApplier
+	alreadyMigrated bool
+}
+
+func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, factory migrationApplierFactory, db *DB) (Migration, error) {
+	migrationDB := db.Clone()
+	if err := migrationDB.Begin(ctx); err != nil {
+		return nil, err
+	}
+	metaKey := "Migration" + uniqueMigrationName + "Done"
+	previouslyMigrated, err := getMetaBool(ctx, migrationDB.SessionInterface, metaKey)
+	if err != nil && !errors.Is(err, ErrEmptyDB) {
+		migrationDB.Rollback()
+		return nil, err
+	}
+	applier, err := factory.New(migrationDB)
+	if err != nil {
+		migrationDB.Rollback()
+		return nil, err
+	}
+	guardedMigration := &guardedMigration{
+		guardMetaKey:    metaKey,
+		db:              migrationDB,
+		migration:       applier,
+		alreadyMigrated: previouslyMigrated,
+	}
+	return guardedMigration, nil
+}
+
+func (g *guardedMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
+	if g.alreadyMigrated {
+		return nil
+	}
+	return g.migration.Apply(ctx, meta)
+}
+
+func (g *guardedMigration) Commit(ctx context.Context) error {
+	if g.alreadyMigrated {
+		return nil
+	}
+	err := setMetaBool(ctx, g.db.SessionInterface, g.guardMetaKey)
+	if err != nil {
+		return errors.Join(err, g.Rollback(ctx))
+	}
+	return g.db.Commit()
+}
+
+func (g *guardedMigration) Rollback(ctx context.Context) error {
+	return g.db.Rollback()
+}
+
+func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) {
+	migrationName := "TransactionsTable"
+	factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase)
+	m, err := newGuardedDataMigration(ctx, migrationName, factory, db)
+	if err != nil {
+		return nil, fmt.Errorf("creating guarded transaction migration: %w", err)
+	}
+	// Add other migrations here
+	return multiMigration{m}, nil
+}
diff --git a/cmd/soroban-rpc/internal/db/migrations/01_init.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/01_init.sql
similarity index 100%
rename from cmd/soroban-rpc/internal/db/migrations/01_init.sql
rename to cmd/soroban-rpc/internal/db/sqlmigrations/01_init.sql
diff --git a/cmd/soroban-rpc/internal/db/migrations/02_transactions.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/02_transactions.sql
similarity index 100%
rename from cmd/soroban-rpc/internal/db/migrations/02_transactions.sql
rename to cmd/soroban-rpc/internal/db/sqlmigrations/02_transactions.sql
diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index 817719ed..5ea51492 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -45,15 +45,6 @@ type TransactionReader interface {
 	GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error)
 }
 
-func NewTransactionWriter(log *log.Entry, db db.SessionInterface, networkPassphrase string) TransactionWriter {
-	return &transactionHandler{
-		log:        log,
-		db:         db,
-		stmtCache:  sq.NewStmtCache(db.GetTx()),
-		passphrase: networkPassphrase,
-	}
-}
-
 type transactionHandler struct {
 	log        *log.Entry
 	db         db.SessionInterface
@@ -311,3 +302,39 @@ func ParseTransaction(lcm xdr.LedgerCloseMeta, ingestTx ingest.LedgerTransaction
 
 	return tx, nil
 }
+
+type transactionTableMigration struct {
+	firstLedger uint32
+	writer      TransactionWriter
+}
+
+func (t *transactionTableMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
+	if meta.LedgerSequence() < t.firstLedger {
+		return nil
+	}
+	return t.writer.InsertTransactions(meta)
+}
+
+func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string) migrationApplierFactory {
+	return migrationApplierFactoryF(func(db *DB) (MigrationApplier, error) {
+		latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx)
+		if err != nil && err != ErrEmptyDB {
+			return nil, errors.Wrap(err, "couldn't get latest ledger sequence")
+		}
+		firstLedgerToMigrate := uint32(2)
+		writer := &transactionHandler{
+			log:        logger,
+			db:         db.SessionInterface,
+			stmtCache:  sq.NewStmtCache(db.GetTx()),
+			passphrase: passphrase,
+		}
+		if latestLedger > retentionWindow {
+			firstLedgerToMigrate = latestLedger - retentionWindow
+		}
+		migration := transactionTableMigration{
+			firstLedger: firstLedgerToMigrate,
+			writer:      writer,
+		}
+		return &migration, nil
+	})
+}
diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go
index 068f793d..b09e890d 100644
--- a/cmd/soroban-rpc/internal/db/transaction_test.go
+++ b/cmd/soroban-rpc/internal/db/transaction_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/log"
 	"github.com/stellar/go/xdr"
+
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces"
 
 	"github.com/stretchr/testify/assert"
@@ -21,7 +22,7 @@ func TestTransactionNotFound(t *testing.T) {
 	log := log.DefaultLogger
 	log.SetLevel(logrus.TraceLevel)
 
-	reader := NewTransactionReader(log, db, passphrase)
+	reader := NewTransactionReader(log, db.SessionInterface, passphrase)
 	_, _, err := reader.GetTransaction(context.TODO(), xdr.Hash{})
 	require.Error(t, err, ErrNoTransaction)
 }
@@ -51,7 +52,7 @@ func TestTransactionFound(t *testing.T) {
 	require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence()))
 
 	// check 404 case
-	reader := NewTransactionReader(log, db, passphrase)
+	reader := NewTransactionReader(log, db.SessionInterface, passphrase)
 	_, _, err = reader.GetTransaction(ctx, xdr.Hash{})
 	require.Error(t, err, ErrNoTransaction)
 
@@ -91,7 +92,7 @@ func BenchmarkTransactionFetch(b *testing.B) {
 		require.NoError(b, txW.InsertTransactions(lcm))
 	}
 	require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence()))
-	reader := NewTransactionReader(log, db, passphrase)
+	reader := NewTransactionReader(log, db.SessionInterface, passphrase)
 
 	randoms := make([]int, b.N)
 	for i := 0; i < b.N; i++ {

From 3776163aa190b0193ffc49b61ecdfe23216b1fb9 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 15:26:45 +0200
Subject: [PATCH 06/36] Truncate table before migration

---
 cmd/soroban-rpc/internal/db/transaction.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index 5ea51492..15def34d 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -331,6 +331,15 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retent
 		if latestLedger > retentionWindow {
 			firstLedgerToMigrate = latestLedger - retentionWindow
 		}
+		// Truncate the table, since it may contain data, causing insert conflicts later on.
+		// (the migration was shipped after the actual transactions table change)
+		// FIXME: this can be simply replaced by an upper limit in the ledgers to migrate
+		//        but ... it can't be done until https://github.com/stellar/soroban-rpc/issues/208
+		//        is addressed
+		_, err = db.Exec(ctx, sq.Delete(transactionTableName))
+		if err != nil {
+			return nil, errors.Wrap(err, "couldn't truncate transactions table")
+		}
 		migration := transactionTableMigration{
 			firstLedger: firstLedgerToMigrate,
 			writer:      writer,

From 8940d75fcbf1d05929b1f8f21540528e87f95435 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 16:05:32 +0200
Subject: [PATCH 07/36] Add applicable ranges to migrations

---
 cmd/soroban-rpc/internal/daemon/daemon.go  |  9 ++-
 cmd/soroban-rpc/internal/db/migration.go   | 71 +++++++++++++++++-----
 cmd/soroban-rpc/internal/db/transaction.go | 22 ++++---
 3 files changed, 75 insertions(+), 27 deletions(-)

diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index 0afe7596..f70f86ef 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -277,7 +277,6 @@ func MustNew(cfg *config.Config) *Daemon {
 }
 
 // mustInitializeStorage initializes the storage using what was on the DB
-// TODO: clean up once we remove the in-memory storage
 func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindows, *events.MemoryStore) {
 	eventStore := events.NewMemoryStore(
 		d,
@@ -315,8 +314,12 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 		if err := feewindows.IngestFees(txmeta); err != nil {
 			d.logger.WithError(err).Fatal("could not initialize fee stats")
 		}
-		if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil {
-			d.logger.WithError(err).Fatal("could not run migrations")
+		// TODO: clean up once we remove the in-memory storage.
+		//       (we should only stream over the required range)
+		if r := dataMigrations.ApplicableRange(); r.IsLedgerIncluded(currentSeq) {
+			if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil {
+				d.logger.WithError(err).Fatal("could not run migrations")
+			}
 		}
 		return nil
 	})
diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go
index 419f34df..32857300 100644
--- a/cmd/soroban-rpc/internal/db/migration.go
+++ b/cmd/soroban-rpc/internal/db/migration.go
@@ -11,18 +11,33 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 )
 
+type MigrationLedgerRange struct {
+	firstLedgerSeq uint32
+	lastLedgerSeq  uint32
+}
+
+func (mlr *MigrationLedgerRange) IsLedgerIncluded(ledgerSeq uint32) bool {
+	if mlr == nil {
+		return false
+	}
+	return ledgerSeq >= mlr.firstLedgerSeq && ledgerSeq <= mlr.lastLedgerSeq
+}
+
 type MigrationApplier interface {
 	Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error
+	// ApplicableRange returns the closed ledger sequence interval,
+	// a null result indicates the empty range
+	ApplicableRange() *MigrationLedgerRange
 }
 
 type migrationApplierFactory interface {
-	New(db *DB) (MigrationApplier, error)
+	New(db *DB, latestLedger uint32) (MigrationApplier, error)
 }
 
-type migrationApplierFactoryF func(db *DB) (MigrationApplier, error)
+type migrationApplierFactoryF func(db *DB, latestLedger uint32) (MigrationApplier, error)
 
-func (m migrationApplierFactoryF) New(db *DB) (MigrationApplier, error) {
-	return m(db)
+func (m migrationApplierFactoryF) New(db *DB, latestLedger uint32) (MigrationApplier, error) {
+	return m(db, latestLedger)
 }
 
 type Migration interface {
@@ -33,30 +48,46 @@ type Migration interface {
 
 type multiMigration []Migration
 
-func (m multiMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
+func (mm multiMigration) ApplicableRange() *MigrationLedgerRange {
+	var result *MigrationLedgerRange
+	for _, m := range mm {
+		r := m.ApplicableRange()
+		if r != nil {
+			if result == nil {
+				result = r
+			} else {
+				result.firstLedgerSeq = min(r.firstLedgerSeq, result.firstLedgerSeq)
+				result.lastLedgerSeq = max(r.lastLedgerSeq, result.lastLedgerSeq)
+			}
+		}
+	}
+	return result
+}
+
+func (mm multiMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
 	var err error
-	for _, data := range m {
-		if localErr := data.Apply(ctx, meta); localErr != nil {
+	for _, m := range mm {
+		if localErr := m.Apply(ctx, meta); localErr != nil {
 			err = errors.Join(err, localErr)
 		}
 	}
 	return err
 }
 
-func (m multiMigration) Commit(ctx context.Context) error {
+func (mm multiMigration) Commit(ctx context.Context) error {
 	var err error
-	for _, data := range m {
-		if localErr := data.Commit(ctx); localErr != nil {
+	for _, m := range mm {
+		if localErr := m.Commit(ctx); localErr != nil {
 			err = errors.Join(err, localErr)
 		}
 	}
 	return err
 }
 
-func (m multiMigration) Rollback(ctx context.Context) error {
+func (mm multiMigration) Rollback(ctx context.Context) error {
 	var err error
-	for _, data := range m {
-		if localErr := data.Rollback(ctx); localErr != nil {
+	for _, m := range mm {
+		if localErr := m.Rollback(ctx); localErr != nil {
 			err = errors.Join(err, localErr)
 		}
 	}
@@ -83,7 +114,12 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa
 		migrationDB.Rollback()
 		return nil, err
 	}
-	applier, err := factory.New(migrationDB)
+	latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx)
+	if err != nil && err != ErrEmptyDB {
+		migrationDB.Rollback()
+		return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err)
+	}
+	applier, err := factory.New(migrationDB, latestLedger)
 	if err != nil {
 		migrationDB.Rollback()
 		return nil, err
@@ -104,6 +140,13 @@ func (g *guardedMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta)
 	return g.migration.Apply(ctx, meta)
 }
 
+func (g *guardedMigration) ApplicableRange() *MigrationLedgerRange {
+	if g.alreadyMigrated {
+		return nil
+	}
+	return g.migration.ApplicableRange()
+}
+
 func (g *guardedMigration) Commit(ctx context.Context) error {
 	if g.alreadyMigrated {
 		return nil
diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index 15def34d..3231a36c 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -305,22 +305,23 @@ func ParseTransaction(lcm xdr.LedgerCloseMeta, ingestTx ingest.LedgerTransaction
 
 type transactionTableMigration struct {
 	firstLedger uint32
+	lastLedger  uint32
 	writer      TransactionWriter
 }
 
-func (t *transactionTableMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
-	if meta.LedgerSequence() < t.firstLedger {
-		return nil
+func (t *transactionTableMigration) ApplicableRange() *MigrationLedgerRange {
+	return &MigrationLedgerRange{
+		firstLedgerSeq: t.firstLedger,
+		lastLedgerSeq:  t.lastLedger,
 	}
+}
+
+func (t *transactionTableMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
 	return t.writer.InsertTransactions(meta)
 }
 
 func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string) migrationApplierFactory {
-	return migrationApplierFactoryF(func(db *DB) (MigrationApplier, error) {
-		latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx)
-		if err != nil && err != ErrEmptyDB {
-			return nil, errors.Wrap(err, "couldn't get latest ledger sequence")
-		}
+	return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) {
 		firstLedgerToMigrate := uint32(2)
 		writer := &transactionHandler{
 			log:        logger,
@@ -336,12 +337,13 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retent
 		// FIXME: this can be simply replaced by an upper limit in the ledgers to migrate
 		//        but ... it can't be done until https://github.com/stellar/soroban-rpc/issues/208
 		//        is addressed
-		_, err = db.Exec(ctx, sq.Delete(transactionTableName))
+		_, err := db.Exec(ctx, sq.Delete(transactionTableName))
 		if err != nil {
-			return nil, errors.Wrap(err, "couldn't truncate transactions table")
+			return nil, fmt.Errorf("couldn't delete table %q: %w", transactionTableName, err)
 		}
 		migration := transactionTableMigration{
 			firstLedger: firstLedgerToMigrate,
+			lastLedger:  latestLedger,
 			writer:      writer,
 		}
 		return &migration, nil

From ddc4035e40148e504c1e36b269b361eab649adde Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 17:26:06 +0200
Subject: [PATCH 08/36] Address review feedback

---
 cmd/soroban-rpc/internal/db/db.go        | 4 ++--
 cmd/soroban-rpc/internal/db/migration.go | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go
index 45bbad0b..837a22b7 100644
--- a/cmd/soroban-rpc/internal/db/db.go
+++ b/cmd/soroban-rpc/internal/db/db.go
@@ -116,9 +116,9 @@ func getMetaBool(ctx context.Context, q db.SessionInterface, key string) (bool,
 	return strconv.ParseBool(valueStr)
 }
 
-func setMetaBool(ctx context.Context, q db.SessionInterface, key string) error {
+func setMetaBool(ctx context.Context, q db.SessionInterface, key string, value bool) error {
 	query := sq.Replace(metaTableName).
-		Values(key, "true")
+		Values(key, strconv.FormatBool(value))
 	_, err := q.Exec(ctx, query)
 	return err
 }
diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go
index 32857300..574184b5 100644
--- a/cmd/soroban-rpc/internal/db/migration.go
+++ b/cmd/soroban-rpc/internal/db/migration.go
@@ -151,8 +151,9 @@ func (g *guardedMigration) Commit(ctx context.Context) error {
 	if g.alreadyMigrated {
 		return nil
 	}
-	err := setMetaBool(ctx, g.db.SessionInterface, g.guardMetaKey)
+	err := setMetaBool(ctx, g.db.SessionInterface, g.guardMetaKey, true)
 	if err != nil {
+		g.Rollback(ctx)
 		return errors.Join(err, g.Rollback(ctx))
 	}
 	return g.db.Commit()

From e3711bff5bd94a7461d1fc7a0b0d463d97f7d3d8 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 21:23:47 +0200
Subject: [PATCH 09/36] Remove DB Clone() method

---
 cmd/soroban-rpc/internal/daemon/daemon.go       |  2 +-
 cmd/soroban-rpc/internal/db/db.go               | 11 ++---------
 cmd/soroban-rpc/internal/db/ledger_test.go      | 12 +++---------
 cmd/soroban-rpc/internal/db/ledgerentry.go      |  6 +++---
 cmd/soroban-rpc/internal/db/migration.go        |  9 ++++++---
 cmd/soroban-rpc/internal/db/transaction.go      |  2 +-
 cmd/soroban-rpc/internal/db/transaction_test.go |  6 +++---
 7 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index f70f86ef..fe0cece5 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -241,7 +241,7 @@ func MustNew(cfg *config.Config) *Daemon {
 		Logger:            logger,
 		LedgerReader:      db.NewLedgerReader(dbConn),
 		LedgerEntryReader: db.NewLedgerEntryReader(dbConn),
-		TransactionReader: db.NewTransactionReader(logger, dbConn.SessionInterface, cfg.NetworkPassphrase),
+		TransactionReader: db.NewTransactionReader(logger, dbConn, cfg.NetworkPassphrase),
 		PreflightGetter:   preflightWorkerPool,
 	})
 
diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go
index 837a22b7..a3d54a6f 100644
--- a/cmd/soroban-rpc/internal/db/db.go
+++ b/cmd/soroban-rpc/internal/db/db.go
@@ -56,13 +56,6 @@ type DB struct {
 	cache *dbCache
 }
 
-func (db *DB) Clone() *DB {
-	return &DB{
-		SessionInterface: db.SessionInterface.Clone(),
-		cache:            db.cache,
-	}
-}
-
 func openSQLiteDB(dbFilePath string) (*db.Session, error) {
 	// 1. Use Write-Ahead Logging (WAL).
 	// 2. Disable WAL auto-checkpointing (we will do the checkpointing ourselves with wal_checkpoint pragmas
@@ -222,11 +215,11 @@ func NewReadWriter(
 }
 
 func (rw *readWriter) GetLatestLedgerSequence(ctx context.Context) (uint32, error) {
-	return getLatestLedgerSequence(ctx, rw.db.SessionInterface, rw.db.cache)
+	return getLatestLedgerSequence(ctx, rw.db, rw.db.cache)
 }
 
 func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
-	txSession := rw.db.SessionInterface.Clone()
+	txSession := rw.db.Clone()
 	if err := txSession.Begin(ctx); err != nil {
 		return nil, err
 	}
diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go
index 9026b784..25369fac 100644
--- a/cmd/soroban-rpc/internal/db/ledger_test.go
+++ b/cmd/soroban-rpc/internal/db/ledger_test.go
@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/log"
@@ -107,16 +108,9 @@ func NewTestDB(tb testing.TB) *DB {
 	tmp := tb.TempDir()
 	dbPath := path.Join(tmp, "db.sqlite")
 	db, err := OpenSQLiteDB(dbPath)
-	if err != nil {
-		assert.NoError(tb, db.Close())
-	}
+	require.NoError(tb, err)
 	tb.Cleanup(func() {
 		assert.NoError(tb, db.Close())
 	})
-	return &DB{
-		SessionInterface: db.SessionInterface,
-		cache: &dbCache{
-			ledgerEntries: newTransactionalCache(),
-		},
-	}
+	return db
 }
diff --git a/cmd/soroban-rpc/internal/db/ledgerentry.go b/cmd/soroban-rpc/internal/db/ledgerentry.go
index d9d60d37..aaffed06 100644
--- a/cmd/soroban-rpc/internal/db/ledgerentry.go
+++ b/cmd/soroban-rpc/internal/db/ledgerentry.go
@@ -341,12 +341,12 @@ func NewLedgerEntryReader(db *DB) LedgerEntryReader {
 }
 
 func (r ledgerEntryReader) GetLatestLedgerSequence(ctx context.Context) (uint32, error) {
-	return getLatestLedgerSequence(ctx, r.db.SessionInterface, r.db.cache)
+	return getLatestLedgerSequence(ctx, r.db, r.db.cache)
 }
 
 // NewCachedTx() caches all accessed ledger entries and select statements. If many ledger entries are accessed, it will grow without bounds.
 func (r ledgerEntryReader) NewCachedTx(ctx context.Context) (LedgerEntryReadTx, error) {
-	txSession := r.db.SessionInterface.Clone()
+	txSession := r.db.Clone()
 	// We need to copy the cached ledger entries locally when we start the transaction
 	// since otherwise we would break the consistency between the transaction and the cache.
 
@@ -370,7 +370,7 @@ func (r ledgerEntryReader) NewCachedTx(ctx context.Context) (LedgerEntryReadTx,
 }
 
 func (r ledgerEntryReader) NewTx(ctx context.Context) (LedgerEntryReadTx, error) {
-	txSession := r.db.SessionInterface.Clone()
+	txSession := r.db.Clone()
 	if err := txSession.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}); err != nil {
 		return nil, err
 	}
diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go
index 574184b5..7f802c96 100644
--- a/cmd/soroban-rpc/internal/db/migration.go
+++ b/cmd/soroban-rpc/internal/db/migration.go
@@ -104,12 +104,15 @@ type guardedMigration struct {
 }
 
 func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, factory migrationApplierFactory, db *DB) (Migration, error) {
-	migrationDB := db.Clone()
+	migrationDB := &DB{
+		cache:            db.cache,
+		SessionInterface: db.SessionInterface.Clone(),
+	}
 	if err := migrationDB.Begin(ctx); err != nil {
 		return nil, err
 	}
 	metaKey := "Migration" + uniqueMigrationName + "Done"
-	previouslyMigrated, err := getMetaBool(ctx, migrationDB.SessionInterface, metaKey)
+	previouslyMigrated, err := getMetaBool(ctx, migrationDB, metaKey)
 	if err != nil && !errors.Is(err, ErrEmptyDB) {
 		migrationDB.Rollback()
 		return nil, err
@@ -151,7 +154,7 @@ func (g *guardedMigration) Commit(ctx context.Context) error {
 	if g.alreadyMigrated {
 		return nil
 	}
-	err := setMetaBool(ctx, g.db.SessionInterface, g.guardMetaKey, true)
+	err := setMetaBool(ctx, g.db, g.guardMetaKey, true)
 	if err != nil {
 		g.Rollback(ctx)
 		return errors.Join(err, g.Rollback(ctx))
diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index 3231a36c..d07fca71 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -325,7 +325,7 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retent
 		firstLedgerToMigrate := uint32(2)
 		writer := &transactionHandler{
 			log:        logger,
-			db:         db.SessionInterface,
+			db:         db,
 			stmtCache:  sq.NewStmtCache(db.GetTx()),
 			passphrase: passphrase,
 		}
diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go
index b09e890d..7257c1e1 100644
--- a/cmd/soroban-rpc/internal/db/transaction_test.go
+++ b/cmd/soroban-rpc/internal/db/transaction_test.go
@@ -22,7 +22,7 @@ func TestTransactionNotFound(t *testing.T) {
 	log := log.DefaultLogger
 	log.SetLevel(logrus.TraceLevel)
 
-	reader := NewTransactionReader(log, db.SessionInterface, passphrase)
+	reader := NewTransactionReader(log, db, passphrase)
 	_, _, err := reader.GetTransaction(context.TODO(), xdr.Hash{})
 	require.Error(t, err, ErrNoTransaction)
 }
@@ -52,7 +52,7 @@ func TestTransactionFound(t *testing.T) {
 	require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence()))
 
 	// check 404 case
-	reader := NewTransactionReader(log, db.SessionInterface, passphrase)
+	reader := NewTransactionReader(log, db, passphrase)
 	_, _, err = reader.GetTransaction(ctx, xdr.Hash{})
 	require.Error(t, err, ErrNoTransaction)
 
@@ -92,7 +92,7 @@ func BenchmarkTransactionFetch(b *testing.B) {
 		require.NoError(b, txW.InsertTransactions(lcm))
 	}
 	require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence()))
-	reader := NewTransactionReader(log, db.SessionInterface, passphrase)
+	reader := NewTransactionReader(log, db, passphrase)
 
 	randoms := make([]int, b.N)
 	for i := 0; i < b.N; i++ {

From 638165569a28309be49de52f4ef3e7b10f9c79b4 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Tue, 11 Jun 2024 21:30:20 +0200
Subject: [PATCH 10/36] Cleanup ledger sequence ranges even further

---
 cmd/soroban-rpc/internal/db/migration.go   | 35 ++++++++++++----------
 cmd/soroban-rpc/internal/db/transaction.go |  4 +--
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go
index 7f802c96..336c0eef 100644
--- a/cmd/soroban-rpc/internal/db/migration.go
+++ b/cmd/soroban-rpc/internal/db/migration.go
@@ -11,23 +11,36 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 )
 
-type MigrationLedgerRange struct {
+type LedgerSeqRange struct {
 	firstLedgerSeq uint32
 	lastLedgerSeq  uint32
 }
 
-func (mlr *MigrationLedgerRange) IsLedgerIncluded(ledgerSeq uint32) bool {
+func (mlr *LedgerSeqRange) IsLedgerIncluded(ledgerSeq uint32) bool {
 	if mlr == nil {
 		return false
 	}
 	return ledgerSeq >= mlr.firstLedgerSeq && ledgerSeq <= mlr.lastLedgerSeq
 }
 
+func (mlr *LedgerSeqRange) Merge(other *LedgerSeqRange) *LedgerSeqRange {
+	if mlr == nil {
+		return other
+	}
+	if other == nil {
+		return mlr
+	}
+	return &LedgerSeqRange{
+		firstLedgerSeq: min(mlr.firstLedgerSeq, other.firstLedgerSeq),
+		lastLedgerSeq:  max(mlr.lastLedgerSeq, other.lastLedgerSeq),
+	}
+}
+
 type MigrationApplier interface {
 	Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error
 	// ApplicableRange returns the closed ledger sequence interval,
 	// a null result indicates the empty range
-	ApplicableRange() *MigrationLedgerRange
+	ApplicableRange() *LedgerSeqRange
 }
 
 type migrationApplierFactory interface {
@@ -48,18 +61,10 @@ type Migration interface {
 
 type multiMigration []Migration
 
-func (mm multiMigration) ApplicableRange() *MigrationLedgerRange {
-	var result *MigrationLedgerRange
+func (mm multiMigration) ApplicableRange() *LedgerSeqRange {
+	var result *LedgerSeqRange
 	for _, m := range mm {
-		r := m.ApplicableRange()
-		if r != nil {
-			if result == nil {
-				result = r
-			} else {
-				result.firstLedgerSeq = min(r.firstLedgerSeq, result.firstLedgerSeq)
-				result.lastLedgerSeq = max(r.lastLedgerSeq, result.lastLedgerSeq)
-			}
-		}
+		result = m.ApplicableRange().Merge(result)
 	}
 	return result
 }
@@ -143,7 +148,7 @@ func (g *guardedMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta)
 	return g.migration.Apply(ctx, meta)
 }
 
-func (g *guardedMigration) ApplicableRange() *MigrationLedgerRange {
+func (g *guardedMigration) ApplicableRange() *LedgerSeqRange {
 	if g.alreadyMigrated {
 		return nil
 	}
diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index d07fca71..81d61db5 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -309,8 +309,8 @@ type transactionTableMigration struct {
 	writer      TransactionWriter
 }
 
-func (t *transactionTableMigration) ApplicableRange() *MigrationLedgerRange {
-	return &MigrationLedgerRange{
+func (t *transactionTableMigration) ApplicableRange() *LedgerSeqRange {
+	return &LedgerSeqRange{
 		firstLedgerSeq: t.firstLedger,
 		lastLedgerSeq:  t.lastLedger,
 	}

From 52ba0ac6350a567a02b14eba9caf0527d997c82d Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Wed, 12 Jun 2024 02:15:48 +0200
Subject: [PATCH 11/36] Add comments and make sure multiMigration only applies
 needed migrations

---
 cmd/soroban-rpc/internal/db/migration.go | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go
index 336c0eef..2ed97946 100644
--- a/cmd/soroban-rpc/internal/db/migration.go
+++ b/cmd/soroban-rpc/internal/db/migration.go
@@ -30,6 +30,8 @@ func (mlr *LedgerSeqRange) Merge(other *LedgerSeqRange) *LedgerSeqRange {
 	if other == nil {
 		return mlr
 	}
+	// TODO: using min/max can result in a much larger range than needed,
+	//       as an optimization, we should probably use a sequence of ranges instead.
 	return &LedgerSeqRange{
 		firstLedgerSeq: min(mlr.firstLedgerSeq, other.firstLedgerSeq),
 		lastLedgerSeq:  max(mlr.lastLedgerSeq, other.lastLedgerSeq),
@@ -37,10 +39,12 @@ func (mlr *LedgerSeqRange) Merge(other *LedgerSeqRange) *LedgerSeqRange {
 }
 
 type MigrationApplier interface {
-	Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error
 	// ApplicableRange returns the closed ledger sequence interval,
-	// a null result indicates the empty range
+	// where Apply() should be called. A null result indicates the empty range
 	ApplicableRange() *LedgerSeqRange
+	// Apply applies the migration on a ledger. It should never be applied
+	// in ledgers outside the ApplicableRange()
+	Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error
 }
 
 type migrationApplierFactory interface {
@@ -72,6 +76,11 @@ func (mm multiMigration) ApplicableRange() *LedgerSeqRange {
 func (mm multiMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
 	var err error
 	for _, m := range mm {
+		ledgerSeq := meta.LedgerSequence()
+		if !m.ApplicableRange().IsLedgerIncluded(ledgerSeq) {
+			// The range of a sub-migration can be smaller than the global range.
+			continue
+		}
 		if localErr := m.Apply(ctx, meta); localErr != nil {
 			err = errors.Join(err, localErr)
 		}
@@ -143,6 +152,8 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa
 
 func (g *guardedMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error {
 	if g.alreadyMigrated {
+		// This shouldn't happen since we would be out of the applicable range
+		// but, just in case.
 		return nil
 	}
 	return g.migration.Apply(ctx, meta)

From 242242218265ed0a471e27482f302228116f4b72 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 01:46:00 +0200
Subject: [PATCH 12/36] Add infrastructure to run integration tests from the
 soroban-rpc process

---
 cmd/soroban-rpc/internal/test/archive_test.go |   4 +-
 cmd/soroban-rpc/internal/test/integration.go  | 132 ++++++++++++++----
 cmd/soroban-rpc/internal/test/metrics_test.go |  11 +-
 .../test/simulate_transaction_test.go         |   2 +-
 cmd/soroban-rpc/internal/test/upgrade_test.go |   2 +-
 5 files changed, 111 insertions(+), 40 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/archive_test.go b/cmd/soroban-rpc/internal/test/archive_test.go
index 127e6e61..d0aebe49 100644
--- a/cmd/soroban-rpc/internal/test/archive_test.go
+++ b/cmd/soroban-rpc/internal/test/archive_test.go
@@ -17,9 +17,9 @@ func TestArchiveUserAgent(t *testing.T) {
 	}
 	NewTest(t, cfg)
 
-	_, ok := userAgents.Load("testing")
+	_, ok := userAgents.Load("soroban-rpc/0.0.0")
 	assert.True(t, ok, "rpc service should set user agent for history archives")
 
-	_, ok = userAgents.Load("testing/captivecore")
+	_, ok = userAgents.Load("soroban-rpc/0.0.0/captivecore")
 	assert.True(t, ok, "rpc captive core should set user agent for history archives")
 }
diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 03e86237..0c6689f7 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -18,16 +18,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/stellar/go/clients/stellarcore"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/txnbuild"
+	"github.com/stretchr/testify/require"
+
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 )
 
 const (
@@ -48,6 +49,13 @@ const (
 type TestConfig struct {
 	historyArchiveProxyCallback func(*http.Request)
 	ProtocolVersion             uint32
+	UseRealRPCVersion           string
+	UseSQLitePath               string
+}
+
+type daemonOrCommand struct {
+	daemon  *daemon.Daemon
+	command *exec.Cmd
 }
 
 type Test struct {
@@ -57,7 +65,7 @@ type Test struct {
 
 	protocolVersion uint32
 
-	daemon *daemon.Daemon
+	rpcInstance daemonOrCommand
 
 	historyArchiveProxy         *httptest.Server
 	historyArchiveProxyCallback func(*http.Request)
@@ -87,9 +95,13 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 		AccountID: i.MasterKey().Address(),
 		Sequence:  0,
 	}
+	realRPCVersion := ""
+	sqlLitePath := ""
 	if cfg != nil {
 		i.historyArchiveProxyCallback = cfg.historyArchiveProxyCallback
 		i.protocolVersion = cfg.ProtocolVersion
+		realRPCVersion = cfg.UseRealRPCVersion
+		sqlLitePath = cfg.UseSQLitePath
 	}
 
 	if i.protocolVersion == 0 {
@@ -111,7 +123,7 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 	i.coreClient = &stellarcore.Client{URL: "http://localhost:" + strconv.Itoa(stellarCorePort)}
 	i.waitForCore()
 	i.waitForCheckpoint()
-	i.launchDaemon(coreBinaryPath)
+	i.launchRPC(coreBinaryPath, realRPCVersion, sqlLitePath)
 
 	return i
 }
@@ -153,7 +165,7 @@ func (i *Test) waitForCheckpoint() {
 	i.t.Fatal("Core could not reach checkpoint ledger after 30s")
 }
 
-func (i *Test) launchDaemon(coreBinaryPath string) {
+func (i *Test) launchRPC(coreBinaryPath string, realRPCVersion string, sqlitePath string) {
 	var config config.Config
 	cmd := &cobra.Command{}
 	if err := config.AddFlags(cmd); err != nil {
@@ -162,30 +174,37 @@ func (i *Test) launchDaemon(coreBinaryPath string) {
 	if err := config.SetValues(func(string) (string, bool) { return "", false }); err != nil {
 		i.t.FailNow()
 	}
+	if sqlitePath == "" {
+		sqlitePath = path.Join(i.t.TempDir(), "soroban_rpc.sqlite")
+	}
+	env := map[string]string{
+		"ENDPOINT":                       fmt.Sprintf("localhost:%d", sorobanRPCPort),
+		"ADMIN_ENDPOINT":                 fmt.Sprintf("localhost:%d", adminPort),
+		"STELLAR_CORE_URL":               "http://localhost:" + strconv.Itoa(stellarCorePort),
+		"CORE_REQUEST_TIMEOUT":           "2s",
+		"STELLAR_CORE_BINARY_PATH":       coreBinaryPath,
+		"CAPTIVE_CORE_CONFIG_PATH":       path.Join(i.composePath, "captive-core-integration-tests.cfg"),
+		"CAPTIVE_CORE_STORAGE_PATH":      i.t.TempDir(),
+		"STELLAR_CAPTIVE_CORE_HTTP_PORT": "0",
+		"FRIENDBOT_URL":                  friendbotURL,
+		"NETWORK_PASSPHRASE":             StandaloneNetworkPassphrase,
+		"HISTORY_ARCHIVE_URLS":           i.historyArchiveProxy.URL,
+		"LOG_LEVEL":                      "debug",
+		"DB_PATH":                        sqlitePath,
+		"INGESTION_TIMEOUT":              "10m",
+		"EVENT_LEDGER_RETENTION_WINDOW":  strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers),
+		"TRANSACTION_RETENTION_WINDOW":   strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers),
+		"CHECKPOINT_FREQUENCY":           strconv.Itoa(checkpointFrequency),
+		"MAX_HEALTHY_LEDGER_LATENCY":     "10s",
+		"PREFLIGHT_ENABLE_DEBUG":         "true",
+	}
 
-	config.Endpoint = fmt.Sprintf("localhost:%d", sorobanRPCPort)
-	config.AdminEndpoint = fmt.Sprintf("localhost:%d", adminPort)
-	config.StellarCoreURL = "http://localhost:" + strconv.Itoa(stellarCorePort)
-	config.CoreRequestTimeout = time.Second * 2
-	config.StellarCoreBinaryPath = coreBinaryPath
-	config.CaptiveCoreConfigPath = path.Join(i.composePath, "captive-core-integration-tests.cfg")
-	config.CaptiveCoreStoragePath = i.t.TempDir()
-	config.CaptiveCoreHTTPPort = 0
-	config.FriendbotURL = friendbotURL
-	config.NetworkPassphrase = StandaloneNetworkPassphrase
-	config.HistoryArchiveURLs = []string{i.historyArchiveProxy.URL}
-	config.LogLevel = logrus.DebugLevel
-	config.SQLiteDBPath = path.Join(i.t.TempDir(), "soroban_rpc.sqlite")
-	config.IngestionTimeout = 10 * time.Minute
-	config.EventLedgerRetentionWindow = ledgerbucketwindow.OneDayOfLedgers
-	config.TransactionLedgerRetentionWindow = ledgerbucketwindow.OneDayOfLedgers
-	config.CheckpointFrequency = checkpointFrequency
-	config.MaxHealthyLedgerLatency = time.Second * 10
-	config.PreflightEnableDebug = true
-	config.HistoryArchiveUserAgent = "testing"
-
-	i.daemon = daemon.MustNew(&config)
-	go i.daemon.Run()
+	if realRPCVersion != "" {
+		i.rpcInstance.command = i.compileAndStartRPC(env, realRPCVersion)
+	} else {
+		i.rpcInstance.daemon = i.createDaemon(env)
+		go i.rpcInstance.daemon.Run()
+	}
 
 	// wait for the storage to catch up for 1 minute
 	info, err := i.coreClient.Info(context.Background())
@@ -194,7 +213,9 @@ func (i *Test) launchDaemon(coreBinaryPath string) {
 	}
 	targetLedgerSequence := uint32(info.Info.Ledger.Num)
 
-	reader := db.NewLedgerEntryReader(i.daemon.GetDB())
+	dbConn, err := db.OpenSQLiteDB(sqlitePath)
+	require.NoError(i.t, err)
+	reader := db.NewLedgerEntryReader(dbConn)
 	success := false
 	for t := 30; t >= 0; t -= 1 {
 		sequence, err := reader.GetLatestLedgerSequence(context.Background())
@@ -215,6 +236,51 @@ func (i *Test) launchDaemon(coreBinaryPath string) {
 	}
 }
 
+func (i *Test) compileAndStartRPC(env map[string]string, version string) *exec.Cmd {
+	newRPCDir := i.t.TempDir()
+
+	Command := func(name string, arg ...string) *exec.Cmd {
+		cmd := exec.Command(name, arg...)
+		cmd.Dir = newRPCDir
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = os.Stdin
+		return cmd
+	}
+
+	// Clone
+	rootDir := path.Join(i.composePath, "..", "..", "..", "..")
+	err := Command("git", "clone", "--depth", "1", "--branch", version, "file://"+rootDir, newRPCDir).Run()
+	require.NoError(i.t, err)
+
+	// Compile
+	cmd := Command("make", "build-libpreflight")
+	require.NoError(i.t, cmd.Run())
+	cmd = Command("go", "build", "./cmd/soroban-rpc")
+	require.NoError(i.t, cmd.Run())
+
+	// Run
+	cmd = Command(path.Join(newRPCDir, "soroban-rpc"))
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+	}
+	require.NoError(i.t, cmd.Start())
+
+	return cmd
+}
+
+func (i *Test) createDaemon(env map[string]string) *daemon.Daemon {
+	var cfg config.Config
+	lookup := func(s string) (string, bool) {
+		ret, ok := env[s]
+		return ret, ok
+	}
+	require.NoError(i.t, cfg.SetValues(lookup))
+	require.NoError(i.t, cfg.Validate())
+	cfg.HistoryArchiveUserAgent = fmt.Sprintf("soroban-rpc/%s", config.Version)
+	return daemon.MustNew(&cfg)
+}
+
 // Runs a docker-compose command applied to the above configs
 func (i *Test) runComposeCommand(args ...string) {
 	integrationYaml := filepath.Join(i.composePath, "docker-compose.yml")
@@ -244,8 +310,12 @@ func (i *Test) runComposeCommand(args ...string) {
 func (i *Test) prepareShutdownHandlers() {
 	i.shutdownCalls = append(i.shutdownCalls,
 		func() {
-			if i.daemon != nil {
-				i.daemon.Close()
+			if i.rpcInstance.daemon != nil {
+				i.rpcInstance.daemon.Close()
+			}
+			if i.rpcInstance.command != nil {
+				require.NoError(i.t, i.rpcInstance.command.Process.Kill())
+				i.rpcInstance.command.Wait()
 			}
 			if i.historyArchiveProxy != nil {
 				i.historyArchiveProxy.Close()
diff --git a/cmd/soroban-rpc/internal/test/metrics_test.go b/cmd/soroban-rpc/internal/test/metrics_test.go
index 9bf63a48..eaac747a 100644
--- a/cmd/soroban-rpc/internal/test/metrics_test.go
+++ b/cmd/soroban-rpc/internal/test/metrics_test.go
@@ -2,15 +2,16 @@ package test
 
 import (
 	"fmt"
-	io_prometheus_client "github.com/prometheus/client_model/go"
-	"github.com/stellar/go/support/errors"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"net/http"
 	"net/url"
 	"runtime"
 	"testing"
 
+	io_prometheus_client "github.com/prometheus/client_model/go"
+	"github.com/stellar/go/support/errors"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
@@ -29,12 +30,12 @@ func TestMetrics(t *testing.T) {
 	)
 	require.Contains(t, metrics, buildMetric)
 
-	logger := test.daemon.Logger()
+	logger := test.rpcInstance.daemon.Logger()
 	err := errors.Errorf("test-error")
 	logger.WithError(err).Error("test error 1")
 	logger.WithError(err).Error("test error 2")
 
-	metricFamilies, err := test.daemon.MetricsRegistry().Gather()
+	metricFamilies, err := test.rpcInstance.daemon.MetricsRegistry().Gather()
 	assert.NoError(t, err)
 	var metric *io_prometheus_client.MetricFamily
 	for _, mf := range metricFamilies {
diff --git a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
index 65c5767d..61ee8e4f 100644
--- a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
+++ b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
@@ -563,7 +563,7 @@ func TestSimulateInvokeContractTransactionSucceeds(t *testing.T) {
 }
 
 func TestSimulateTransactionError(t *testing.T) {
-	test := NewTest(t, nil)
+	test := NewTest(t, &TestConfig{UseRealRPCVersion: "main"})
 
 	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
 	client := jrpc2.NewClient(ch, nil)
diff --git a/cmd/soroban-rpc/internal/test/upgrade_test.go b/cmd/soroban-rpc/internal/test/upgrade_test.go
index 6889e614..fdc0c95b 100644
--- a/cmd/soroban-rpc/internal/test/upgrade_test.go
+++ b/cmd/soroban-rpc/internal/test/upgrade_test.go
@@ -53,7 +53,7 @@ func TestUpgradeFrom20To21(t *testing.T) {
 	// estimations
 	test.UpgradeProtocol(21)
 	// Wait for the ledger to advance, so that the simulation library passes the right protocol number
-	rpcDB := test.daemon.GetDB()
+	rpcDB := test.rpcInstance.daemon.GetDB()
 	initialLedgerSequence, err := db.NewLedgerEntryReader(rpcDB).GetLatestLedgerSequence(context.Background())
 	require.Eventually(t,
 		func() bool {

From d1e790c150a5a300c4d0a4a6ac9d96c3a026c4c0 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 06:05:38 +0200
Subject: [PATCH 13/36] Use docker container instead

---
 cmd/soroban-rpc/internal/db/transaction.go    |   3 +-
 .../internal/test/docker-compose.rpc.yml      |  15 ++
 cmd/soroban-rpc/internal/test/integration.go  | 214 ++++++++++--------
 cmd/soroban-rpc/internal/test/metrics_test.go |   4 +-
 cmd/soroban-rpc/internal/test/migrate_test.go | 125 ++++++++++
 .../test/simulate_transaction_test.go         |   8 +-
 cmd/soroban-rpc/internal/test/upgrade_test.go |   2 +-
 7 files changed, 267 insertions(+), 104 deletions(-)
 create mode 100644 cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
 create mode 100644 cmd/soroban-rpc/internal/test/migrate_test.go

diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go
index 81d61db5..099b4a15 100644
--- a/cmd/soroban-rpc/internal/db/transaction.go
+++ b/cmd/soroban-rpc/internal/db/transaction.go
@@ -107,8 +107,7 @@ func (txn *transactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error
 	}
 	_, err = query.RunWith(txn.stmtCache).Exec()
 
-	L.WithError(err).
-		WithField("duration", time.Since(start)).
+	L.WithField("duration", time.Since(start)).
 		Infof("Ingested %d transaction lookups", len(transactions))
 
 	return err
diff --git a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
new file mode 100644
index 00000000..d142ed9c
--- /dev/null
+++ b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
@@ -0,0 +1,15 @@
+version: '3'
+services:
+  rpc:
+    platform: linux/amd64
+    image: stellar/soroban-rpc:${RPC_IMAGE_TAG}
+    depends_on:
+      - core
+    ports:
+      - "8000:8000"
+      - "8080:8080"
+    command: --config-path /soroban-rpc.config
+    volumes:
+      - ${RPC_CONFIG_MOUNT_DIR}/stellar-core-integration-tests.cfg:/stellar-core.cfg
+      - ${RPC_CONFIG_MOUNT_DIR}/soroban-rpc.config:/soroban-rpc.config
+      - ${RPC_SQLITE_MOUNT_DIR}:/db/
\ No newline at end of file
diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 0c6689f7..c9f0cb8a 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -1,8 +1,10 @@
 package test
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
@@ -13,27 +15,29 @@ import (
 	"path"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 	"testing"
 	"time"
 
-	"github.com/spf13/cobra"
+	"github.com/creachadair/jrpc2"
+	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stellar/go/clients/stellarcore"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stretchr/testify/require"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db"
 )
 
 const (
 	StandaloneNetworkPassphrase = "Standalone Network ; February 2017"
-	maxSupportedProtocolVersion = 21
+	MaxSupportedProtocolVersion = 21
 	stellarCorePort             = 11626
 	stellarCoreArchiveHost      = "localhost:1570"
 	goModFile                   = "go.mod"
@@ -53,11 +57,6 @@ type TestConfig struct {
 	UseSQLitePath               string
 }
 
-type daemonOrCommand struct {
-	daemon  *daemon.Daemon
-	command *exec.Cmd
-}
-
 type Test struct {
 	t *testing.T
 
@@ -65,7 +64,11 @@ type Test struct {
 
 	protocolVersion uint32
 
-	rpcInstance daemonOrCommand
+	rpcContainerVersion string
+	rpcConfigMountDir   string
+	rpcSQLiteMountDir   string
+
+	daemon *daemon.Daemon
 
 	historyArchiveProxy         *httptest.Server
 	historyArchiveProxyCallback func(*http.Request)
@@ -81,11 +84,6 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 	if os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_ENABLED") == "" {
 		t.Skip("skipping integration test: SOROBAN_RPC_INTEGRATION_TESTS_ENABLED not set")
 	}
-	coreBinaryPath := os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")
-	if coreBinaryPath == "" {
-		t.Fatal("missing SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")
-	}
-
 	i := &Test{
 		t:           t,
 		composePath: findDockerComposePath(),
@@ -95,12 +93,12 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 		AccountID: i.MasterKey().Address(),
 		Sequence:  0,
 	}
-	realRPCVersion := ""
+
 	sqlLitePath := ""
 	if cfg != nil {
+		i.rpcContainerVersion = cfg.UseRealRPCVersion
 		i.historyArchiveProxyCallback = cfg.historyArchiveProxyCallback
 		i.protocolVersion = cfg.ProtocolVersion
-		realRPCVersion = cfg.UseRealRPCVersion
 		sqlLitePath = cfg.UseSQLitePath
 	}
 
@@ -118,12 +116,20 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 		proxy.ServeHTTP(w, r)
 	}))
 
+	rpcCfg := i.getRPConfig(sqlLitePath)
+	if i.rpcContainerVersion != "" {
+		i.rpcConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
+	}
 	i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color")
 	i.prepareShutdownHandlers()
 	i.coreClient = &stellarcore.Client{URL: "http://localhost:" + strconv.Itoa(stellarCorePort)}
 	i.waitForCore()
 	i.waitForCheckpoint()
-	i.launchRPC(coreBinaryPath, realRPCVersion, sqlLitePath)
+	if i.rpcContainerVersion == "" {
+		i.daemon = i.createDaemon(rpcCfg)
+		go i.daemon.Run()
+	}
+	i.waitForRPC()
 
 	return i
 }
@@ -165,30 +171,55 @@ func (i *Test) waitForCheckpoint() {
 	i.t.Fatal("Core could not reach checkpoint ledger after 30s")
 }
 
-func (i *Test) launchRPC(coreBinaryPath string, realRPCVersion string, sqlitePath string) {
-	var config config.Config
-	cmd := &cobra.Command{}
-	if err := config.AddFlags(cmd); err != nil {
-		i.t.FailNow()
-	}
-	if err := config.SetValues(func(string) (string, bool) { return "", false }); err != nil {
-		i.t.FailNow()
-	}
+func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 	if sqlitePath == "" {
 		sqlitePath = path.Join(i.t.TempDir(), "soroban_rpc.sqlite")
 	}
-	env := map[string]string{
-		"ENDPOINT":                       fmt.Sprintf("localhost:%d", sorobanRPCPort),
-		"ADMIN_ENDPOINT":                 fmt.Sprintf("localhost:%d", adminPort),
-		"STELLAR_CORE_URL":               "http://localhost:" + strconv.Itoa(stellarCorePort),
+
+	// Container default path
+	coreBinaryPath := "/usr/bin/stellar-core"
+	if i.rpcContainerVersion == "" {
+		coreBinaryPath = os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")
+		if coreBinaryPath == "" {
+			i.t.Fatal("missing SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")
+		}
+	}
+
+	// Out of the container default file
+	captiveCoreConfigPath := path.Join(i.composePath, "captive-core-integration-tests.cfg")
+	archiveProxyURL := i.historyArchiveProxy.URL
+	stellarCoreURL := fmt.Sprintf("http://localhost:%d", stellarCorePort)
+	bindHost := "localhost"
+	if i.rpcContainerVersion != "" {
+		// The file will be inside the container
+		captiveCoreConfigPath = "/stellar-core.cfg"
+		// the archive needs to be accessed from the container
+		url, err := url.Parse(i.historyArchiveProxy.URL)
+		require.NoError(i.t, err)
+		_, port, err := net.SplitHostPort(url.Host)
+		require.NoError(i.t, err)
+		url.Host = net.JoinHostPort("host.docker.internal", port)
+		archiveProxyURL = url.String()
+		// The container needs to listen on all interfaces, not just localhost
+		bindHost = "0.0.0.0"
+		// The container needs to use the sqlite mount point
+		i.rpcSQLiteMountDir = filepath.Dir(sqlitePath)
+		sqlitePath = "/db/" + filepath.Base(sqlitePath)
+		stellarCoreURL = fmt.Sprintf("http://core:%d", stellarCorePort)
+	}
+
+	return map[string]string{
+		"ENDPOINT":                       fmt.Sprintf("%s:%d", bindHost, sorobanRPCPort),
+		"ADMIN_ENDPOINT":                 fmt.Sprintf("%s:%d", bindHost, adminPort),
+		"STELLAR_CORE_URL":               stellarCoreURL,
 		"CORE_REQUEST_TIMEOUT":           "2s",
 		"STELLAR_CORE_BINARY_PATH":       coreBinaryPath,
-		"CAPTIVE_CORE_CONFIG_PATH":       path.Join(i.composePath, "captive-core-integration-tests.cfg"),
+		"CAPTIVE_CORE_CONFIG_PATH":       captiveCoreConfigPath,
 		"CAPTIVE_CORE_STORAGE_PATH":      i.t.TempDir(),
 		"STELLAR_CAPTIVE_CORE_HTTP_PORT": "0",
 		"FRIENDBOT_URL":                  friendbotURL,
 		"NETWORK_PASSPHRASE":             StandaloneNetworkPassphrase,
-		"HISTORY_ARCHIVE_URLS":           i.historyArchiveProxy.URL,
+		"HISTORY_ARCHIVE_URLS":           archiveProxyURL,
 		"LOG_LEVEL":                      "debug",
 		"DB_PATH":                        sqlitePath,
 		"INGESTION_TIMEOUT":              "10m",
@@ -198,75 +229,57 @@ func (i *Test) launchRPC(coreBinaryPath string, realRPCVersion string, sqlitePat
 		"MAX_HEALTHY_LEDGER_LATENCY":     "10s",
 		"PREFLIGHT_ENABLE_DEBUG":         "true",
 	}
+}
 
-	if realRPCVersion != "" {
-		i.rpcInstance.command = i.compileAndStartRPC(env, realRPCVersion)
-	} else {
-		i.rpcInstance.daemon = i.createDaemon(env)
-		go i.rpcInstance.daemon.Run()
-	}
+func (i *Test) waitForRPC() {
+	i.t.Log("Waiting for RPC to be up...")
 
-	// wait for the storage to catch up for 1 minute
-	info, err := i.coreClient.Info(context.Background())
-	if err != nil {
-		i.t.Fatalf("cannot obtain latest ledger from core: %v", err)
-	}
-	targetLedgerSequence := uint32(info.Info.Ledger.Num)
+	ch := jhttp.NewChannel(i.sorobanRPCURL(), nil)
+	client := jrpc2.NewClient(ch, nil)
+	defer client.Close()
 
-	dbConn, err := db.OpenSQLiteDB(sqlitePath)
-	require.NoError(i.t, err)
-	reader := db.NewLedgerEntryReader(dbConn)
+	var result methods.HealthCheckResult
 	success := false
-	for t := 30; t >= 0; t -= 1 {
-		sequence, err := reader.GetLatestLedgerSequence(context.Background())
-		if err != nil {
-			if err != db.ErrEmptyDB {
-				i.t.Fatalf("cannot access ledger entry storage: %v", err)
-			}
-		} else {
-			if sequence >= targetLedgerSequence {
+	for t := 30; t >= 0; t-- {
+		err := client.CallResult(context.Background(), "getHealth", nil, &result)
+		if err == nil {
+			if result.Status == "healthy" {
 				success = true
 				break
 			}
 		}
+		i.t.Log("RPC still unhealthy")
 		time.Sleep(time.Second)
 	}
 	if !success {
-		i.t.Fatal("LedgerEntryStorage failed to sync in 1 minute")
+		i.t.Fatal("RPC failed to get healthy in 30 seconds")
 	}
 }
 
-func (i *Test) compileAndStartRPC(env map[string]string, version string) *exec.Cmd {
-	newRPCDir := i.t.TempDir()
-
-	Command := func(name string, arg ...string) *exec.Cmd {
-		cmd := exec.Command(name, arg...)
-		cmd.Dir = newRPCDir
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		cmd.Stdin = os.Stdin
-		return cmd
-	}
+func (i *Test) createRPCContainerMountDir(rpcConfig map[string]string) string {
+	mountDir := i.t.TempDir()
+	// Get old version of captive-core-integration-tests.cfg
+	var out bytes.Buffer
+	cmd := exec.Command("git", "show", fmt.Sprintf("v%s:./captive-core-integration-tests.cfg", i.rpcContainerVersion))
+	cmd.Stdout = &out
+	cmd.Stderr = os.Stderr
+	cmd.Dir = i.composePath
+	require.NoError(i.t, cmd.Run())
 
-	// Clone
-	rootDir := path.Join(i.composePath, "..", "..", "..", "..")
-	err := Command("git", "clone", "--depth", "1", "--branch", version, "file://"+rootDir, newRPCDir).Run()
+	// replace ADDRESS="localhost" by ADDRESS="core", so that the container can find core
+	captiveCoreCfgContents := strings.Replace(out.String(), `ADDRESS="localhost"`, `ADDRESS="core"`, -1)
+	err := os.WriteFile(filepath.Join(mountDir, "stellar-core-integration-tests.cfg"), []byte(captiveCoreCfgContents), 0666)
 	require.NoError(i.t, err)
 
-	// Compile
-	cmd := Command("make", "build-libpreflight")
-	require.NoError(i.t, cmd.Run())
-	cmd = Command("go", "build", "./cmd/soroban-rpc")
-	require.NoError(i.t, cmd.Run())
-
-	// Run
-	cmd = Command(path.Join(newRPCDir, "soroban-rpc"))
-	for k, v := range env {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+	// Generate config file
+	cfgFileContents := ""
+	for k, v := range rpcConfig {
+		cfgFileContents += fmt.Sprintf("%s=%q\n", k, v)
 	}
-	require.NoError(i.t, cmd.Start())
+	err = os.WriteFile(filepath.Join(mountDir, "soroban-rpc.config"), []byte(cfgFileContents), 0666)
+	require.NoError(i.t, err)
 
-	return cmd
+	return mountDir
 }
 
 func (i *Test) createDaemon(env map[string]string) *daemon.Daemon {
@@ -284,17 +297,32 @@ func (i *Test) createDaemon(env map[string]string) *daemon.Daemon {
 // Runs a docker-compose command applied to the above configs
 func (i *Test) runComposeCommand(args ...string) {
 	integrationYaml := filepath.Join(i.composePath, "docker-compose.yml")
-
-	cmdline := append([]string{"-f", integrationYaml}, args...)
+	configFiles := []string{"-f", integrationYaml}
+	if i.rpcContainerVersion != "" {
+		rpcYaml := filepath.Join(i.composePath, "docker-compose.rpc.yml")
+		configFiles = append(configFiles, "-f", rpcYaml)
+	}
+	cmdline := append(configFiles, args...)
 	cmd := exec.Command("docker-compose", cmdline...)
 
 	if img := os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_DOCKER_IMG"); img != "" {
-		cmd.Env = os.Environ()
 		cmd.Env = append(
-			cmd.Environ(),
-			fmt.Sprintf("CORE_IMAGE=%s", img),
+			cmd.Env,
+			"CORE_IMAGE=%s"+img,
 		)
 	}
+	if i.rpcContainerVersion != "" {
+		cmd.Env = append(
+			cmd.Env,
+			"RPC_IMAGE_TAG="+i.rpcContainerVersion,
+			"RPC_CONFIG_MOUNT_DIR="+i.rpcConfigMountDir,
+			"RPC_SQLITE_MOUNT_DIR="+i.rpcSQLiteMountDir,
+		)
+	}
+	if len(cmd.Env) > 0 {
+		cmd.Env = append(cmd.Env, os.Environ()...)
+	}
+
 	i.t.Log("Running", cmd.Env, cmd.Args)
 	out, innerErr := cmd.Output()
 	if exitErr, ok := innerErr.(*exec.ExitError); ok {
@@ -310,12 +338,8 @@ func (i *Test) runComposeCommand(args ...string) {
 func (i *Test) prepareShutdownHandlers() {
 	i.shutdownCalls = append(i.shutdownCalls,
 		func() {
-			if i.rpcInstance.daemon != nil {
-				i.rpcInstance.daemon.Close()
-			}
-			if i.rpcInstance.command != nil {
-				require.NoError(i.t, i.rpcInstance.command.Process.Kill())
-				i.rpcInstance.command.Wait()
+			if i.daemon != nil {
+				i.daemon.Close()
 			}
 			if i.historyArchiveProxy != nil {
 				i.historyArchiveProxy.Close()
@@ -475,11 +499,11 @@ func findDockerComposePath() string {
 func GetCoreMaxSupportedProtocol() uint32 {
 	str := os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL")
 	if str == "" {
-		return maxSupportedProtocolVersion
+		return MaxSupportedProtocolVersion
 	}
 	version, err := strconv.ParseUint(str, 10, 32)
 	if err != nil {
-		return maxSupportedProtocolVersion
+		return MaxSupportedProtocolVersion
 	}
 
 	return uint32(version)
diff --git a/cmd/soroban-rpc/internal/test/metrics_test.go b/cmd/soroban-rpc/internal/test/metrics_test.go
index eaac747a..57211ad1 100644
--- a/cmd/soroban-rpc/internal/test/metrics_test.go
+++ b/cmd/soroban-rpc/internal/test/metrics_test.go
@@ -30,12 +30,12 @@ func TestMetrics(t *testing.T) {
 	)
 	require.Contains(t, metrics, buildMetric)
 
-	logger := test.rpcInstance.daemon.Logger()
+	logger := test.daemon.Logger()
 	err := errors.Errorf("test-error")
 	logger.WithError(err).Error("test error 1")
 	logger.WithError(err).Error("test error 2")
 
-	metricFamilies, err := test.rpcInstance.daemon.MetricsRegistry().Gather()
+	metricFamilies, err := test.daemon.MetricsRegistry().Gather()
 	assert.NoError(t, err)
 	var metric *io_prometheus_client.MetricFamily
 	for _, mf := range metricFamilies {
diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
new file mode 100644
index 00000000..8b4f54c9
--- /dev/null
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -0,0 +1,125 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/creachadair/jrpc2"
+	"github.com/creachadair/jrpc2/jhttp"
+	"github.com/stellar/go/keypair"
+	"github.com/stellar/go/txnbuild"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
+)
+
+// Test that every Soroban RPC version (within the current protocol) can migrate cleanly to the current version
+// We cannot test prior protocol versions since the Transaction XDR used for the test would be incompatible
+func TestMigrate(t *testing.T) {
+	if GetCoreMaxSupportedProtocol() != MaxSupportedProtocolVersion {
+		t.Skip("Only test this for the latest protocol: ", MaxSupportedProtocolVersion)
+	}
+
+	for _, originVersion := range getCurrentProtocolReleaseVersions(t) {
+		t.Run(originVersion, func(t *testing.T) {
+			testMigrateFromVersion(t, originVersion)
+		})
+	}
+
+}
+
+func testMigrateFromVersion(t *testing.T, version string) {
+	sqliteFile := filepath.Join(t.TempDir(), "soroban-rpc.db")
+	it := NewTest(t, &TestConfig{
+		UseRealRPCVersion: version,
+		UseSQLitePath:     sqliteFile,
+	})
+
+	ch := jhttp.NewChannel(it.sorobanRPCURL(), nil)
+	client := jrpc2.NewClient(ch, nil)
+
+	// Submit an event-logging transaction in the version to migrate from
+	kp := keypair.Root(StandaloneNetworkPassphrase)
+	address := kp.Address()
+	account := txnbuild.NewSimpleAccount(address, 0)
+
+	contractBinary := getHelloWorldContract(t)
+	params := preflightTransactionParams(t, client, txnbuild.TransactionParams{
+		SourceAccount:        &account,
+		IncrementSequenceNum: true,
+		Operations: []txnbuild.Operation{
+			createInstallContractCodeOperation(account.AccountID, contractBinary),
+		},
+		BaseFee: txnbuild.MinBaseFee,
+		Preconditions: txnbuild.Preconditions{
+			TimeBounds: txnbuild.NewInfiniteTimeout(),
+		},
+	})
+	tx, err := txnbuild.NewTransaction(params)
+	assert.NoError(t, err)
+	submitTransactionResponse := sendSuccessfulTransaction(t, client, kp, tx)
+
+	// Run another test with the current RPC, but the previous network and sql database (causing a data migration)
+	// TODO: create a dedicated method
+	it.runComposeCommand("down", "rpc", "-v")
+	it = NewTest(t, &TestConfig{UseSQLitePath: sqliteFile})
+
+	// make sure that the instance is healthy
+	var healthResult methods.HealthCheckResult
+	err = client.CallResult(context.Background(), "getHealth", nil, &healthResult)
+	require.NoError(t, err)
+	require.Equal(t, "healthy", healthResult.Status)
+
+	// make sure that the transaction submitted before and its events exist
+	var transactionsResult methods.GetTransactionsResponse
+	getTransactions := methods.GetTransactionsRequest{
+		StartLedger: submitTransactionResponse.Ledger,
+		Pagination: &methods.TransactionsPaginationOptions{
+			Limit: 1,
+		},
+	}
+	err = client.CallResult(context.Background(), "getTransactions", getTransactions, &transactionsResult)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(transactionsResult.Transactions))
+	require.Equal(t, submitTransactionResponse.Ledger, transactionsResult.Transactions[0].Ledger)
+
+	var eventsResult methods.GetEventsResponse
+	getEventsRequest := methods.GetEventsRequest{
+		StartLedger: submitTransactionResponse.Ledger,
+		Pagination: &methods.PaginationOptions{
+			Limit: 1,
+		},
+	}
+	err = client.CallResult(context.Background(), "getEvents", getEventsRequest, &eventsResult)
+	require.NoError(t, err)
+	require.Equal(t, len(eventsResult.Events), 1)
+	require.Equal(t, submitTransactionResponse.Ledger, uint32(eventsResult.Events[0].Ledger))
+}
+
+func getCurrentProtocolReleaseVersions(t *testing.T) []string {
+	protocolStr := strconv.Itoa(MaxSupportedProtocolVersion)
+	_, currentFilename, _, _ := runtime.Caller(0)
+	currentDir := filepath.Dir(currentFilename)
+	var out bytes.Buffer
+	cmd := exec.Command("git", "tag")
+	cmd.Dir = currentDir
+	cmd.Stdout = &out
+	cmd.Stderr = os.Stderr
+	require.NoError(t, cmd.Run())
+	tags := strings.Split(out.String(), "\n")
+	filteredTags := make([]string, 0, len(tags))
+	for _, tag := range tags {
+		if strings.HasPrefix(tag, "v"+protocolStr) {
+			filteredTags = append(filteredTags, tag[1:])
+		}
+	}
+	return filteredTags
+}
diff --git a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
index 61ee8e4f..318142e1 100644
--- a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
+++ b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
@@ -124,14 +124,14 @@ func simulateTransactionFromTxParams(t *testing.T, client *jrpc2.Client, params
 	savedAutoIncrement := params.IncrementSequenceNum
 	params.IncrementSequenceNum = false
 	tx, err := txnbuild.NewTransaction(params)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	params.IncrementSequenceNum = savedAutoIncrement
 	txB64, err := tx.Base64()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	request := methods.SimulateTransactionRequest{Transaction: txB64}
 	var response methods.SimulateTransactionResponse
 	err = client.CallResult(context.Background(), "simulateTransaction", request, &response)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	return response
 }
 
@@ -563,7 +563,7 @@ func TestSimulateInvokeContractTransactionSucceeds(t *testing.T) {
 }
 
 func TestSimulateTransactionError(t *testing.T) {
-	test := NewTest(t, &TestConfig{UseRealRPCVersion: "main"})
+	test := NewTest(t, nil)
 
 	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
 	client := jrpc2.NewClient(ch, nil)
diff --git a/cmd/soroban-rpc/internal/test/upgrade_test.go b/cmd/soroban-rpc/internal/test/upgrade_test.go
index fdc0c95b..6889e614 100644
--- a/cmd/soroban-rpc/internal/test/upgrade_test.go
+++ b/cmd/soroban-rpc/internal/test/upgrade_test.go
@@ -53,7 +53,7 @@ func TestUpgradeFrom20To21(t *testing.T) {
 	// estimations
 	test.UpgradeProtocol(21)
 	// Wait for the ledger to advance, so that the simulation library passes the right protocol number
-	rpcDB := test.rpcInstance.daemon.GetDB()
+	rpcDB := test.daemon.GetDB()
 	initialLedgerSequence, err := db.NewLedgerEntryReader(rpcDB).GetLatestLedgerSequence(context.Background())
 	require.Eventually(t,
 		func() bool {

From 4f90b57b1ecbfc737fa2b4045a9d3cf229b803a0 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 06:21:47 +0200
Subject: [PATCH 14/36] blacklist certain docker versions

---
 cmd/soroban-rpc/internal/test/migrate_test.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index 8b4f54c9..f9b40595 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -27,8 +27,16 @@ func TestMigrate(t *testing.T) {
 	if GetCoreMaxSupportedProtocol() != MaxSupportedProtocolVersion {
 		t.Skip("Only test this for the latest protocol: ", MaxSupportedProtocolVersion)
 	}
-
 	for _, originVersion := range getCurrentProtocolReleaseVersions(t) {
+		if originVersion == "21.1.0" {
+			// This version of the RPC container fails to even start with its captive core companion file
+			// (it fails Invalid configuration: DEPRECATED_SQL_LEDGER_STATE not set.)
+			continue
+		}
+		if originVersion == "21.3.0" {
+			// This version of RPC wasn't published as a docker container
+			continue
+		}
 		t.Run(originVersion, func(t *testing.T) {
 			testMigrateFromVersion(t, originVersion)
 		})

From 567dd0d7cc642a23ed5a899a19cdc56cdfb26abc Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 06:24:32 +0200
Subject: [PATCH 15/36] Tweak comment

---
 cmd/soroban-rpc/internal/test/migrate_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index f9b40595..6963a594 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -75,7 +75,7 @@ func testMigrateFromVersion(t *testing.T, version string) {
 	assert.NoError(t, err)
 	submitTransactionResponse := sendSuccessfulTransaction(t, client, kp, tx)
 
-	// Run another test with the current RPC, but the previous network and sql database (causing a data migration)
+	// Check the transaction with current RPC, but the previous network and sql database (causing a data migration)
 	// TODO: create a dedicated method
 	it.runComposeCommand("down", "rpc", "-v")
 	it = NewTest(t, &TestConfig{UseSQLitePath: sqliteFile})

From 7f849905eb310299c5109aea42dfded4356306f3 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 06:31:13 +0200
Subject: [PATCH 16/36] Fix bug setting core image

---
 cmd/soroban-rpc/internal/test/integration.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index c9f0cb8a..669bcc33 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -308,7 +308,7 @@ func (i *Test) runComposeCommand(args ...string) {
 	if img := os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_DOCKER_IMG"); img != "" {
 		cmd.Env = append(
 			cmd.Env,
-			"CORE_IMAGE=%s"+img,
+			"CORE_IMAGE="+img,
 		)
 	}
 	if i.rpcContainerVersion != "" {

From c1620d256984d4e5f96bc9a3bd14811b73250bfd Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 06:32:42 +0200
Subject: [PATCH 17/36] Tweak comment

---
 cmd/soroban-rpc/internal/test/migrate_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index 6963a594..10d37fa1 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -22,7 +22,7 @@ import (
 )
 
 // Test that every Soroban RPC version (within the current protocol) can migrate cleanly to the current version
-// We cannot test prior protocol versions since the Transaction XDR used for the test would be incompatible
+// We cannot test prior protocol versions since the Transaction XDR used for the test could be incompatible
 func TestMigrate(t *testing.T) {
 	if GetCoreMaxSupportedProtocol() != MaxSupportedProtocolVersion {
 		t.Skip("Only test this for the latest protocol: ", MaxSupportedProtocolVersion)

From 670d3d831cc0c3a4e325548175007848b1b8cd61 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 06:49:04 +0200
Subject: [PATCH 18/36] Multiple cleanups

---
 cmd/soroban-rpc/internal/test/integration.go  | 35 ++++++++++++-------
 cmd/soroban-rpc/internal/test/migrate_test.go | 22 +++++-------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 669bcc33..e24d8539 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -53,8 +53,9 @@ const (
 type TestConfig struct {
 	historyArchiveProxyCallback func(*http.Request)
 	ProtocolVersion             uint32
-	UseRealRPCVersion           string
-	UseSQLitePath               string
+	// Run a previously released version of RPC (in a container) instead of the current version
+	UseReleasedRPCVersion string
+	UseSQLitePath         string
 }
 
 type Test struct {
@@ -64,9 +65,9 @@ type Test struct {
 
 	protocolVersion uint32
 
-	rpcContainerVersion string
-	rpcConfigMountDir   string
-	rpcSQLiteMountDir   string
+	rpcContainerVersion        string
+	rpcContainerConfigMountDir string
+	rpcContainerSQLiteMountDir string
 
 	daemon *daemon.Daemon
 
@@ -96,7 +97,7 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 
 	sqlLitePath := ""
 	if cfg != nil {
-		i.rpcContainerVersion = cfg.UseRealRPCVersion
+		i.rpcContainerVersion = cfg.UseReleasedRPCVersion
 		i.historyArchiveProxyCallback = cfg.historyArchiveProxyCallback
 		i.protocolVersion = cfg.ProtocolVersion
 		sqlLitePath = cfg.UseSQLitePath
@@ -118,7 +119,7 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 
 	rpcCfg := i.getRPConfig(sqlLitePath)
 	if i.rpcContainerVersion != "" {
-		i.rpcConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
+		i.rpcContainerConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
 	}
 	i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color")
 	i.prepareShutdownHandlers()
@@ -203,7 +204,7 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 		// The container needs to listen on all interfaces, not just localhost
 		bindHost = "0.0.0.0"
 		// The container needs to use the sqlite mount point
-		i.rpcSQLiteMountDir = filepath.Dir(sqlitePath)
+		i.rpcContainerSQLiteMountDir = filepath.Dir(sqlitePath)
 		sqlitePath = "/db/" + filepath.Base(sqlitePath)
 		stellarCoreURL = fmt.Sprintf("http://core:%d", stellarCorePort)
 	}
@@ -315,8 +316,8 @@ func (i *Test) runComposeCommand(args ...string) {
 		cmd.Env = append(
 			cmd.Env,
 			"RPC_IMAGE_TAG="+i.rpcContainerVersion,
-			"RPC_CONFIG_MOUNT_DIR="+i.rpcConfigMountDir,
-			"RPC_SQLITE_MOUNT_DIR="+i.rpcSQLiteMountDir,
+			"RPC_CONFIG_MOUNT_DIR="+i.rpcContainerConfigMountDir,
+			"RPC_SQLITE_MOUNT_DIR="+i.rpcContainerSQLiteMountDir,
 		)
 	}
 	if len(cmd.Env) > 0 {
@@ -338,9 +339,7 @@ func (i *Test) runComposeCommand(args ...string) {
 func (i *Test) prepareShutdownHandlers() {
 	i.shutdownCalls = append(i.shutdownCalls,
 		func() {
-			if i.daemon != nil {
-				i.daemon.Close()
-			}
+			i.StopRPC()
 			if i.historyArchiveProxy != nil {
 				i.historyArchiveProxy.Close()
 			}
@@ -435,6 +434,16 @@ func (i *Test) UpgradeProtocol(version uint32) {
 	i.t.Fatalf("could not upgrade protocol in 10s")
 }
 
+func (i *Test) StopRPC() {
+	if i.daemon != nil {
+		i.daemon.Close()
+		i.daemon = nil
+	}
+	if i.rpcContainerVersion != "" {
+		i.runComposeCommand("down", "rpc", "-v")
+	}
+}
+
 // Cluttering code with if err != nil is absolute nonsense.
 func panicIf(err error) {
 	if err != nil {
diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index 10d37fa1..d7d565cf 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -23,11 +23,12 @@ import (
 
 // Test that every Soroban RPC version (within the current protocol) can migrate cleanly to the current version
 // We cannot test prior protocol versions since the Transaction XDR used for the test could be incompatible
+// TODO: find a way to test migrations between protocols
 func TestMigrate(t *testing.T) {
 	if GetCoreMaxSupportedProtocol() != MaxSupportedProtocolVersion {
 		t.Skip("Only test this for the latest protocol: ", MaxSupportedProtocolVersion)
 	}
-	for _, originVersion := range getCurrentProtocolReleaseVersions(t) {
+	for _, originVersion := range getCurrentProtocolReleasedVersions(t) {
 		if originVersion == "21.1.0" {
 			// This version of the RPC container fails to even start with its captive core companion file
 			// (it fails Invalid configuration: DEPRECATED_SQL_LEDGER_STATE not set.)
@@ -47,8 +48,8 @@ func TestMigrate(t *testing.T) {
 func testMigrateFromVersion(t *testing.T, version string) {
 	sqliteFile := filepath.Join(t.TempDir(), "soroban-rpc.db")
 	it := NewTest(t, &TestConfig{
-		UseRealRPCVersion: version,
-		UseSQLitePath:     sqliteFile,
+		UseReleasedRPCVersion: version,
+		UseSQLitePath:         sqliteFile,
 	})
 
 	ch := jhttp.NewChannel(it.sorobanRPCURL(), nil)
@@ -75,18 +76,11 @@ func testMigrateFromVersion(t *testing.T, version string) {
 	assert.NoError(t, err)
 	submitTransactionResponse := sendSuccessfulTransaction(t, client, kp, tx)
 
-	// Check the transaction with current RPC, but the previous network and sql database (causing a data migration)
-	// TODO: create a dedicated method
-	it.runComposeCommand("down", "rpc", "-v")
+	// Run the current RPC version, but the previous network and sql database (causing a data migration if needed)
+	it.StopRPC()
 	it = NewTest(t, &TestConfig{UseSQLitePath: sqliteFile})
 
-	// make sure that the instance is healthy
-	var healthResult methods.HealthCheckResult
-	err = client.CallResult(context.Background(), "getHealth", nil, &healthResult)
-	require.NoError(t, err)
-	require.Equal(t, "healthy", healthResult.Status)
-
-	// make sure that the transaction submitted before and its events exist
+	// make sure that the transaction submitted before and its events exist in current RPC
 	var transactionsResult methods.GetTransactionsResponse
 	getTransactions := methods.GetTransactionsRequest{
 		StartLedger: submitTransactionResponse.Ledger,
@@ -112,7 +106,7 @@ func testMigrateFromVersion(t *testing.T, version string) {
 	require.Equal(t, submitTransactionResponse.Ledger, uint32(eventsResult.Events[0].Ledger))
 }
 
-func getCurrentProtocolReleaseVersions(t *testing.T) []string {
+func getCurrentProtocolReleasedVersions(t *testing.T) []string {
 	protocolStr := strconv.Itoa(MaxSupportedProtocolVersion)
 	_, currentFilename, _, _ := runtime.Caller(0)
 	currentDir := filepath.Dir(currentFilename)

From d922aff0ba59de9d7a869cdd20bcdbf68a7c51b1 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 14:11:52 +0200
Subject: [PATCH 19/36] Debug why RPC keeps unhealthy in CI

---
 cmd/soroban-rpc/internal/test/integration.go  | 2 +-
 cmd/soroban-rpc/internal/test/migrate_test.go | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index e24d8539..d63d1b3c 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -249,7 +249,7 @@ func (i *Test) waitForRPC() {
 				break
 			}
 		}
-		i.t.Log("RPC still unhealthy")
+		i.t.Log("RPC still unhealthy", err, result.Status)
 		time.Sleep(time.Second)
 	}
 	if !success {
diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index d7d565cf..eded0e61 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -25,6 +25,7 @@ import (
 // We cannot test prior protocol versions since the Transaction XDR used for the test could be incompatible
 // TODO: find a way to test migrations between protocols
 func TestMigrate(t *testing.T) {
+	t.Skip("see if it works when we skip this test")
 	if GetCoreMaxSupportedProtocol() != MaxSupportedProtocolVersion {
 		t.Skip("Only test this for the latest protocol: ", MaxSupportedProtocolVersion)
 	}

From 9e581ab7be3a08bc8ab06d6070310044592058c7 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 15:33:39 +0200
Subject: [PATCH 20/36] Print backtrace when RPC fails

---
 cmd/soroban-rpc/internal/test/integration.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index d63d1b3c..7f1efd41 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"log"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -14,6 +15,7 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -233,28 +235,30 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 }
 
 func (i *Test) waitForRPC() {
-	i.t.Log("Waiting for RPC to be up...")
+	i.t.Log("Waiting for RPC to be healthy...")
 
 	ch := jhttp.NewChannel(i.sorobanRPCURL(), nil)
 	client := jrpc2.NewClient(ch, nil)
 	defer client.Close()
 
 	var result methods.HealthCheckResult
-	success := false
 	for t := 30; t >= 0; t-- {
 		err := client.CallResult(context.Background(), "getHealth", nil, &result)
 		if err == nil {
 			if result.Status == "healthy" {
-				success = true
-				break
+				i.t.Log("RPC is healthy")
+				return
 			}
 		}
 		i.t.Log("RPC still unhealthy", err, result.Status)
 		time.Sleep(time.Second)
 	}
-	if !success {
-		i.t.Fatal("RPC failed to get healthy in 30 seconds")
-	}
+
+	// Print stack trace and fail
+	buf := make([]byte, 1<<16)
+	stackSize := runtime.Stack(buf, true)
+	log.Printf("%s\n", string(buf[0:stackSize]))
+	i.t.Fatal("RPC failed to get healthy in 30 seconds")
 }
 
 func (i *Test) createRPCContainerMountDir(rpcConfig map[string]string) string {

From c624ce17faf585b169705ce34bedd90138eca734 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 16:04:27 +0200
Subject: [PATCH 21/36] Increase timeout

---
 cmd/soroban-rpc/internal/test/integration.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 7f1efd41..eec2775b 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -242,7 +242,7 @@ func (i *Test) waitForRPC() {
 	defer client.Close()
 
 	var result methods.HealthCheckResult
-	for t := 30; t >= 0; t-- {
+	for t := 120; t >= 0; t-- {
 		err := client.CallResult(context.Background(), "getHealth", nil, &result)
 		if err == nil {
 			if result.Status == "healthy" {
@@ -258,7 +258,7 @@ func (i *Test) waitForRPC() {
 	buf := make([]byte, 1<<16)
 	stackSize := runtime.Stack(buf, true)
 	log.Printf("%s\n", string(buf[0:stackSize]))
-	i.t.Fatal("RPC failed to get healthy in 30 seconds")
+	i.t.Fatal("RPC failed to get healthy in 120 seconds")
 }
 
 func (i *Test) createRPCContainerMountDir(rpcConfig map[string]string) string {

From 8908c6256113944dcc95fa55dca356e593615cfc Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 16:45:03 +0200
Subject: [PATCH 22/36] Fix goroutine leak

---
 cmd/soroban-rpc/internal/test/integration.go | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index eec2775b..99d74f3f 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -341,8 +341,10 @@ func (i *Test) runComposeCommand(args ...string) {
 }
 
 func (i *Test) prepareShutdownHandlers() {
+	done := make(chan struct{})
 	i.shutdownCalls = append(i.shutdownCalls,
 		func() {
+			close(done)
 			i.StopRPC()
 			if i.historyArchiveProxy != nil {
 				i.historyArchiveProxy.Close()
@@ -358,9 +360,12 @@ func (i *Test) prepareShutdownHandlers() {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	go func() {
-		<-c
-		i.Shutdown()
-		os.Exit(int(syscall.SIGTERM))
+		select {
+		case <-c:
+			i.Shutdown()
+			os.Exit(int(syscall.SIGTERM))
+		case <-done:
+		}
 	}()
 }
 

From ce7194d60e73378d4e2316ebb8b0430e94453157 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 17:04:40 +0200
Subject: [PATCH 23/36] Remove JSONRPC client leaks

---
 .../network/requestdurationlimiter_test.go    |  3 ++
 .../internal/test/get_fee_stats_test.go       |  6 +---
 .../internal/test/get_ledger_entries_test.go  | 10 ++----
 .../internal/test/get_ledger_entry_test.go    | 10 ++----
 .../internal/test/get_network_test.go         |  5 +--
 .../internal/test/get_transactions_test.go    |  4 +--
 .../internal/test/get_version_info_test.go    |  8 ++---
 cmd/soroban-rpc/internal/test/health_test.go  |  5 +--
 cmd/soroban-rpc/internal/test/integration.go  | 15 ++++++---
 cmd/soroban-rpc/internal/test/migrate_test.go |  5 +--
 .../test/simulate_transaction_test.go         | 32 ++++++-------------
 .../internal/test/transaction_test.go         | 19 ++++-------
 cmd/soroban-rpc/internal/test/upgrade_test.go |  5 +--
 13 files changed, 44 insertions(+), 83 deletions(-)

diff --git a/cmd/soroban-rpc/internal/network/requestdurationlimiter_test.go b/cmd/soroban-rpc/internal/network/requestdurationlimiter_test.go
index 5be64cea..5bc3a40f 100644
--- a/cmd/soroban-rpc/internal/network/requestdurationlimiter_test.go
+++ b/cmd/soroban-rpc/internal/network/requestdurationlimiter_test.go
@@ -207,6 +207,7 @@ func TestJRPCRequestDurationLimiter_Limiting(t *testing.T) {
 
 	ch := jhttp.NewChannel("http://"+addr+"/", nil)
 	client := jrpc2.NewClient(ch, nil)
+	defer client.Close()
 
 	var res interface{}
 	req := struct {
@@ -251,6 +252,7 @@ func TestJRPCRequestDurationLimiter_NoLimiting(t *testing.T) {
 
 	ch := jhttp.NewChannel("http://"+addr+"/", nil)
 	client := jrpc2.NewClient(ch, nil)
+	defer client.Close()
 
 	var res interface{}
 	req := struct {
@@ -292,6 +294,7 @@ func TestJRPCRequestDurationLimiter_NoLimiting_Warn(t *testing.T) {
 
 	ch := jhttp.NewChannel("http://"+addr+"/", nil)
 	client := jrpc2.NewClient(ch, nil)
+	defer client.Close()
 
 	var res interface{}
 	req := struct {
diff --git a/cmd/soroban-rpc/internal/test/get_fee_stats_test.go b/cmd/soroban-rpc/internal/test/get_fee_stats_test.go
index b1de24e7..15969aea 100644
--- a/cmd/soroban-rpc/internal/test/get_fee_stats_test.go
+++ b/cmd/soroban-rpc/internal/test/get_fee_stats_test.go
@@ -4,8 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stellar/go/xdr"
@@ -18,9 +16,7 @@ import (
 func TestGetFeeStats(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
-
+	client := test.GetRPCLient()
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
 	account := txnbuild.NewSimpleAccount(address, 0)
diff --git a/cmd/soroban-rpc/internal/test/get_ledger_entries_test.go b/cmd/soroban-rpc/internal/test/get_ledger_entries_test.go
index 50c0af9e..835f183f 100644
--- a/cmd/soroban-rpc/internal/test/get_ledger_entries_test.go
+++ b/cmd/soroban-rpc/internal/test/get_ledger_entries_test.go
@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -20,8 +19,7 @@ import (
 func TestGetLedgerEntriesNotFound(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address()
 	contractID := getContractID(t, sourceAccount, testSalt, StandaloneNetworkPassphrase)
@@ -58,8 +56,7 @@ func TestGetLedgerEntriesNotFound(t *testing.T) {
 func TestGetLedgerEntriesInvalidParams(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	var keys []string
 	keys = append(keys, "<>@@#$")
@@ -76,8 +73,7 @@ func TestGetLedgerEntriesInvalidParams(t *testing.T) {
 func TestGetLedgerEntriesSucceeds(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
diff --git a/cmd/soroban-rpc/internal/test/get_ledger_entry_test.go b/cmd/soroban-rpc/internal/test/get_ledger_entry_test.go
index df606bfc..007dd0f2 100644
--- a/cmd/soroban-rpc/internal/test/get_ledger_entry_test.go
+++ b/cmd/soroban-rpc/internal/test/get_ledger_entry_test.go
@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -20,8 +19,7 @@ import (
 func TestGetLedgerEntryNotFound(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address()
 	contractID := getContractID(t, sourceAccount, testSalt, StandaloneNetworkPassphrase)
@@ -53,8 +51,7 @@ func TestGetLedgerEntryNotFound(t *testing.T) {
 func TestGetLedgerEntryInvalidParams(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	request := methods.GetLedgerEntryRequest{
 		Key: "<>@@#$",
@@ -69,8 +66,7 @@ func TestGetLedgerEntryInvalidParams(t *testing.T) {
 func TestGetLedgerEntrySucceeds(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	account := txnbuild.NewSimpleAccount(kp.Address(), 0)
diff --git a/cmd/soroban-rpc/internal/test/get_network_test.go b/cmd/soroban-rpc/internal/test/get_network_test.go
index dad90771..777a48e4 100644
--- a/cmd/soroban-rpc/internal/test/get_network_test.go
+++ b/cmd/soroban-rpc/internal/test/get_network_test.go
@@ -4,8 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
@@ -14,8 +12,7 @@ import (
 func TestGetNetworkSucceeds(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	request := methods.GetNetworkRequest{}
 
diff --git a/cmd/soroban-rpc/internal/test/get_transactions_test.go b/cmd/soroban-rpc/internal/test/get_transactions_test.go
index f3da92dd..48371cea 100644
--- a/cmd/soroban-rpc/internal/test/get_transactions_test.go
+++ b/cmd/soroban-rpc/internal/test/get_transactions_test.go
@@ -5,7 +5,6 @@ import (
 	"testing"
 
 	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stretchr/testify/assert"
@@ -57,8 +56,7 @@ func sendTransactions(t *testing.T, client *jrpc2.Client) []uint32 {
 
 func TestGetTransactions(t *testing.T) {
 	test := NewTest(t, nil)
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	ledgers := sendTransactions(t, client)
 
diff --git a/cmd/soroban-rpc/internal/test/get_version_info_test.go b/cmd/soroban-rpc/internal/test/get_version_info_test.go
index 9f406b8d..25a43f62 100644
--- a/cmd/soroban-rpc/internal/test/get_version_info_test.go
+++ b/cmd/soroban-rpc/internal/test/get_version_info_test.go
@@ -3,12 +3,11 @@ package test
 import (
 	"context"
 	"fmt"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"os/exec"
 	"testing"
 
-	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
@@ -28,8 +27,7 @@ func TestGetVersionInfoSucceeds(t *testing.T) {
 		config.BuildTimestamp = buildTimeStamp
 	})
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	var result methods.GetVersionInfoResponse
 	err := client.CallResult(context.Background(), "getVersionInfo", nil, &result)
diff --git a/cmd/soroban-rpc/internal/test/health_test.go b/cmd/soroban-rpc/internal/test/health_test.go
index 0840959c..adae006c 100644
--- a/cmd/soroban-rpc/internal/test/health_test.go
+++ b/cmd/soroban-rpc/internal/test/health_test.go
@@ -4,8 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
@@ -15,8 +13,7 @@ import (
 func TestHealth(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	var result methods.HealthCheckResult
 	if err := client.CallResult(context.Background(), "getHealth", nil, &result); err != nil {
diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 99d74f3f..5011eb71 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -81,6 +81,7 @@ type Test struct {
 	masterAccount txnbuild.Account
 	shutdownOnce  sync.Once
 	shutdownCalls []func()
+	rpcClient     *jrpc2.Client
 }
 
 func NewTest(t *testing.T, cfg *TestConfig) *Test {
@@ -119,6 +120,8 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 		proxy.ServeHTTP(w, r)
 	}))
 
+	ch := jhttp.NewChannel(i.sorobanRPCURL(), nil)
+	i.rpcClient = jrpc2.NewClient(ch, nil)
 	rpcCfg := i.getRPConfig(sqlLitePath)
 	if i.rpcContainerVersion != "" {
 		i.rpcContainerConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
@@ -137,6 +140,9 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 	return i
 }
 
+func (i *Test) GetRPCLient() *jrpc2.Client {
+	return i.rpcClient
+}
 func (i *Test) MasterKey() *keypair.Full {
 	return keypair.Root(StandaloneNetworkPassphrase)
 }
@@ -237,13 +243,9 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 func (i *Test) waitForRPC() {
 	i.t.Log("Waiting for RPC to be healthy...")
 
-	ch := jhttp.NewChannel(i.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
-	defer client.Close()
-
 	var result methods.HealthCheckResult
 	for t := 120; t >= 0; t-- {
-		err := client.CallResult(context.Background(), "getHealth", nil, &result)
+		err := i.rpcClient.CallResult(context.Background(), "getHealth", nil, &result)
 		if err == nil {
 			if result.Status == "healthy" {
 				i.t.Log("RPC is healthy")
@@ -349,6 +351,9 @@ func (i *Test) prepareShutdownHandlers() {
 			if i.historyArchiveProxy != nil {
 				i.historyArchiveProxy.Close()
 			}
+			if i.rpcClient != nil {
+				i.rpcClient.Close()
+			}
 			i.runComposeCommand("down", "-v")
 		},
 	)
diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index eded0e61..c99f74d8 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -11,8 +11,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stretchr/testify/assert"
@@ -53,8 +51,7 @@ func testMigrateFromVersion(t *testing.T, version string) {
 		UseSQLitePath:         sqliteFile,
 	})
 
-	ch := jhttp.NewChannel(it.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := it.GetRPCLient()
 
 	// Submit an event-logging transaction in the version to migrate from
 	kp := keypair.Root(StandaloneNetworkPassphrase)
diff --git a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
index 318142e1..642c7937 100644
--- a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
+++ b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go
@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -191,9 +190,7 @@ func preflightTransactionParams(t *testing.T, client *jrpc2.Client, params txnbu
 func TestSimulateTransactionSucceeds(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
-
+	client := test.GetRPCLient()
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address()
 	contractBinary := getHelloWorldContract(t)
 	params := txnbuild.TransactionParams{
@@ -323,8 +320,7 @@ func TestSimulateTransactionSucceeds(t *testing.T) {
 func TestSimulateTransactionWithAuth(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
@@ -381,8 +377,7 @@ func TestSimulateTransactionWithAuth(t *testing.T) {
 func TestSimulateInvokeContractTransactionSucceeds(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
@@ -565,8 +560,7 @@ func TestSimulateInvokeContractTransactionSucceeds(t *testing.T) {
 func TestSimulateTransactionError(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address()
 	invokeHostOp := createInvokeHostOperation(sourceAccount, xdr.Hash{}, "noMethod")
@@ -605,8 +599,7 @@ func TestSimulateTransactionError(t *testing.T) {
 func TestSimulateTransactionMultipleOperations(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address()
 	contractBinary := getHelloWorldContract(t)
@@ -640,8 +633,7 @@ func TestSimulateTransactionMultipleOperations(t *testing.T) {
 func TestSimulateTransactionWithoutInvokeHostFunction(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	params := txnbuild.TransactionParams{
 		SourceAccount: &txnbuild.SimpleAccount{
@@ -671,8 +663,7 @@ func TestSimulateTransactionWithoutInvokeHostFunction(t *testing.T) {
 func TestSimulateTransactionUnmarshalError(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	request := methods.SimulateTransactionRequest{Transaction: "invalid"}
 	var result methods.SimulateTransactionResponse
@@ -688,8 +679,7 @@ func TestSimulateTransactionUnmarshalError(t *testing.T) {
 func TestSimulateTransactionExtendAndRestoreFootprint(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
@@ -921,8 +911,7 @@ func waitUntilLedgerEntryTTL(t *testing.T, client *jrpc2.Client, ledgerKey xdr.L
 func TestSimulateInvokePrng_u64_in_range(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
@@ -1032,8 +1021,7 @@ func TestSimulateInvokePrng_u64_in_range(t *testing.T) {
 func TestSimulateSystemEvent(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()
diff --git a/cmd/soroban-rpc/internal/test/transaction_test.go b/cmd/soroban-rpc/internal/test/transaction_test.go
index f838ad6b..4a8461f6 100644
--- a/cmd/soroban-rpc/internal/test/transaction_test.go
+++ b/cmd/soroban-rpc/internal/test/transaction_test.go
@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -23,8 +22,7 @@ import (
 func TestSendTransactionSucceedsWithoutResults(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	address := kp.Address()
@@ -48,8 +46,7 @@ func TestSendTransactionSucceedsWithoutResults(t *testing.T) {
 func TestSendTransactionSucceedsWithResults(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	address := kp.Address()
@@ -112,8 +109,7 @@ func TestSendTransactionSucceedsWithResults(t *testing.T) {
 func TestSendTransactionBadSequence(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	address := kp.Address()
@@ -154,8 +150,7 @@ func TestSendTransactionBadSequence(t *testing.T) {
 func TestSendTransactionFailedInsufficientResourceFee(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	address := kp.Address()
@@ -206,8 +201,7 @@ func TestSendTransactionFailedInsufficientResourceFee(t *testing.T) {
 func TestSendTransactionFailedInLedger(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	address := kp.Address()
@@ -268,8 +262,7 @@ func TestSendTransactionFailedInLedger(t *testing.T) {
 func TestSendTransactionFailedInvalidXDR(t *testing.T) {
 	test := NewTest(t, nil)
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	request := methods.SendTransactionRequest{Transaction: "abcdef"}
 	var response methods.SendTransactionResponse
diff --git a/cmd/soroban-rpc/internal/test/upgrade_test.go b/cmd/soroban-rpc/internal/test/upgrade_test.go
index 6889e614..6a59cefd 100644
--- a/cmd/soroban-rpc/internal/test/upgrade_test.go
+++ b/cmd/soroban-rpc/internal/test/upgrade_test.go
@@ -5,8 +5,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/jhttp"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stellar/go/xdr"
@@ -24,8 +22,7 @@ func TestUpgradeFrom20To21(t *testing.T) {
 		ProtocolVersion: 20,
 	})
 
-	ch := jhttp.NewChannel(test.sorobanRPCURL(), nil)
-	client := jrpc2.NewClient(ch, nil)
+	client := test.GetRPCLient()
 
 	sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
 	address := sourceAccount.Address()

From 3773bf8a564168b4e6690d1aa5ab5f88d31d5830 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 17:34:07 +0200
Subject: [PATCH 24/36] Bring down the wait to 30 seconds again

---
 cmd/soroban-rpc/internal/test/integration.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 5011eb71..1467dcdc 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -244,7 +244,7 @@ func (i *Test) waitForRPC() {
 	i.t.Log("Waiting for RPC to be healthy...")
 
 	var result methods.HealthCheckResult
-	for t := 120; t >= 0; t-- {
+	for t := 30; t >= 0; t-- {
 		err := i.rpcClient.CallResult(context.Background(), "getHealth", nil, &result)
 		if err == nil {
 			if result.Status == "healthy" {
@@ -260,7 +260,7 @@ func (i *Test) waitForRPC() {
 	buf := make([]byte, 1<<16)
 	stackSize := runtime.Stack(buf, true)
 	log.Printf("%s\n", string(buf[0:stackSize]))
-	i.t.Fatal("RPC failed to get healthy in 120 seconds")
+	i.t.Fatal("RPC failed to get healthy in 30 seconds")
 }
 
 func (i *Test) createRPCContainerMountDir(rpcConfig map[string]string) string {

From 3f4ca3f1a6f6f34b0cb7999deb112b0609f796cb Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 19:55:16 +0200
Subject: [PATCH 25/36] Traceback ancestors in goroutine dumps

---
 .github/workflows/soroban-rpc.yml         | 2 +-
 cmd/soroban-rpc/internal/daemon/daemon.go | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/soroban-rpc.yml b/.github/workflows/soroban-rpc.yml
index fabc9835..2a5c7e54 100644
--- a/.github/workflows/soroban-rpc.yml
+++ b/.github/workflows/soroban-rpc.yml
@@ -194,4 +194,4 @@ jobs:
       - name: Run Soroban RPC Integration Tests
         run: |
           make install_rust
-          go test -race -timeout 60m -v ./cmd/soroban-rpc/internal/test/...
+          GODEBUG="tracebackancestors=2" go test -race -timeout 60m -v ./cmd/soroban-rpc/internal/test/...
diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go
index fe0cece5..39685927 100644
--- a/cmd/soroban-rpc/internal/daemon/daemon.go
+++ b/cmd/soroban-rpc/internal/daemon/daemon.go
@@ -40,7 +40,6 @@ const (
 	defaultReadTimeout                    = 5 * time.Second
 	defaultShutdownGracePeriod            = 10 * time.Second
 	inMemoryInitializationLedgerLogPeriod = 1_000_000
-	transactionsTableMigrationDoneMetaKey = "TransactionsTableMigrationDone"
 )
 
 type Daemon struct {

From c1d4d774dc9350336eb606fb63e210f938d26403 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 20:10:56 +0200
Subject: [PATCH 26/36] Allow up to 1MB for the stacktrace

---
 cmd/soroban-rpc/internal/test/integration.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 1467dcdc..fd02d17d 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -257,7 +257,7 @@ func (i *Test) waitForRPC() {
 	}
 
 	// Print stack trace and fail
-	buf := make([]byte, 1<<16)
+	buf := make([]byte, 1<<20)
 	stackSize := runtime.Stack(buf, true)
 	log.Printf("%s\n", string(buf[0:stackSize]))
 	i.t.Fatal("RPC failed to get healthy in 30 seconds")

From a0e7bbe759613ce1141a23d23cb8278b65fe8eaa Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 20:29:53 +0200
Subject: [PATCH 27/36] Print if someone is listening

---
 cmd/soroban-rpc/internal/test/integration.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index fd02d17d..ac383478 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -243,12 +243,22 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 func (i *Test) waitForRPC() {
 	i.t.Log("Waiting for RPC to be healthy...")
 
+	// show if anybody is listening on RPC's port
+	outputListeningProc := func() {
+		fmt.Println("Who is listening on RPC port?")
+		cmd := exec.Command("lsof", "-i", fmt.Sprintf(":%d", sorobanRPCPort))
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		require.NoError(i.t, cmd.Run())
+	}
+
 	var result methods.HealthCheckResult
 	for t := 30; t >= 0; t-- {
 		err := i.rpcClient.CallResult(context.Background(), "getHealth", nil, &result)
 		if err == nil {
 			if result.Status == "healthy" {
 				i.t.Log("RPC is healthy")
+				outputListeningProc()
 				return
 			}
 		}
@@ -256,6 +266,7 @@ func (i *Test) waitForRPC() {
 		time.Sleep(time.Second)
 	}
 
+	outputListeningProc()
 	// Print stack trace and fail
 	buf := make([]byte, 1<<20)
 	stackSize := runtime.Stack(buf, true)

From 36c66c689aefdef0ac907dc8c1c394d621969078 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 21:21:16 +0200
Subject: [PATCH 28/36] Refresh soroban rpc client on every failure

---
 cmd/soroban-rpc/internal/test/integration.go  | 25 ++++++-------------
 cmd/soroban-rpc/internal/test/migrate_test.go |  1 -
 2 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index ac383478..13779378 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -15,7 +14,6 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -120,8 +118,6 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 		proxy.ServeHTTP(w, r)
 	}))
 
-	ch := jhttp.NewChannel(i.sorobanRPCURL(), nil)
-	i.rpcClient = jrpc2.NewClient(ch, nil)
 	rpcCfg := i.getRPConfig(sqlLitePath)
 	if i.rpcContainerVersion != "" {
 		i.rpcContainerConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
@@ -243,22 +239,22 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 func (i *Test) waitForRPC() {
 	i.t.Log("Waiting for RPC to be healthy...")
 
-	// show if anybody is listening on RPC's port
-	outputListeningProc := func() {
-		fmt.Println("Who is listening on RPC port?")
-		cmd := exec.Command("lsof", "-i", fmt.Sprintf(":%d", sorobanRPCPort))
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		require.NoError(i.t, cmd.Run())
+	// This is needed because if https://github.com/creachadair/jrpc2/issues/118
+	refreshClient := func() {
+		if i.rpcClient != nil {
+			i.rpcClient.Close()
+		}
+		ch := jhttp.NewChannel(i.sorobanRPCURL(), nil)
+		i.rpcClient = jrpc2.NewClient(ch, nil)
 	}
 
 	var result methods.HealthCheckResult
 	for t := 30; t >= 0; t-- {
+		refreshClient()
 		err := i.rpcClient.CallResult(context.Background(), "getHealth", nil, &result)
 		if err == nil {
 			if result.Status == "healthy" {
 				i.t.Log("RPC is healthy")
-				outputListeningProc()
 				return
 			}
 		}
@@ -266,11 +262,6 @@ func (i *Test) waitForRPC() {
 		time.Sleep(time.Second)
 	}
 
-	outputListeningProc()
-	// Print stack trace and fail
-	buf := make([]byte, 1<<20)
-	stackSize := runtime.Stack(buf, true)
-	log.Printf("%s\n", string(buf[0:stackSize]))
 	i.t.Fatal("RPC failed to get healthy in 30 seconds")
 }
 
diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index c99f74d8..d421f8f0 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -23,7 +23,6 @@ import (
 // We cannot test prior protocol versions since the Transaction XDR used for the test could be incompatible
 // TODO: find a way to test migrations between protocols
 func TestMigrate(t *testing.T) {
-	t.Skip("see if it works when we skip this test")
 	if GetCoreMaxSupportedProtocol() != MaxSupportedProtocolVersion {
 		t.Skip("Only test this for the latest protocol: ", MaxSupportedProtocolVersion)
 	}

From 078fe48f5e90dbc6e29b7d3513fb21c202b723ba Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 21:23:40 +0200
Subject: [PATCH 29/36] Remove debig leftover

---
 .github/workflows/soroban-rpc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soroban-rpc.yml b/.github/workflows/soroban-rpc.yml
index 2a5c7e54..fabc9835 100644
--- a/.github/workflows/soroban-rpc.yml
+++ b/.github/workflows/soroban-rpc.yml
@@ -194,4 +194,4 @@ jobs:
       - name: Run Soroban RPC Integration Tests
         run: |
           make install_rust
-          GODEBUG="tracebackancestors=2" go test -race -timeout 60m -v ./cmd/soroban-rpc/internal/test/...
+          go test -race -timeout 60m -v ./cmd/soroban-rpc/internal/test/...

From 5c4c87e57b4bd4aff0a7b82973fc445ba77dd892 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 21:46:47 +0200
Subject: [PATCH 30/36] Debug RPC container

---
 cmd/soroban-rpc/internal/test/integration.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 13779378..9594d560 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -334,8 +334,12 @@ func (i *Test) runComposeCommand(args ...string) {
 
 	i.t.Log("Running", cmd.Env, cmd.Args)
 	out, innerErr := cmd.Output()
-	if exitErr, ok := innerErr.(*exec.ExitError); ok {
+	exitErr, ok := innerErr.(*exec.ExitError)
+	// TODO: make this cleaner
+	if ok || args[0] == "logs" {
 		fmt.Printf("stdout:\n%s\n", string(out))
+	}
+	if ok {
 		fmt.Printf("stderr:\n%s\n", string(exitErr.Stderr))
 	}
 
@@ -456,6 +460,7 @@ func (i *Test) StopRPC() {
 		i.daemon = nil
 	}
 	if i.rpcContainerVersion != "" {
+		i.runComposeCommand("logs", "rpc")
 		i.runComposeCommand("down", "rpc", "-v")
 	}
 }

From 410fd0d331a134a468172f9e534aeb9b573d4766 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 22:01:32 +0200
Subject: [PATCH 31/36] Add host.docker.internal to rpc container

---
 cmd/soroban-rpc/internal/test/docker-compose.rpc.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
index d142ed9c..46c560ae 100644
--- a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
+++ b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
@@ -12,4 +12,6 @@ services:
     volumes:
       - ${RPC_CONFIG_MOUNT_DIR}/stellar-core-integration-tests.cfg:/stellar-core.cfg
       - ${RPC_CONFIG_MOUNT_DIR}/soroban-rpc.config:/soroban-rpc.config
-      - ${RPC_SQLITE_MOUNT_DIR}:/db/
\ No newline at end of file
+      - ${RPC_SQLITE_MOUNT_DIR}:/db/
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

From cf06605fa901ed9a0a996d372f180d58f9bb2347 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Thu, 13 Jun 2024 23:58:06 +0200
Subject: [PATCH 32/36] Fix archive host names

---
 cmd/soroban-rpc/internal/test/archive_test.go | 18 +++++--
 .../internal/test/docker-compose.rpc.yml      |  3 --
 .../internal/test/docker-compose.yml          |  1 -
 cmd/soroban-rpc/internal/test/integration.go  | 51 +++++++------------
 4 files changed, 32 insertions(+), 41 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/archive_test.go b/cmd/soroban-rpc/internal/test/archive_test.go
index d0aebe49..eaa4578e 100644
--- a/cmd/soroban-rpc/internal/test/archive_test.go
+++ b/cmd/soroban-rpc/internal/test/archive_test.go
@@ -1,7 +1,12 @@
 package test
 
 import (
+	"net"
 	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
+	"strconv"
 	"sync"
 	"testing"
 
@@ -9,12 +14,19 @@ import (
 )
 
 func TestArchiveUserAgent(t *testing.T) {
+	archiveHost := net.JoinHostPort("localhost", strconv.Itoa(StellarCoreArchivePort))
+	proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: archiveHost})
 	userAgents := sync.Map{}
+	historyArchiveProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		userAgents.Store(r.Header["User-Agent"][0], "")
+		proxy.ServeHTTP(w, r)
+	}))
+	defer historyArchiveProxy.Close()
+
 	cfg := &TestConfig{
-		historyArchiveProxyCallback: func(r *http.Request) {
-			userAgents.Store(r.Header["User-Agent"][0], "")
-		},
+		HistoryArchiveURL: historyArchiveProxy.URL,
 	}
+
 	NewTest(t, cfg)
 
 	_, ok := userAgents.Load("soroban-rpc/0.0.0")
diff --git a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
index 46c560ae..ae540528 100644
--- a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
+++ b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   rpc:
     platform: linux/amd64
@@ -13,5 +12,3 @@ services:
       - ${RPC_CONFIG_MOUNT_DIR}/stellar-core-integration-tests.cfg:/stellar-core.cfg
       - ${RPC_CONFIG_MOUNT_DIR}/soroban-rpc.config:/soroban-rpc.config
       - ${RPC_SQLITE_MOUNT_DIR}:/db/
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
diff --git a/cmd/soroban-rpc/internal/test/docker-compose.yml b/cmd/soroban-rpc/internal/test/docker-compose.yml
index 4b246f7b..cf3e7b0d 100644
--- a/cmd/soroban-rpc/internal/test/docker-compose.yml
+++ b/cmd/soroban-rpc/internal/test/docker-compose.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   core-postgres:
     image: postgres:9.6.17-alpine
diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 9594d560..0ed331c3 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -4,11 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/http/httputil"
-	"net/url"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -39,7 +34,7 @@ const (
 	StandaloneNetworkPassphrase = "Standalone Network ; February 2017"
 	MaxSupportedProtocolVersion = 21
 	stellarCorePort             = 11626
-	stellarCoreArchiveHost      = "localhost:1570"
+	StellarCoreArchivePort      = 1570
 	goModFile                   = "go.mod"
 
 	friendbotURL = "http://localhost:8000/friendbot"
@@ -51,11 +46,11 @@ const (
 )
 
 type TestConfig struct {
-	historyArchiveProxyCallback func(*http.Request)
-	ProtocolVersion             uint32
+	ProtocolVersion uint32
 	// Run a previously released version of RPC (in a container) instead of the current version
 	UseReleasedRPCVersion string
 	UseSQLitePath         string
+	HistoryArchiveURL     string
 }
 
 type Test struct {
@@ -65,21 +60,20 @@ type Test struct {
 
 	protocolVersion uint32
 
+	historyArchiveURL string
+
 	rpcContainerVersion        string
 	rpcContainerConfigMountDir string
 	rpcContainerSQLiteMountDir string
+	rpcClient                  *jrpc2.Client
 
 	daemon *daemon.Daemon
 
-	historyArchiveProxy         *httptest.Server
-	historyArchiveProxyCallback func(*http.Request)
-
 	coreClient *stellarcore.Client
 
 	masterAccount txnbuild.Account
 	shutdownOnce  sync.Once
 	shutdownCalls []func()
-	rpcClient     *jrpc2.Client
 }
 
 func NewTest(t *testing.T, cfg *TestConfig) *Test {
@@ -98,8 +92,8 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 
 	sqlLitePath := ""
 	if cfg != nil {
+		i.historyArchiveURL = cfg.HistoryArchiveURL
 		i.rpcContainerVersion = cfg.UseReleasedRPCVersion
-		i.historyArchiveProxyCallback = cfg.historyArchiveProxyCallback
 		i.protocolVersion = cfg.ProtocolVersion
 		sqlLitePath = cfg.UseSQLitePath
 	}
@@ -109,15 +103,6 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 		i.protocolVersion = GetCoreMaxSupportedProtocol()
 	}
 
-	proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: stellarCoreArchiveHost})
-
-	i.historyArchiveProxy = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if i.historyArchiveProxyCallback != nil {
-			i.historyArchiveProxyCallback(r)
-		}
-		proxy.ServeHTTP(w, r)
-	}))
-
 	rpcCfg := i.getRPConfig(sqlLitePath)
 	if i.rpcContainerVersion != "" {
 		i.rpcContainerConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
@@ -192,19 +177,20 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 
 	// Out of the container default file
 	captiveCoreConfigPath := path.Join(i.composePath, "captive-core-integration-tests.cfg")
-	archiveProxyURL := i.historyArchiveProxy.URL
+	archiveURL := fmt.Sprintf("http://localhost:%d", StellarCoreArchivePort)
+	if i.historyArchiveURL != "" {
+		archiveURL = i.historyArchiveURL
+	} else if i.rpcContainerVersion != "" {
+		// the archive needs to be accessed from the container
+		// where core is Core's hostname
+		archiveURL = fmt.Sprintf("http://core:%d", StellarCoreArchivePort)
+	}
+
 	stellarCoreURL := fmt.Sprintf("http://localhost:%d", stellarCorePort)
 	bindHost := "localhost"
 	if i.rpcContainerVersion != "" {
 		// The file will be inside the container
 		captiveCoreConfigPath = "/stellar-core.cfg"
-		// the archive needs to be accessed from the container
-		url, err := url.Parse(i.historyArchiveProxy.URL)
-		require.NoError(i.t, err)
-		_, port, err := net.SplitHostPort(url.Host)
-		require.NoError(i.t, err)
-		url.Host = net.JoinHostPort("host.docker.internal", port)
-		archiveProxyURL = url.String()
 		// The container needs to listen on all interfaces, not just localhost
 		bindHost = "0.0.0.0"
 		// The container needs to use the sqlite mount point
@@ -224,7 +210,7 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 		"STELLAR_CAPTIVE_CORE_HTTP_PORT": "0",
 		"FRIENDBOT_URL":                  friendbotURL,
 		"NETWORK_PASSPHRASE":             StandaloneNetworkPassphrase,
-		"HISTORY_ARCHIVE_URLS":           archiveProxyURL,
+		"HISTORY_ARCHIVE_URLS":           archiveURL,
 		"LOG_LEVEL":                      "debug",
 		"DB_PATH":                        sqlitePath,
 		"INGESTION_TIMEOUT":              "10m",
@@ -354,9 +340,6 @@ func (i *Test) prepareShutdownHandlers() {
 		func() {
 			close(done)
 			i.StopRPC()
-			if i.historyArchiveProxy != nil {
-				i.historyArchiveProxy.Close()
-			}
 			if i.rpcClient != nil {
 				i.rpcClient.Close()
 			}

From 618aac8c04e401a188863190f56e0ba7bf2ffaa3 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Fri, 14 Jun 2024 00:26:41 +0200
Subject: [PATCH 33/36] Run docker-compose logs -f rpc in the background

---
 .../internal/test/docker-compose.rpc.yml      |  3 +
 cmd/soroban-rpc/internal/test/integration.go  | 66 ++++++++++++-------
 2 files changed, 44 insertions(+), 25 deletions(-)

diff --git a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
index ae540528..3443aff4 100644
--- a/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
+++ b/cmd/soroban-rpc/internal/test/docker-compose.rpc.yml
@@ -12,3 +12,6 @@ services:
       - ${RPC_CONFIG_MOUNT_DIR}/stellar-core-integration-tests.cfg:/stellar-core.cfg
       - ${RPC_CONFIG_MOUNT_DIR}/soroban-rpc.config:/soroban-rpc.config
       - ${RPC_SQLITE_MOUNT_DIR}:/db/
+    # Needed so that the sql database files created in the container
+    # have the same uid and gid as in the host
+    user: "${RPC_UID}:${RPC_GID}"
diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 0ed331c3..dcd10b15 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -65,12 +65,13 @@ type Test struct {
 	rpcContainerVersion        string
 	rpcContainerConfigMountDir string
 	rpcContainerSQLiteMountDir string
-	rpcClient                  *jrpc2.Client
-
-	daemon *daemon.Daemon
+	rpcContainerLogsCommand    *exec.Cmd
 
+	rpcClient  *jrpc2.Client
 	coreClient *stellarcore.Client
 
+	daemon *daemon.Daemon
+
 	masterAccount txnbuild.Account
 	shutdownOnce  sync.Once
 	shutdownCalls []func()
@@ -104,15 +105,21 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 	}
 
 	rpcCfg := i.getRPConfig(sqlLitePath)
-	if i.rpcContainerVersion != "" {
+	if i.runRPCInContainer() {
 		i.rpcContainerConfigMountDir = i.createRPCContainerMountDir(rpcCfg)
 	}
 	i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color")
+	if i.runRPCInContainer() {
+		cmd := i.getComposeCommand("logs", "-f", "rpc")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		require.NoError(t, cmd.Start())
+	}
 	i.prepareShutdownHandlers()
 	i.coreClient = &stellarcore.Client{URL: "http://localhost:" + strconv.Itoa(stellarCorePort)}
 	i.waitForCore()
 	i.waitForCheckpoint()
-	if i.rpcContainerVersion == "" {
+	if !i.runRPCInContainer() {
 		i.daemon = i.createDaemon(rpcCfg)
 		go i.daemon.Run()
 	}
@@ -121,6 +128,10 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test {
 	return i
 }
 
+func (i *Test) runRPCInContainer() bool {
+	return i.rpcContainerVersion != ""
+}
+
 func (i *Test) GetRPCLient() *jrpc2.Client {
 	return i.rpcClient
 }
@@ -166,29 +177,30 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 		sqlitePath = path.Join(i.t.TempDir(), "soroban_rpc.sqlite")
 	}
 
-	// Container default path
+	// Container's default path to captive core
 	coreBinaryPath := "/usr/bin/stellar-core"
-	if i.rpcContainerVersion == "" {
+	if !i.runRPCInContainer() {
 		coreBinaryPath = os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")
 		if coreBinaryPath == "" {
 			i.t.Fatal("missing SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")
 		}
 	}
 
-	// Out of the container default file
-	captiveCoreConfigPath := path.Join(i.composePath, "captive-core-integration-tests.cfg")
 	archiveURL := fmt.Sprintf("http://localhost:%d", StellarCoreArchivePort)
-	if i.historyArchiveURL != "" {
-		archiveURL = i.historyArchiveURL
-	} else if i.rpcContainerVersion != "" {
+	if i.runRPCInContainer() {
 		// the archive needs to be accessed from the container
 		// where core is Core's hostname
 		archiveURL = fmt.Sprintf("http://core:%d", StellarCoreArchivePort)
 	}
+	if i.historyArchiveURL != "" {
+		// an archive URL was supplied explicitly
+		archiveURL = i.historyArchiveURL
+	}
 
-	stellarCoreURL := fmt.Sprintf("http://localhost:%d", stellarCorePort)
+	captiveCoreConfigPath := path.Join(i.composePath, "captive-core-integration-tests.cfg")
 	bindHost := "localhost"
-	if i.rpcContainerVersion != "" {
+	stellarCoreURL := fmt.Sprintf("http://localhost:%d", stellarCorePort)
+	if i.runRPCInContainer() {
 		// The file will be inside the container
 		captiveCoreConfigPath = "/stellar-core.cfg"
 		// The container needs to listen on all interfaces, not just localhost
@@ -289,11 +301,10 @@ func (i *Test) createDaemon(env map[string]string) *daemon.Daemon {
 	return daemon.MustNew(&cfg)
 }
 
-// Runs a docker-compose command applied to the above configs
-func (i *Test) runComposeCommand(args ...string) {
+func (i *Test) getComposeCommand(args ...string) *exec.Cmd {
 	integrationYaml := filepath.Join(i.composePath, "docker-compose.yml")
 	configFiles := []string{"-f", integrationYaml}
-	if i.rpcContainerVersion != "" {
+	if i.runRPCInContainer() {
 		rpcYaml := filepath.Join(i.composePath, "docker-compose.rpc.yml")
 		configFiles = append(configFiles, "-f", rpcYaml)
 	}
@@ -306,26 +317,29 @@ func (i *Test) runComposeCommand(args ...string) {
 			"CORE_IMAGE="+img,
 		)
 	}
-	if i.rpcContainerVersion != "" {
+	if i.runRPCInContainer() {
 		cmd.Env = append(
 			cmd.Env,
 			"RPC_IMAGE_TAG="+i.rpcContainerVersion,
 			"RPC_CONFIG_MOUNT_DIR="+i.rpcContainerConfigMountDir,
 			"RPC_SQLITE_MOUNT_DIR="+i.rpcContainerSQLiteMountDir,
+			"RPC_UID="+strconv.Itoa(os.Getuid()),
+			"RPC_GID="+strconv.Itoa(os.Getgid()),
 		)
 	}
 	if len(cmd.Env) > 0 {
 		cmd.Env = append(cmd.Env, os.Environ()...)
 	}
+	return cmd
+}
 
+// Runs a docker-compose command applied to the above configs
+func (i *Test) runComposeCommand(args ...string) {
+	cmd := i.getComposeCommand(args...)
 	i.t.Log("Running", cmd.Env, cmd.Args)
 	out, innerErr := cmd.Output()
-	exitErr, ok := innerErr.(*exec.ExitError)
-	// TODO: make this cleaner
-	if ok || args[0] == "logs" {
+	if exitErr, ok := innerErr.(*exec.ExitError); ok {
 		fmt.Printf("stdout:\n%s\n", string(out))
-	}
-	if ok {
 		fmt.Printf("stderr:\n%s\n", string(exitErr.Stderr))
 	}
 
@@ -344,6 +358,9 @@ func (i *Test) prepareShutdownHandlers() {
 				i.rpcClient.Close()
 			}
 			i.runComposeCommand("down", "-v")
+			if i.rpcContainerLogsCommand != nil {
+				i.rpcContainerLogsCommand.Wait()
+			}
 		},
 	)
 
@@ -442,8 +459,7 @@ func (i *Test) StopRPC() {
 		i.daemon.Close()
 		i.daemon = nil
 	}
-	if i.rpcContainerVersion != "" {
-		i.runComposeCommand("logs", "rpc")
+	if i.runRPCInContainer() {
 		i.runComposeCommand("down", "rpc", "-v")
 	}
 }

From 10e09bf998e93a7ad3fbb053ddbf42e29591fb2f Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Fri, 14 Jun 2024 00:30:00 +0200
Subject: [PATCH 34/36] Fix captive core storage path

---
 cmd/soroban-rpc/internal/test/integration.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index dcd10b15..0425febb 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -211,6 +211,12 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 		stellarCoreURL = fmt.Sprintf("http://core:%d", stellarCorePort)
 	}
 
+	// in the container
+	captiveCoreStoragePath := "/tmp/captive-core"
+	if !i.runRPCInContainer() {
+		captiveCoreStoragePath = i.t.TempDir()
+	}
+
 	return map[string]string{
 		"ENDPOINT":                       fmt.Sprintf("%s:%d", bindHost, sorobanRPCPort),
 		"ADMIN_ENDPOINT":                 fmt.Sprintf("%s:%d", bindHost, adminPort),
@@ -218,7 +224,7 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 		"CORE_REQUEST_TIMEOUT":           "2s",
 		"STELLAR_CORE_BINARY_PATH":       coreBinaryPath,
 		"CAPTIVE_CORE_CONFIG_PATH":       captiveCoreConfigPath,
-		"CAPTIVE_CORE_STORAGE_PATH":      i.t.TempDir(),
+		"CAPTIVE_CORE_STORAGE_PATH":      captiveCoreStoragePath,
 		"STELLAR_CAPTIVE_CORE_HTTP_PORT": "0",
 		"FRIENDBOT_URL":                  friendbotURL,
 		"NETWORK_PASSPHRASE":             StandaloneNetworkPassphrase,

From 7a3daca44abd5ee5fc220898fd35e152e958aa98 Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Fri, 14 Jun 2024 00:34:45 +0200
Subject: [PATCH 35/36] Stop printing env variables with docker-compose

---
 cmd/soroban-rpc/internal/test/integration.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index 0425febb..b013ff47 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -342,7 +342,7 @@ func (i *Test) getComposeCommand(args ...string) *exec.Cmd {
 // Runs a docker-compose command applied to the above configs
 func (i *Test) runComposeCommand(args ...string) {
 	cmd := i.getComposeCommand(args...)
-	i.t.Log("Running", cmd.Env, cmd.Args)
+	i.t.Log("Running", cmd.Args)
 	out, innerErr := cmd.Output()
 	if exitErr, ok := innerErr.(*exec.ExitError); ok {
 		fmt.Printf("stdout:\n%s\n", string(out))

From 8bff2dc86d9ac9e096406fd378d56e963e830f2d Mon Sep 17 00:00:00 2001
From: Alfonso Acosta <alfonso@stellar.org>
Date: Fri, 14 Jun 2024 01:00:32 +0200
Subject: [PATCH 36/36] Final cleanup

---
 cmd/soroban-rpc/internal/test/integration.go  | 131 +++++-------------
 .../internal/test/integration_test.go         |  15 --
 cmd/soroban-rpc/internal/test/migrate_test.go |  16 +--
 3 files changed, 35 insertions(+), 127 deletions(-)
 delete mode 100644 cmd/soroban-rpc/internal/test/integration_test.go

diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go
index b013ff47..531a162f 100644
--- a/cmd/soroban-rpc/internal/test/integration.go
+++ b/cmd/soroban-rpc/internal/test/integration.go
@@ -1,7 +1,6 @@
 package test
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"os"
@@ -9,6 +8,7 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -33,11 +33,9 @@ import (
 const (
 	StandaloneNetworkPassphrase = "Standalone Network ; February 2017"
 	MaxSupportedProtocolVersion = 21
-	stellarCorePort             = 11626
 	StellarCoreArchivePort      = 1570
-	goModFile                   = "go.mod"
-
-	friendbotURL = "http://localhost:8000/friendbot"
+	stellarCorePort             = 11626
+	friendbotURL                = "http://localhost:8000/friendbot"
 	// Needed when Core is run with ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 	checkpointFrequency    = 8
 	sorobanRPCPort         = 8000
@@ -56,8 +54,6 @@ type TestConfig struct {
 type Test struct {
 	t *testing.T
 
-	composePath string // docker compose yml file
-
 	protocolVersion uint32
 
 	historyArchiveURL string
@@ -74,17 +70,14 @@ type Test struct {
 
 	masterAccount txnbuild.Account
 	shutdownOnce  sync.Once
-	shutdownCalls []func()
+	shutdown      func()
 }
 
 func NewTest(t *testing.T, cfg *TestConfig) *Test {
 	if os.Getenv("SOROBAN_RPC_INTEGRATION_TESTS_ENABLED") == "" {
 		t.Skip("skipping integration test: SOROBAN_RPC_INTEGRATION_TESTS_ENABLED not set")
 	}
-	i := &Test{
-		t:           t,
-		composePath: findDockerComposePath(),
-	}
+	i := &Test{t: t}
 
 	i.masterAccount = &txnbuild.SimpleAccount{
 		AccountID: i.MasterKey().Address(),
@@ -197,7 +190,7 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string {
 		archiveURL = i.historyArchiveURL
 	}
 
-	captiveCoreConfigPath := path.Join(i.composePath, "captive-core-integration-tests.cfg")
+	captiveCoreConfigPath := path.Join(GetCurrentDirectory(), "captive-core-integration-tests.cfg")
 	bindHost := "localhost"
 	stellarCoreURL := fmt.Sprintf("http://localhost:%d", stellarCorePort)
 	if i.runRPCInContainer() {
@@ -272,16 +265,14 @@ func (i *Test) waitForRPC() {
 func (i *Test) createRPCContainerMountDir(rpcConfig map[string]string) string {
 	mountDir := i.t.TempDir()
 	// Get old version of captive-core-integration-tests.cfg
-	var out bytes.Buffer
 	cmd := exec.Command("git", "show", fmt.Sprintf("v%s:./captive-core-integration-tests.cfg", i.rpcContainerVersion))
-	cmd.Stdout = &out
-	cmd.Stderr = os.Stderr
-	cmd.Dir = i.composePath
-	require.NoError(i.t, cmd.Run())
+	cmd.Dir = GetCurrentDirectory()
+	out, err := cmd.Output()
+	require.NoError(i.t, err)
 
 	// replace ADDRESS="localhost" by ADDRESS="core", so that the container can find core
-	captiveCoreCfgContents := strings.Replace(out.String(), `ADDRESS="localhost"`, `ADDRESS="core"`, -1)
-	err := os.WriteFile(filepath.Join(mountDir, "stellar-core-integration-tests.cfg"), []byte(captiveCoreCfgContents), 0666)
+	captiveCoreCfgContents := strings.Replace(string(out), `ADDRESS="localhost"`, `ADDRESS="core"`, -1)
+	err = os.WriteFile(filepath.Join(mountDir, "stellar-core-integration-tests.cfg"), []byte(captiveCoreCfgContents), 0666)
 	require.NoError(i.t, err)
 
 	// Generate config file
@@ -308,10 +299,10 @@ func (i *Test) createDaemon(env map[string]string) *daemon.Daemon {
 }
 
 func (i *Test) getComposeCommand(args ...string) *exec.Cmd {
-	integrationYaml := filepath.Join(i.composePath, "docker-compose.yml")
+	integrationYaml := filepath.Join(GetCurrentDirectory(), "docker-compose.yml")
 	configFiles := []string{"-f", integrationYaml}
 	if i.runRPCInContainer() {
-		rpcYaml := filepath.Join(i.composePath, "docker-compose.rpc.yml")
+		rpcYaml := filepath.Join(GetCurrentDirectory(), "docker-compose.rpc.yml")
 		configFiles = append(configFiles, "-f", rpcYaml)
 	}
 	cmdline := append(configFiles, args...)
@@ -356,21 +347,19 @@ func (i *Test) runComposeCommand(args ...string) {
 
 func (i *Test) prepareShutdownHandlers() {
 	done := make(chan struct{})
-	i.shutdownCalls = append(i.shutdownCalls,
-		func() {
-			close(done)
-			i.StopRPC()
-			if i.rpcClient != nil {
-				i.rpcClient.Close()
-			}
-			i.runComposeCommand("down", "-v")
-			if i.rpcContainerLogsCommand != nil {
-				i.rpcContainerLogsCommand.Wait()
-			}
-		},
-	)
+	i.shutdown = func() {
+		close(done)
+		i.StopRPC()
+		if i.rpcClient != nil {
+			i.rpcClient.Close()
+		}
+		i.runComposeCommand("down", "-v")
+		if i.rpcContainerLogsCommand != nil {
+			i.rpcContainerLogsCommand.Wait()
+		}
+	}
 
-	// Register cleanup handlers (on panic and ctrl+c) so the containers are
+	// Register shutdown handlers (on panic and ctrl+c) so the containers are
 	// stopped even if ingestion or testing fails.
 	i.t.Cleanup(i.Shutdown)
 
@@ -392,10 +381,7 @@ func (i *Test) prepareShutdownHandlers() {
 // called before.
 func (i *Test) Shutdown() {
 	i.shutdownOnce.Do(func() {
-		// run them in the opposite order in which they where added
-		for callI := len(i.shutdownCalls) - 1; callI >= 0; callI-- {
-			i.shutdownCalls[callI]()
-		}
+		i.shutdown()
 	})
 }
 
@@ -407,7 +393,7 @@ func (i *Test) waitForCore() {
 		_, err := i.coreClient.Info(ctx)
 		cancel()
 		if err != nil {
-			i.t.Logf("could not obtain info response: %v", err)
+			i.t.Logf("Core is not up: %v", err)
 			time.Sleep(time.Second)
 			continue
 		}
@@ -470,65 +456,10 @@ func (i *Test) StopRPC() {
 	}
 }
 
-// Cluttering code with if err != nil is absolute nonsense.
-func panicIf(err error) {
-	if err != nil {
-		panic(err)
-	}
-}
-
-// findProjectRoot iterates upward on the directory until go.mod file is found.
-func findProjectRoot(current string) string {
-	// Lets you check if a particular directory contains a file.
-	directoryContainsFilename := func(dir string, filename string) bool {
-		files, innerErr := os.ReadDir(dir)
-		panicIf(innerErr)
-
-		for _, file := range files {
-			if file.Name() == filename {
-				return true
-			}
-		}
-		return false
-	}
-	var err error
-
-	// In either case, we try to walk up the tree until we find "go.mod",
-	// which we hope is the root directory of the project.
-	for !directoryContainsFilename(current, goModFile) {
-		current, err = filepath.Abs(filepath.Join(current, ".."))
-
-		// FIXME: This only works on *nix-like systems.
-		if err != nil || filepath.Base(current)[0] == filepath.Separator {
-			fmt.Println("Failed to establish project root directory.")
-			panic(err)
-		}
-	}
-	return current
-}
-
-// findDockerComposePath performs a best-effort attempt to find the project's
-// Docker Compose files.
-func findDockerComposePath() string {
-	current, err := os.Getwd()
-	panicIf(err)
-
-	//
-	// We have a primary and backup attempt for finding the necessary docker
-	// files: via $GOPATH and via local directory traversal.
-	//
-
-	if gopath := os.Getenv("GOPATH"); gopath != "" {
-		monorepo := filepath.Join(gopath, "src", "github.com", "stellar", "soroban-rpc")
-		if _, err = os.Stat(monorepo); !os.IsNotExist(err) {
-			current = monorepo
-		}
-	}
-
-	current = findProjectRoot(current)
-
-	// Directly jump down to the folder that should contain the configs
-	return filepath.Join(current, "cmd", "soroban-rpc", "internal", "test")
+//go:noinline
+func GetCurrentDirectory() string {
+	_, currentFilename, _, _ := runtime.Caller(1)
+	return filepath.Dir(currentFilename)
 }
 
 func GetCoreMaxSupportedProtocol() uint32 {
diff --git a/cmd/soroban-rpc/internal/test/integration_test.go b/cmd/soroban-rpc/internal/test/integration_test.go
deleted file mode 100644
index 684a61ad..00000000
--- a/cmd/soroban-rpc/internal/test/integration_test.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package test
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestFindDockerComposePath(t *testing.T) {
-	dockerPath := findDockerComposePath()
-
-	if len(dockerPath) == 0 {
-		t.Fail()
-	}
-	fmt.Printf("docker compose path is %s\n", dockerPath)
-}
diff --git a/cmd/soroban-rpc/internal/test/migrate_test.go b/cmd/soroban-rpc/internal/test/migrate_test.go
index d421f8f0..c8fcc7ec 100644
--- a/cmd/soroban-rpc/internal/test/migrate_test.go
+++ b/cmd/soroban-rpc/internal/test/migrate_test.go
@@ -1,12 +1,9 @@
 package test
 
 import (
-	"bytes"
 	"context"
-	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -40,7 +37,6 @@ func TestMigrate(t *testing.T) {
 			testMigrateFromVersion(t, originVersion)
 		})
 	}
-
 }
 
 func testMigrateFromVersion(t *testing.T, version string) {
@@ -105,15 +101,11 @@ func testMigrateFromVersion(t *testing.T, version string) {
 
 func getCurrentProtocolReleasedVersions(t *testing.T) []string {
 	protocolStr := strconv.Itoa(MaxSupportedProtocolVersion)
-	_, currentFilename, _, _ := runtime.Caller(0)
-	currentDir := filepath.Dir(currentFilename)
-	var out bytes.Buffer
 	cmd := exec.Command("git", "tag")
-	cmd.Dir = currentDir
-	cmd.Stdout = &out
-	cmd.Stderr = os.Stderr
-	require.NoError(t, cmd.Run())
-	tags := strings.Split(out.String(), "\n")
+	cmd.Dir = GetCurrentDirectory()
+	out, err := cmd.Output()
+	require.NoError(t, err)
+	tags := strings.Split(string(out), "\n")
 	filteredTags := make([]string, 0, len(tags))
 	for _, tag := range tags {
 		if strings.HasPrefix(tag, "v"+protocolStr) {