diff --git a/cmd/soroban-rpc/internal/config/config.go b/cmd/soroban-rpc/internal/config/config.go index 268689f68e..24e892655f 100644 --- a/cmd/soroban-rpc/internal/config/config.go +++ b/cmd/soroban-rpc/internal/config/config.go @@ -13,4 +13,5 @@ type LocalConfig struct { LogLevel logrus.Level TxConcurrency int TxQueueSize int + SQLiteDBPath string } diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index d90d5b3840..c41395d747 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -14,6 +14,7 @@ import ( "github.com/stellar/soroban-tools/cmd/soroban-rpc/internal" "github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/config" + "github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/ledgerentry_storage" "github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/methods" ) @@ -59,7 +60,12 @@ func Start(cfg config.LocalConfig) (exitCode int) { logger.Fatalf("could not connect to history archive: %v", err) } - storage, err := internal.NewLedgerEntryStorage(cfg.NetworkPassphrase, historyArchive, core) + db, err := ledgerentry_storage.OpenSQLiteDB(cfg.SQLiteDBPath) + if err != nil { + logger.Fatalf("could not open database: %v", err) + } + + storage, err := ledgerentry_storage.NewLedgerEntryStorage(logger, db, cfg.NetworkPassphrase, historyArchive, core) if err != nil { logger.Fatalf("could not initialize ledger entry storage: %v", err) } diff --git a/cmd/soroban-rpc/internal/ledgerentry_storage.go b/cmd/soroban-rpc/internal/ledgerentry_storage.go deleted file mode 100644 index 6b21177a1c..0000000000 --- a/cmd/soroban-rpc/internal/ledgerentry_storage.go +++ /dev/null @@ -1,210 +0,0 @@ -package internal - -import ( - "context" - "errors" - "fmt" - "io" - "sync" - "time" - - "github.com/stellar/go/historyarchive" - "github.com/stellar/go/ingest" - backends "github.com/stellar/go/ingest/ledgerbackend" - "github.com/stellar/go/xdr" -) - -type LedgerEntryStorage interface { - GetLedgerEntry(key xdr.LedgerKey) (xdr.LedgerEntry, bool, uint32, error) - io.Closer -} - -func NewLedgerEntryStorage( - networkPassPhrase string, - archive historyarchive.ArchiveInterface, - ledgerBackend backends.LedgerBackend) (LedgerEntryStorage, error) { - root, err := archive.GetRootHAS() - if err != nil { - return nil, err - } - checkpointLedger := root.CurrentLedger - ctx, done := context.WithCancel(context.Background()) - ls := ledgerEntryStorage{ - networkPassPhrase: networkPassPhrase, - storage: map[string]xdr.LedgerEntry{}, - done: done, - } - ls.wg.Add(1) - go ls.run(ctx, checkpointLedger, archive, ledgerBackend) - return &ls, nil -} - -type ledgerEntryStorage struct { - encodingBuffer *xdr.EncodingBuffer - networkPassPhrase string - // from serialized ledger key to ledger entry - storage map[string]xdr.LedgerEntry - // What's the latest processed ledger - latestLedger uint32 - sync.RWMutex - done context.CancelFunc - wg sync.WaitGroup -} - -func (ls *ledgerEntryStorage) GetLedgerEntry(key xdr.LedgerKey) (xdr.LedgerEntry, bool, uint32, error) { - stringKey := getRelevantLedgerKey(ls.encodingBuffer, key) - if stringKey == "" { - return xdr.LedgerEntry{}, false, 0, nil - } - ls.RLock() - defer ls.RUnlock() - if ls.latestLedger == 0 { - // we haven't yet processed the first checkpoint - return xdr.LedgerEntry{}, false, 0, errors.New("Ledger storage not initialized yet") - } - - entry, present := ls.storage[stringKey] - if !present { - return xdr.LedgerEntry{}, false, 0, nil - } - return entry, true, ls.latestLedger, nil -} - -func (ls *ledgerEntryStorage) Close() error { - ls.done() - ls.wg.Wait() - return nil -} - -func (ls *ledgerEntryStorage) run(ctx context.Context, startCheckpointLedger uint32, archive historyarchive.ArchiveInterface, ledgerBackend backends.LedgerBackend) { - defer ls.wg.Done() - - // First, process the checkpoint - // TODO: use a logger - fmt.Println("Starting processing of checkpoint", startCheckpointLedger) - checkpointCtx, cancelCheckpointCtx := context.WithTimeout(ctx, 30*time.Minute) - reader, err := ingest.NewCheckpointChangeReader(checkpointCtx, archive, startCheckpointLedger) - if err != nil { - // TODO: implement retries instead - panic(err) - } - // We intentionally use this local encoding buffer to avoid race conditions with the main one - buffer := xdr.NewEncodingBuffer() - - for { - select { - case <-ctx.Done(): - cancelCheckpointCtx() - return - default: - } - change, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - // TODO: we probably shouldn't panic, at least in case of timeout - panic(err) - } - - entry := change.Post - key := getRelevantLedgerKeyFromData(buffer, entry.Data) - if key == "" { - // not relevant - continue - } - - // no need to Write-lock until we process the full checkpoint, since the reader checks latestLedger to be non-zero - ls.storage[key] = *entry - - if len(ls.storage)%2000 == 0 { - fmt.Printf(" processed %d checkpoint ledger entries\n", len(ls.storage)) - } - } - - cancelCheckpointCtx() - - fmt.Println("Finished checkpoint processing") - ls.Lock() - ls.latestLedger = startCheckpointLedger - ls.Unlock() - - // Now, continuously process txmeta deltas - - // TODO: we can probably do the preparation in parallel with the checkpoint processing - prepareRangeCtx, cancelPrepareRange := context.WithTimeout(ctx, 30*time.Minute) - if err := ledgerBackend.PrepareRange(prepareRangeCtx, backends.UnboundedRange(startCheckpointLedger)); err != nil { - // TODO: we probably shouldn't panic, at least in case of timeout - panic(err) - } - cancelPrepareRange() - - nextLedger := startCheckpointLedger + 1 - for { - fmt.Println("Processing txmeta of ledger", nextLedger) - reader, err := ingest.NewLedgerChangeReader(ctx, ledgerBackend, ls.networkPassPhrase, nextLedger) - if err != nil { - // TODO: we probably shouldn't panic, at least in case of timeout/cancellation - panic(err) - } - - // TODO: completely blocking reads between ledgers being processed may not be acceptable - // however, we don't want to return ledger entries inbetween ledger updates - ls.Lock() - for { - change, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - // TODO: we probably shouldn't panic, at least in case of timeout/cancellation - panic(err) - } - if change.Post == nil { - key := getRelevantLedgerKeyFromData(buffer, change.Pre.Data) - if key == "" { - continue - } - delete(ls.storage, key) - } else { - key := getRelevantLedgerKeyFromData(buffer, change.Post.Data) - if key == "" { - continue - } - ls.storage[key] = *change.Post - } - } - ls.latestLedger = nextLedger - nextLedger++ - fmt.Println("Ledger entry count", len(ls.storage)) - ls.Unlock() - reader.Close() - } - -} - -func getRelevantLedgerKey(buffer *xdr.EncodingBuffer, key xdr.LedgerKey) string { - // this is safe since we are converting to string right away, which causes a copy - binKey, err := buffer.LedgerKeyUnsafeMarshalBinaryCompress(key) - if err != nil { - // TODO: we probably don't want to panic - panic(err) - } - return string(binKey) -} - -func getRelevantLedgerKeyFromData(buffer *xdr.EncodingBuffer, data xdr.LedgerEntryData) string { - var key xdr.LedgerKey - switch data.Type { - case xdr.LedgerEntryTypeAccount: - key.SetAccount(data.Account.AccountId) - case xdr.LedgerEntryTypeTrustline: - key.SetTrustline(data.TrustLine.AccountId, data.TrustLine.Asset) - case xdr.LedgerEntryTypeContractData: - key.SetContractData(data.ContractData.ContractId, data.ContractData.Val) - default: - // we don't care about any other entry types for now - return "" - } - return getRelevantLedgerKey(buffer, key) -} diff --git a/cmd/soroban-rpc/internal/ledgerentry_storage/db.go b/cmd/soroban-rpc/internal/ledgerentry_storage/db.go new file mode 100644 index 0000000000..fde75d70b6 --- /dev/null +++ b/cmd/soroban-rpc/internal/ledgerentry_storage/db.go @@ -0,0 +1,295 @@ +package ledgerentry_storage + +import ( + "context" + "database/sql" + "embed" + "fmt" + "strconv" + + sq "github.com/Masterminds/squirrel" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + migrate "github.com/rubenv/sql-migrate" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" +) + +//go:embed migrations/*.sql +var migrations embed.FS + +var ErrEmptyDB = errors.New("DB is empty") + +const ( + ledgerEntriesTableName = "ledger_entries" + ledgerEntriesMetaTableName = "ledger_entries_meta" + latestLedgerSequenceMetaKey = "LatestLedgerSequence" +) + +type DB interface { + LedgerEntryStorage + GetLatestLedgerSequence() (uint32, error) + NewLedgerEntryUpdaterTx(forLedgerSequence uint32, maxBatchSize int) (LedgerEntryUpdaterTx, error) +} + +type LedgerEntryUpdaterTx interface { + UpsertLedgerEntry(key xdr.LedgerKey, entry xdr.LedgerEntry) error + DeleteLedgerEntry(key xdr.LedgerKey) error + Done() error +} + +type sqlDB struct { + db *sqlx.DB +} + +func OpenSQLiteDB(dbFilePath string) (DB, error) { + db, err := sqlx.Open("sqlite3", dbFilePath) + if err != nil { + return nil, errors.Wrap(err, "open failed") + } + + ret := &sqlDB{ + db: db, + } + + err = runMigrations(ret.db.DB, "sqlite3") + + return ret, err +} + +func getLedgerEntry(tx *sqlx.Tx, buffer *xdr.EncodingBuffer, key xdr.LedgerKey) (xdr.LedgerEntry, error) { + encodedKey, err := encodeLedgerKey(buffer, key) + if err != nil { + return xdr.LedgerEntry{}, err + } + + sqlStr, args, err := sq.Select("entry").From(ledgerEntriesTableName).Where(sq.Eq{"key": encodedKey}).ToSql() + if err != nil { + return xdr.LedgerEntry{}, err + } + var results []string + if err := tx.Select(&results, sqlStr, args...); err != nil { + return xdr.LedgerEntry{}, err + } + if len(results) != 1 { + return xdr.LedgerEntry{}, sql.ErrNoRows + } + ledgerEntryBase64 := results[0] + var result xdr.LedgerEntry + err = xdr.SafeUnmarshalBase64(ledgerEntryBase64, &result) + return result, err +} + +func flushLedgerEntryBatch(tx *sqlx.Tx, encodedKeyEntries map[string]*string) error { + upsertCount := 0 + upsertSQL := sq.Replace(ledgerEntriesTableName) + var deleteKeys = make([]interface{}, 0, len(encodedKeyEntries)) + for key, entry := range encodedKeyEntries { + if entry != nil { + upsertSQL = upsertSQL.Values(key, entry) + upsertCount += 1 + } else { + deleteKeys = append(deleteKeys, interface{}(key)) + } + } + + if upsertCount > 0 { + sqlStr, args, err := upsertSQL.ToSql() + if err != nil { + return err + } + if _, err = tx.Exec(sqlStr, args...); err != nil { + return err + } + } + + if len(deleteKeys) > 0 { + sqlStr, args, err := sq.Delete(ledgerEntriesTableName).Where(sq.Eq{"key": deleteKeys}).ToSql() + if err != nil { + return err + } + _, err = tx.Exec(sqlStr, args...) + if _, err = tx.Exec(sqlStr, args...); err != nil { + return err + } + } + return nil +} + +func getLatestLedgerSequence(tx *sqlx.Tx) (uint32, error) { + sqlStr, args, err := sq.Select("value").From(ledgerEntriesMetaTableName).Where(sq.Eq{"key": latestLedgerSequenceMetaKey}).ToSql() + if err != nil { + return 0, err + } + var results []string + if err := tx.Select(&results, sqlStr, args...); err != nil { + return 0, err + } + if len(results) != 1 { + return 0, ErrEmptyDB + } + latestLedgerStr := results[0] + latestLedger, err := strconv.ParseUint(latestLedgerStr, 10, 32) + if err != nil { + return 0, err + } + return uint32(latestLedger), nil +} + +func upsertLatestLedgerSequence(tx *sqlx.Tx, sequence uint32) error { + sql, args, err := sq.Replace(ledgerEntriesMetaTableName).Values(latestLedgerSequenceMetaKey, fmt.Sprintf("%d", sequence)).ToSql() + if err != nil { + return err + } + _, err = tx.Exec(sql, args...) + return err +} + +func (s *sqlDB) GetLatestLedgerSequence() (uint32, error) { + opts := sql.TxOptions{ + ReadOnly: true, + } + tx, err := s.db.BeginTxx(context.Background(), &opts) + if err != nil { + return 0, err + } + defer tx.Commit() + return getLatestLedgerSequence(tx) +} + +func (s *sqlDB) GetLedgerEntry(key xdr.LedgerKey) (xdr.LedgerEntry, bool, uint32, error) { + opts := sql.TxOptions{ + ReadOnly: true, + } + tx, err := s.db.BeginTxx(context.Background(), &opts) + if err != nil { + return xdr.LedgerEntry{}, false, 0, err + } + seq, err := getLatestLedgerSequence(tx) + if err != nil { + tx.Rollback() + return xdr.LedgerEntry{}, false, 0, err + } + buffer := xdr.NewEncodingBuffer() + entry, err := getLedgerEntry(tx, buffer, key) + if err != nil { + if err == sql.ErrNoRows { + return xdr.LedgerEntry{}, false, seq, nil + } + tx.Rollback() + return xdr.LedgerEntry{}, false, seq, err + } + tx.Commit() + return entry, true, seq, err +} + +func (s *sqlDB) Close() error { + // TODO: What if there is a running transaction? + return s.db.Close() +} + +type ledgerUpdaterTx struct { + tx *sqlx.Tx + // Value to set "latestSequence" to once we are done + forLedgerSequence uint32 + maxBatchSize int + buffer *xdr.EncodingBuffer + // nil implies deleted + keyToEntryBatch map[string]*string +} + +func (s *sqlDB) NewLedgerEntryUpdaterTx(forLedgerSequence uint32, maxBatchSize int) (LedgerEntryUpdaterTx, error) { + tx, err := s.db.BeginTxx(context.Background(), nil) + if err != nil { + return nil, err + } + return &ledgerUpdaterTx{ + maxBatchSize: maxBatchSize, + tx: tx, + forLedgerSequence: forLedgerSequence, + buffer: xdr.NewEncodingBuffer(), + keyToEntryBatch: make(map[string]*string, maxBatchSize), + }, nil +} + +func (l *ledgerUpdaterTx) UpsertLedgerEntry(key xdr.LedgerKey, entry xdr.LedgerEntry) error { + encodedKey, err := encodeLedgerKey(l.buffer, key) + if err != nil { + return err + } + encodedEntry, err := l.buffer.MarshalBase64(&entry) + if err != nil { + return err + } + l.keyToEntryBatch[encodedKey] = &encodedEntry + if len(l.keyToEntryBatch) >= l.maxBatchSize { + if err := flushLedgerEntryBatch(l.tx, l.keyToEntryBatch); err != nil { + l.tx.Rollback() + return err + } + // reset map + l.keyToEntryBatch = make(map[string]*string, maxBatchSize) + } + return nil +} + +func (l *ledgerUpdaterTx) DeleteLedgerEntry(key xdr.LedgerKey) error { + encodedKey, err := encodeLedgerKey(l.buffer, key) + if err != nil { + return err + } + l.keyToEntryBatch[encodedKey] = nil + if len(l.keyToEntryBatch) > l.maxBatchSize { + if err := flushLedgerEntryBatch(l.tx, l.keyToEntryBatch); err != nil { + l.tx.Rollback() + return err + } + // reset map + l.keyToEntryBatch = make(map[string]*string, maxBatchSize) + } + return nil +} + +func (l *ledgerUpdaterTx) Done() error { + if err := flushLedgerEntryBatch(l.tx, l.keyToEntryBatch); err != nil { + l.tx.Rollback() + return err + } + if err := upsertLatestLedgerSequence(l.tx, l.forLedgerSequence); err != nil { + return err + } + return l.tx.Commit() +} + +func encodeLedgerKey(buffer *xdr.EncodingBuffer, key xdr.LedgerKey) (string, error) { + // TODO: we probably want to base64-encode it before it goes into the DB + // this is safe since we are converting to string right away, which causes a copy + binKey, err := buffer.LedgerKeyUnsafeMarshalBinaryCompress(key) + if err != nil { + return "", err + } + return string(binKey), nil +} + +func runMigrations(db *sql.DB, dialect string) error { + m := &migrate.AssetMigrationSource{ + Asset: migrations.ReadFile, + AssetDir: func() func(string) ([]string, error) { + return func(path string) ([]string, error) { + dirEntry, err := migrations.ReadDir(path) + if err != nil { + return nil, err + } + entries := make([]string, 0) + for _, e := range dirEntry { + entries = append(entries, e.Name()) + } + + return entries, nil + } + }(), + Dir: "migrations", + } + _, err := migrate.ExecMax(db, dialect, m, migrate.Up, 0) + return err +} diff --git a/cmd/soroban-rpc/internal/ledgerentry_storage/db_test.go b/cmd/soroban-rpc/internal/ledgerentry_storage/db_test.go new file mode 100644 index 0000000000..efa597dc41 --- /dev/null +++ b/cmd/soroban-rpc/internal/ledgerentry_storage/db_test.go @@ -0,0 +1,211 @@ +package ledgerentry_storage + +import ( + "fmt" + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" + "math/rand" + "os" + "path" + "testing" +) + +func TestSimpleDB(t *testing.T) { + db, path := NewTestDB() + defer func() { + assert.NoError(t, db.Close()) + }() + defer os.Remove(path) + + // Check that we get an empty DB error + _, err := db.GetLatestLedgerSequence() + assert.Equal(t, ErrEmptyDB, err) + + // Fill the DB with a single entry and fetch it + ledgerSequence := uint32(23) + tx, err := db.NewLedgerEntryUpdaterTx(ledgerSequence, 150) + assert.NoError(t, err) + + four := xdr.Uint32(4) + six := xdr.Uint32(6) + data := xdr.ContractDataEntry{ + ContractId: xdr.Hash{0xca, 0xfe}, + Key: xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: &four, + }, + Val: xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: &six, + }, + } + key, entry := getContractDataLedgerEntry(data) + err = tx.UpsertLedgerEntry(key, entry) + assert.NoError(t, err) + err = tx.Done() + assert.NoError(t, err) + + obtainedEntry, present, obtainedLedgerSequence, err := db.GetLedgerEntry(key) + assert.NoError(t, err) + assert.True(t, present) + assert.Equal(t, ledgerSequence, obtainedLedgerSequence) + assert.Equal(t, obtainedEntry.Data.Type, xdr.LedgerEntryTypeContractData) + assert.Equal(t, xdr.Hash{0xca, 0xfe}, obtainedEntry.Data.ContractData.ContractId) + assert.Equal(t, six, *obtainedEntry.Data.ContractData.Val.U32) + + obtainedLedgerSequence, err = db.GetLatestLedgerSequence() + assert.Equal(t, ledgerSequence, obtainedLedgerSequence) + + // Do another round, overwriting the ledger entry + ledgerSequence = uint32(24) + tx, err = db.NewLedgerEntryUpdaterTx(ledgerSequence, 150) + assert.NoError(t, err) + eight := xdr.Uint32(8) + entry.Data.ContractData.Val.U32 = &eight + + err = tx.UpsertLedgerEntry(key, entry) + assert.NoError(t, err) + + err = tx.Done() + assert.NoError(t, err) + + obtainedEntry, present, obtainedLedgerSequence, err = db.GetLedgerEntry(key) + assert.NoError(t, err) + assert.True(t, present) + assert.Equal(t, ledgerSequence, obtainedLedgerSequence) + assert.Equal(t, eight, *obtainedEntry.Data.ContractData.Val.U32) + + // Do another round, deleting the ledger entry + ledgerSequence = uint32(25) + tx, err = db.NewLedgerEntryUpdaterTx(ledgerSequence, 150) + assert.NoError(t, err) + + err = tx.DeleteLedgerEntry(key) + assert.NoError(t, err) + err = tx.Done() + assert.NoError(t, err) + _, present, _, err = db.GetLedgerEntry(key) + assert.NoError(t, err) + assert.False(t, present) + + obtainedLedgerSequence, err = db.GetLatestLedgerSequence() + assert.Equal(t, ledgerSequence, obtainedLedgerSequence) +} + +func getContractDataLedgerEntry(data xdr.ContractDataEntry) (xdr.LedgerKey, xdr.LedgerEntry) { + entry := xdr.LedgerEntry{ + LastModifiedLedgerSeq: 1, + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeContractData, + ContractData: &data, + }, + Ext: xdr.LedgerEntryExt{}, + } + var key xdr.LedgerKey + err := key.SetContractData(data.ContractId, data.Key) + if err != nil { + panic(err) + } + return key, entry +} + +func TestConcurrency(t *testing.T) { + // Make sure that reads can happen while a write-transaction is ongoing + // and writes are only visible once the transaction is committed + db, path := NewTestDB() + defer func() { + assert.NoError(t, db.Close()) + }() + defer os.Remove(path) + + // Check that we get an empty DB error + _, err := db.GetLatestLedgerSequence() + assert.Equal(t, ErrEmptyDB, err) + + // Start filling the DB with a single entry (enforce flushing right away) + ledgerSequence := uint32(23) + tx, err := db.NewLedgerEntryUpdaterTx(ledgerSequence, 0) + + assert.NoError(t, err) + four := xdr.Uint32(4) + six := xdr.Uint32(6) + data := xdr.ContractDataEntry{ + ContractId: xdr.Hash{0xca, 0xfe}, + Key: xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: &four, + }, + Val: xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: &six, + }, + } + key, entry := getContractDataLedgerEntry(data) + err = tx.UpsertLedgerEntry(key, entry) + assert.NoError(t, err) + + // Before committing the changes make sure we can query the DB + _, err = db.GetLatestLedgerSequence() + assert.Equal(t, ErrEmptyDB, err) + _, _, _, err = db.GetLedgerEntry(key) + assert.Equal(t, ErrEmptyDB, err) + + // Finish the transaction and check that the results are present + err = tx.Done() + assert.NoError(t, err) + + obtainedLedgerSequence, err := db.GetLatestLedgerSequence() + assert.Equal(t, ledgerSequence, obtainedLedgerSequence) + + obtainedEntry, present, obtainedLedgerSequence, err := db.GetLedgerEntry(key) + assert.NoError(t, err) + assert.True(t, present) + assert.Equal(t, ledgerSequence, obtainedLedgerSequence) + assert.Equal(t, six, *obtainedEntry.Data.ContractData.Val.U32) +} + +func BenchmarkLedgerUpdate(b *testing.B) { + db, path := NewTestDB() + defer db.Close() + defer os.Remove(path) + keyUint32 := xdr.Uint32(0) + data := xdr.ContractDataEntry{ + ContractId: xdr.Hash{0xca, 0xfe}, + Key: xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: &keyUint32, + }, + Val: xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: &keyUint32, + }, + } + key, entry := getContractDataLedgerEntry(data) + const numEntriesPerOp = 3500 + b.ResetTimer() + for i := 0; i < b.N; i++ { + tx, err := db.NewLedgerEntryUpdaterTx(uint32(i+1), maxBatchSize) + if err != nil { + panic(err) + } + for j := 0; j < numEntriesPerOp; j++ { + keyUint32 = xdr.Uint32(j) + if err := tx.UpsertLedgerEntry(key, entry); err != nil { + panic(err) + } + } + if err := tx.Done(); err != nil { + panic(err) + } + } + b.StopTimer() +} + +func NewTestDB() (DB, string) { + path := path.Join(os.TempDir(), fmt.Sprintf("%08x.sqlite", rand.Int63())) + db, err := OpenSQLiteDB(path) + if err != nil { + panic(err) + } + return db, path +} diff --git a/cmd/soroban-rpc/internal/ledgerentry_storage/ledgerentry_storage.go b/cmd/soroban-rpc/internal/ledgerentry_storage/ledgerentry_storage.go new file mode 100644 index 0000000000..84d097988c --- /dev/null +++ b/cmd/soroban-rpc/internal/ledgerentry_storage/ledgerentry_storage.go @@ -0,0 +1,238 @@ +package ledgerentry_storage + +import ( + "context" + "fmt" + "io" + "sync" + "time" + + "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest" + backends "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" +) + +// TODO: Make this configurable? +const maxBatchSize = 150 + +type LedgerEntryStorage interface { + GetLedgerEntry(key xdr.LedgerKey) (xdr.LedgerEntry, bool, uint32, error) + io.Closer +} + +func NewLedgerEntryStorage(logger *log.Entry, db DB, networkPassPhrase string, archive historyarchive.ArchiveInterface, ledgerBackend backends.LedgerBackend) (LedgerEntryStorage, error) { + ctx, done := context.WithCancel(context.Background()) + ls := ledgerEntryStorage{ + logger: logger, + db: db, + networkPassPhrase: networkPassPhrase, + done: done, + } + ls.wg.Add(1) + go ls.run(ctx, archive, ledgerBackend) + return &ls, nil +} + +type ledgerEntryStorage struct { + logger *log.Entry + db DB + networkPassPhrase string + done context.CancelFunc + wg sync.WaitGroup +} + +func (ls *ledgerEntryStorage) GetLedgerEntry(key xdr.LedgerKey) (xdr.LedgerEntry, bool, uint32, error) { + return ls.db.GetLedgerEntry(key) +} + +func (ls *ledgerEntryStorage) Close() error { + ls.done() + ls.wg.Wait() + ls.db.Close() + return nil +} + +func (ls *ledgerEntryStorage) fillEntriesFromLatestCheckpoint(ctx context.Context, archive historyarchive.ArchiveInterface) (uint32, error) { + root, err := archive.GetRootHAS() + if err != nil { + return 0, err + } + startCheckpointLedger := root.CurrentLedger + + ls.logger.Infof("Starting processing of checkpoint %d", startCheckpointLedger) + // TODO: should we make this configurable? + checkpointCtx, cancelCheckpointCtx := context.WithTimeout(ctx, 30*time.Minute) + defer cancelCheckpointCtx() + reader, err := ingest.NewCheckpointChangeReader(checkpointCtx, archive, startCheckpointLedger) + if err != nil { + return 0, err + } + tx, err := ls.db.NewLedgerEntryUpdaterTx(startCheckpointLedger, maxBatchSize) + if err != nil { + return 0, err + } + // Make sure we finish the updating transaction + entryCount := 0 + + for { + select { + case <-ctx.Done(): + cancelCheckpointCtx() + return 0, context.Canceled + default: + } + change, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout + panic(err) + } + + entry := change.Post + key, relevant, err := getRelevantLedgerKeyFromData(entry.Data) + if err != nil { + return 0, err + } + if !relevant { + continue + } + tx.UpsertLedgerEntry(key, *entry) + entryCount++ + + if entryCount%10000 == 0 { + ls.logger.Infof(" processed %d checkpoint ledger entry changes", entryCount) + } + } + + ls.logger.Info("Committing ledger entries") + if err = tx.Done(); err != nil { + return 0, err + } + + ls.logger.Info("Finished checkpoint processing") + return startCheckpointLedger, nil +} + +func (ls *ledgerEntryStorage) run(ctx context.Context, archive historyarchive.ArchiveInterface, ledgerBackend backends.LedgerBackend) { + defer ls.wg.Done() + + // First, make sure the DB has a complete ledger entry baseline + + startCheckpointLedger, err := ls.db.GetLatestLedgerSequence() + if err != nil && err != ErrEmptyDB { + // TODO: implement retries? + panic(err) + } + if err == ErrEmptyDB { + // DB is empty, let's fill it from a checkpoint + ls.logger.Infof("Found an empty database, filling it in from the most recent checkpoint (this can take up to 30 minutes, depending on the network)") + startCheckpointLedger, err = ls.fillEntriesFromLatestCheckpoint(ctx, archive) + // TODO: implement retries? + if err != nil { + panic(err) + } + } + + // Secondly, continuously process txmeta deltas + + // TODO: we can probably do the preparation in parallel with the checkpoint processing above + prepareRangeCtx, cancelPrepareRange := context.WithTimeout(ctx, 30*time.Minute) + if err := ledgerBackend.PrepareRange(prepareRangeCtx, backends.UnboundedRange(startCheckpointLedger)); err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout + panic(err) + } + cancelPrepareRange() + + nextLedger := startCheckpointLedger + 1 + for { + fmt.Println("Processing txmeta of ledger", nextLedger) + reader, err := ingest.NewLedgerChangeReader(ctx, ledgerBackend, ls.networkPassPhrase, nextLedger) + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + tx, err := ls.db.NewLedgerEntryUpdaterTx(nextLedger, maxBatchSize) + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + + for { + change, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + if change.Post == nil { + key, relevant, err := getRelevantLedgerKeyFromData(change.Pre.Data) + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + if !relevant { + continue + } + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + err = tx.DeleteLedgerEntry(key) + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + } else { + key, relevant, err := getRelevantLedgerKeyFromData(change.Pre.Data) + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + if !relevant { + continue + } + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + err = tx.UpsertLedgerEntry(key, *change.Post) + if err != nil { + // TODO: we probably shouldn't panic, at least in case of timeout/cancellation + panic(err) + } + } + } + tx.Done() + nextLedger++ + reader.Close() + } + +} + +func getRelevantLedgerKeyFromData(data xdr.LedgerEntryData) (xdr.LedgerKey, bool, error) { + var key xdr.LedgerKey + switch data.Type { + case xdr.LedgerEntryTypeAccount: + if err := key.SetAccount(data.Account.AccountId); err != nil { + return xdr.LedgerKey{}, false, err + } + case xdr.LedgerEntryTypeTrustline: + if err := key.SetTrustline(data.TrustLine.AccountId, data.TrustLine.Asset); err != nil { + return xdr.LedgerKey{}, false, err + } + case xdr.LedgerEntryTypeContractData: + if err := key.SetContractData(data.ContractData.ContractId, data.ContractData.Val); err != nil { + return xdr.LedgerKey{}, false, err + } + default: + // we don't care about any other entry types for now + return xdr.LedgerKey{}, false, nil + } + return key, true, nil +} diff --git a/cmd/soroban-rpc/internal/ledgerentry_storage/migrations/01_init.sql b/cmd/soroban-rpc/internal/ledgerentry_storage/migrations/01_init.sql new file mode 100644 index 0000000000..0dd53afdc9 --- /dev/null +++ b/cmd/soroban-rpc/internal/ledgerentry_storage/migrations/01_init.sql @@ -0,0 +1,20 @@ +-- +migrate Up +CREATE TABLE ledger_entries ( + key bigint NOT NULL PRIMARY KEY, + entry TEXT NOT NULL +); + +-- metadata about the content in the ledger_entries table +CREATE TABLE ledger_entries_meta ( + key TEXT NOT NULL PRIMARY KEY, + value TEXT NOT NULL +); + + + +CREATE INDEX ledger_entries_key ON ledger_entries(key); + + +-- +migrate Down +drop table ledger_entries cascade; +drop table ledger_entries_meta cascade; diff --git a/cmd/soroban-rpc/main.go b/cmd/soroban-rpc/main.go index 36c35b8dc6..68e5acc7ff 100644 --- a/cmd/soroban-rpc/main.go +++ b/cmd/soroban-rpc/main.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/pkg/profile" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -19,8 +20,10 @@ import ( ) func main() { - var endpoint, horizonURL, binaryPath, configPath, networkPassphrase string - var captiveCoreHTTPPort uint16 + // CPU profiling by default + defer profile.Start().Stop() + var endpoint, horizonURL, binaryPath, configPath, networkPassphrase, dbPath string + var captiveCoreHTTPPort uint var historyArchiveURLs []string var txConcurrency, txQueueSize int var logLevel logrus.Level @@ -43,13 +46,12 @@ func main() { Usage: "URL used to query Horizon", }, &config.ConfigOption{ - Name: "stellar-captive-core-http-port", - ConfigKey: &captiveCoreHTTPPort, - OptType: types.Uint, - CustomSetValue: config.SetOptionalUint, - Required: false, - FlagDefault: uint(11626), - Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", + Name: "stellar-captive-core-http-port", + ConfigKey: &captiveCoreHTTPPort, + OptType: types.Uint, + Required: false, + FlagDefault: uint(11626), + Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", }, &config.ConfigOption{ Name: "log-level", @@ -121,6 +123,14 @@ func main() { FlagDefault: 10, Required: false, }, + { + Name: "db-path", + Usage: "SQLite DB path", + OptType: types.String, + ConfigKey: &dbPath, + FlagDefault: "soroban_rpc.sqlite", + Required: false, + }, } cmd := &cobra.Command{ Use: "soroban-rpc", @@ -137,12 +147,13 @@ func main() { HorizonURL: horizonURL, StellarCoreBinaryPath: binaryPath, CaptiveCoreConfigPath: configPath, - CaptiveCoreHTTPPort: captiveCoreHTTPPort, + CaptiveCoreHTTPPort: uint16(captiveCoreHTTPPort), NetworkPassphrase: networkPassphrase, HistoryArchiveURLs: historyArchiveURLs, LogLevel: logLevel, TxConcurrency: txConcurrency, TxQueueSize: txQueueSize, + SQLiteDBPath: dbPath, } exitCode := daemon.Start(config) os.Exit(exitCode) diff --git a/go.mod b/go.mod index cabdcedca1..03606f85c4 100644 --- a/go.mod +++ b/go.mod @@ -5,73 +5,82 @@ go 1.18 require ( github.com/creachadair/jrpc2 v0.41.1 github.com/go-git/go-git/v5 v5.4.2 + github.com/mattn/go-sqlite3 v1.9.0 + github.com/pkg/profile v1.7.0 + github.com/rubenv/sql-migrate v0.0.0-20190717103323-87ce952f7079 github.com/sirupsen/logrus v1.4.1 github.com/spf13/cobra v0.0.0-20160830174925-9c28e4bbd74e github.com/spf13/viper v0.0.0-20150621231900-db7ff930a189 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.0 golang.org/x/mod v0.6.0 ) require ( - github.com/Masterminds/squirrel v1.5.0 // indirect - github.com/aws/aws-sdk-go v1.39.5 // indirect - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jmoiron/sqlx v1.2.0 // indirect - github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect - github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/lib/pq v1.2.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect - github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect - github.com/prometheus/common v0.2.0 // indirect - github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - google.golang.org/protobuf v1.26.0 // indirect + github.com/felixge/fgprof v0.9.3 // indirect + github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect + gopkg.in/gorp.v1 v1.7.1 // indirect +) + +require ( github.com/BurntSushi/toml v0.3.1 // indirect + github.com/Masterminds/squirrel v1.5.0 github.com/Microsoft/go-winio v0.4.16 // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/go-chi/chi v4.0.3+incompatible // indirect - github.com/go-errors/errors v0.0.0-20150906023321-a41850380601 // indirect - github.com/pelletier/go-toml v1.9.0 - github.com/pkg/errors v0.9.1 // indirect - github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 - github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 // indirect - github.com/spf13/pflag v0.0.0-20161005214240-4bd69631f475 // indirect - github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - gopkg.in/tylerb/graceful.v1 v1.2.13 // indirect github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/aws/aws-sdk-go v1.39.5 // indirect + github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.12.0 // indirect + github.com/go-chi/chi v4.0.3+incompatible // indirect + github.com/go-errors/errors v0.0.0-20150906023321-a41850380601 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/schema v1.1.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jmoiron/sqlx v1.2.0 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/lib/pq v1.2.0 // indirect github.com/magiconair/properties v1.5.4 // indirect github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366 // indirect + github.com/pelletier/go-toml v1.9.0 + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect + github.com/prometheus/common v0.2.0 // indirect + github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 // indirect + github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect + github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94 // indirect github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431 // indirect + github.com/spf13/pflag v0.0.0-20161005214240-4bd69631f475 // indirect github.com/stellar/go v0.0.0-20221221170028-a8148965dfd4 - github.com/stretchr/objx v0.3.0 // indirect + github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect golang.org/x/crypto v0.1.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.1.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.1.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/tylerb/graceful.v1 v1.2.13 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index dacd63a848..3c003e93fb 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,9 @@ github.com/aws/aws-sdk-go v1.39.5 h1:yoJEE1NJxbpZ3CtPxvOSFJ9ByxiXmBTKk8J+XU5ldtg github.com/aws/aws-sdk-go v1.39.5/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/creachadair/jrpc2 v0.41.1 h1:GnSQNk+vt8B/oayJlfOXVRi4hg8DuB9NsppFGe8iVJ0= github.com/creachadair/jrpc2 v0.41.1/go.mod h1:k2mGfjsgE2h2Vo12C9NzZguUzzl3gnfGCmLIvg84pVE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -32,6 +35,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -56,6 +61,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/packr v1.12.1 h1:+5u3rqgdhswdYXhrX6DHaO7BM4P8oxrbvgZm9H1cRI4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -66,9 +72,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 h1:oERTZ1buOUYlpmKaqlO5fYmz8cZ1rYu5DieJzF4ZVmU= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY= github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= @@ -130,6 +139,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -148,6 +159,8 @@ github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwde github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= +github.com/rubenv/sql-migrate v0.0.0-20190717103323-87ce952f7079 h1:xPeaaIHjF9j8jbYQ5xdvLnFp+lpmGYFG1uBPtXNBHno= +github.com/rubenv/sql-migrate v0.0.0-20190717103323-87ce952f7079/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 h1:S4OC0+OBKz6mJnzuHioeEat74PuQ4Sgvbf8eus695sc= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2/go.mod h1:8zLRYR5npGjaOXgPSKat5+oOh+UHd8OdbS18iqX9F6Y= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= @@ -171,13 +184,14 @@ github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee h1:fbVs0xmXpBvVS4GB github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= -github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= @@ -189,6 +203,7 @@ github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce h1:cVSRGH8cOv github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb h1:06WAhQa+mYv7BiOk13B/ywyTlkoE/S7uu6TBKU6FHnE= github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M= github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce h1:888GrqRxabUce7lj4OaoShPxodm3kXOMpSa85wdYzfY= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -220,6 +235,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -241,6 +257,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 h1:r5ptJ1tBxVAeqw4CrYWhXIMr0SybY3CDHuIbCg5CFVw= +gopkg.in/gorp.v1 v1.7.1 h1:GBB9KrWRATQZh95HJyVGUZrWwOPswitEYEyqlK8JbAA= +gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tylerb/graceful.v1 v1.2.13 h1:UWJlWJHZepntB0PJ9RTgW3X+zVLjfmWbx/V1X/V/XoA= gopkg.in/tylerb/graceful.v1 v1.2.13/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2PAbAC/uJ8= @@ -252,5 +270,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=