From 508a4fc36661eb3c1fe367d79c0d4c883c5cffdf Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 5 Jul 2024 13:51:37 -0700 Subject: [PATCH 1/5] services/horizon: Reingest from precomputed TxMeta --- services/horizon/cmd/db.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 8f3e2ae9eb..bb17dfcb20 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -272,6 +272,11 @@ var ( ledgerBackendType ingest.LedgerBackendType ) +type StorageBackendConfig struct { + DataStoreConfig datastore.DataStoreConfig `toml:"datastore_config"` + BufferedStorageBackendConfig ledgerbackend.BufferedStorageBackendConfig `toml:"buffered_storage_backend_config"` +} + func ingestRangeCmdOpts() support.ConfigOptions { return support.ConfigOptions{ { From fb141141e969bb74ec38e296a937fa1a3c531f25 Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 9 Jul 2024 09:55:08 -0700 Subject: [PATCH 2/5] Global network config for reingestion --- services/horizon/cmd/db.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index bb17dfcb20..8f3e2ae9eb 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -272,11 +272,6 @@ var ( ledgerBackendType ingest.LedgerBackendType ) -type StorageBackendConfig struct { - DataStoreConfig datastore.DataStoreConfig `toml:"datastore_config"` - BufferedStorageBackendConfig ledgerbackend.BufferedStorageBackendConfig `toml:"buffered_storage_backend_config"` -} - func ingestRangeCmdOpts() support.ConfigOptions { return support.ConfigOptions{ { From 4908a1dc963517b8454d0571bb10cc5c0443fb97 Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 9 Jul 2024 16:48:06 -0700 Subject: [PATCH 3/5] Add unit test for new --ledgerbackend flag --- services/horizon/cmd/db_test.go | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index 3942f8d91a..ca51dfbe7e 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -96,3 +96,75 @@ func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForBuffered() { require.NoError(s.T(), RootCmd.Execute()) require.Equal(s.T(), parallelJobSize, uint32(5)) } + +func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() { + tests := []struct { + name string + args []string + ledgerBackend ingest.LedgerBackendType + expectError bool + errorMessage string + }{ + { + name: "default ledgerbackend", + args: []string{"1", "100"}, + expectError: false, + }, + { + name: "captive-core ledgerbackend", + args: []string{"1", "100", "--ledgerbackend", "captive-core"}, + expectError: false, + }, + { + name: "invalid ledgerbackend", + args: []string{"1", "100", "--ledgerbackend", "unknown"}, + expectError: true, + errorMessage: "invalid ledger backend: unknown, must be 'captive-core' or 'datastore'", + }, + { + name: "datastore ledgerbackend without config", + args: []string{"1", "100", "--ledgerbackend", "datastore"}, + expectError: true, + errorMessage: "datastore config file is required for datastore backend type", + }, + { + name: "datastore ledgerbackend missing config file", + args: []string{"1", "100", "--ledgerbackend", "datastore", "--datastore-config", "invalid.config.toml"}, + expectError: true, + errorMessage: "failed to load config file", + }, + { + name: "datastore ledgerbackend", + args: []string{"1", "100", "--ledgerbackend", "datastore", "--datastore-config", "../config.storagebackend.toml"}, + expectError: false, + }, + } + + commands := []struct { + cmd []string + name string + }{ + {[]string{"db", "reingest", "range"}, "TestDbReingestRangeCmd"}, + {[]string{"db", "fill-gaps"}, "TestDbFillGapsCmd"}, + } + + for _, command := range commands { + for _, tt := range tests { + s.T().Run(tt.name+"_"+command.name, func(t *testing.T) { + args := append(command.cmd, tt.args...) + RootCmd.SetArgs(append([]string{ + "--db-url", s.dsn, + "--network", "testnet", + }, args...)) + + if tt.expectError { + err := RootCmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), tt.errorMessage) + } else { + require.NoError(t, RootCmd.Execute()) + } + }) + } + } +} From 7ecb0a47be5d242e41ce7f2b5cf297451040279f Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 10 Jul 2024 10:22:57 -0700 Subject: [PATCH 4/5] Fix unit test --- services/horizon/cmd/db_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index ca51dfbe7e..a58799b8d0 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -3,12 +3,13 @@ package cmd import ( "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + horizon "github.com/stellar/go/services/horizon/internal" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/support/db/dbtest" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" ) func TestDBCommandsTestSuite(t *testing.T) { @@ -155,6 +156,7 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() { RootCmd.SetArgs(append([]string{ "--db-url", s.dsn, "--network", "testnet", + "--stellar-core-binary-path", "/test/core/bin/path", }, args...)) if tt.expectError { From 776bb50fecdf343a90fd7482a2d0d85b72357cd1 Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 10 Jul 2024 17:50:24 -0700 Subject: [PATCH 5/5] Add unittest for network flags validation --- services/horizon/cmd/db_test.go | 131 ++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 16 deletions(-) diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index a58799b8d0..dea3e3777f 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -22,6 +22,20 @@ type DBCommandsTestSuite struct { dsn string } +func (s *DBCommandsTestSuite) SetupTest() { + resetFlags() +} + +func resetFlags() { + RootCmd.ResetFlags() + dbFillGapsCmd.ResetFlags() + dbReingestRangeCmd.ResetFlags() + + globalFlags.Init(RootCmd) + dbFillGapsCmdOpts.Init(dbFillGapsCmd) + dbReingestRangeCmdOpts.Init(dbReingestRangeCmd) +} + func (s *DBCommandsTestSuite) SetupSuite() { runDBReingestRangeFn = func([]history.LedgerRange, bool, uint, horizon.Config, ingest.StorageBackendConfig) error { @@ -107,38 +121,121 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() { errorMessage string }{ { - name: "default ledgerbackend", - args: []string{"1", "100"}, + name: "default; w/ individual network flags", + args: []string{ + "1", "100", + "--network-passphrase", "passphrase", + "--history-archive-urls", "[]", + }, + expectError: false, + }, + { + name: "default; w/o individual network flags", + args: []string{ + "1", "100", + }, + expectError: true, + errorMessage: "network-passphrase must be set", + }, + { + name: "default; no history-archive-urls flag", + args: []string{ + "1", "100", + "--network-passphrase", "passphrase", + }, + expectError: true, + errorMessage: "history-archive-urls must be set", + }, + { + name: "default; w/ network parameter", + args: []string{ + "1", "100", + "--network", "testnet", + }, expectError: false, }, { - name: "captive-core ledgerbackend", - args: []string{"1", "100", "--ledgerbackend", "captive-core"}, + name: "datastore; w/ individual network flags", + args: []string{ + "1", "100", + "--ledgerbackend", "datastore", + "--datastore-config", "../config.storagebackend.toml", + "--network-passphrase", "passphrase", + "--history-archive-urls", "[]", + }, expectError: false, }, { - name: "invalid ledgerbackend", - args: []string{"1", "100", "--ledgerbackend", "unknown"}, + name: "datastore; w/o individual network flags", + args: []string{ + "1", "100", + "--ledgerbackend", "datastore", + "--datastore-config", "../config.storagebackend.toml", + }, expectError: true, - errorMessage: "invalid ledger backend: unknown, must be 'captive-core' or 'datastore'", + errorMessage: "network-passphrase must be set", }, { - name: "datastore ledgerbackend without config", - args: []string{"1", "100", "--ledgerbackend", "datastore"}, + name: "datastore; no history-archive-urls flag", + args: []string{ + "1", "100", + "--ledgerbackend", "datastore", + "--datastore-config", "../config.storagebackend.toml", + "--network-passphrase", "passphrase", + }, expectError: true, - errorMessage: "datastore config file is required for datastore backend type", + errorMessage: "history-archive-urls must be set", + }, + { + name: "captive-core; valid", + args: []string{ + "1", "100", + "--network", "testnet", + "--ledgerbackend", "captive-core", + }, + expectError: false, + }, + { + name: "invalid datastore", + args: []string{ + "1", "100", + "--network", "testnet", + "--ledgerbackend", "unknown", + }, + expectError: true, + errorMessage: "invalid ledger backend: unknown, must be 'captive-core' or 'datastore'", }, { - name: "datastore ledgerbackend missing config file", - args: []string{"1", "100", "--ledgerbackend", "datastore", "--datastore-config", "invalid.config.toml"}, + name: "datastore; missing config file", + args: []string{ + "1", "100", + "--network", "testnet", + "--ledgerbackend", "datastore", + "--datastore-config", "invalid.config.toml", + }, expectError: true, errorMessage: "failed to load config file", }, { - name: "datastore ledgerbackend", - args: []string{"1", "100", "--ledgerbackend", "datastore", "--datastore-config", "../config.storagebackend.toml"}, + name: "datastore; w/ config", + args: []string{ + "1", "100", + "--network", "testnet", + "--ledgerbackend", "datastore", + "--datastore-config", "../config.storagebackend.toml", + }, expectError: false, }, + { + name: "datastore; w/o config", + args: []string{ + "1", "100", + "--network", "testnet", + "--ledgerbackend", "datastore", + }, + expectError: true, + errorMessage: "datastore config file is required for datastore backend type", + }, } commands := []struct { @@ -152,10 +249,12 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() { for _, command := range commands { for _, tt := range tests { s.T().Run(tt.name+"_"+command.name, func(t *testing.T) { - args := append(command.cmd, tt.args...) + resetFlags() + + var args []string + args = append(command.cmd, tt.args...) RootCmd.SetArgs(append([]string{ "--db-url", s.dsn, - "--network", "testnet", "--stellar-core-binary-path", "/test/core/bin/path", }, args...))