From c8fd26ea81a11f173a0c53c429dda6c964011ae5 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Wed, 10 Jul 2024 15:11:49 -0700 Subject: [PATCH 01/13] #4911: added reingest cmd integration test for datastore ingest --- go.mod | 1 + go.sum | 2 + services/horizon/cmd/db.go | 1 + services/horizon/cmd/db_test.go | 19 +++-- .../horizon/internal/integration/db_test.go | 74 +++++++++++++++++++ 5 files changed, 89 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 531a7ecd3f..26c78fd61b 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,7 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/otiai10/copy v1.14.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect diff --git a/go.sum b/go.sum index 13d3a0acf0..50c8ee05ef 100644 --- a/go.sum +++ b/go.sum @@ -401,6 +401,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 8f3e2ae9eb..a6ae341a07 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -101,6 +101,7 @@ func migrate(dir schema.MigrateDir, count int) error { if err != nil { return err } + defer dbConn.Close() numMigrationsRun, err := schema.Migrate(dbConn.DB.DB, dir, count) if err != nil { diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index 3942f8d91a..2386cff509 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -18,7 +18,7 @@ func TestDBCommandsTestSuite(t *testing.T) { type DBCommandsTestSuite struct { suite.Suite - dsn string + db *dbtest.DB } func (s *DBCommandsTestSuite) SetupSuite() { @@ -27,18 +27,21 @@ func (s *DBCommandsTestSuite) SetupSuite() { return nil } - newDB := dbtest.Postgres(s.T()) - s.dsn = newDB.DSN + s.db = dbtest.Postgres(s.T()) RootCmd.SetArgs([]string{ - "db", "migrate", "up", "--db-url", s.dsn}) + "db", "migrate", "up", "--db-url", s.db.DSN}) require.NoError(s.T(), RootCmd.Execute()) } +func (s *DBCommandsTestSuite) TearDownSuite() { + s.db.Close() +} + func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { RootCmd.SetArgs([]string{ "db", "reingest", "range", - "--db-url", s.dsn, + "--db-url", s.db.DSN, "--network", "testnet", "--parallel-workers", "2", "--ledgerbackend", "datastore", @@ -53,7 +56,7 @@ func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForCaptiveBackend() { RootCmd.SetArgs([]string{ "db", "reingest", "range", - "--db-url", s.dsn, + "--db-url", s.db.DSN, "--network", "testnet", "--stellar-core-binary-path", "/test/core/bin/path", "--parallel-workers", "2", @@ -68,7 +71,7 @@ func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForCaptiveBackend() { func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForCaptive() { RootCmd.SetArgs([]string{ "db", "reingest", "range", - "--db-url", s.dsn, + "--db-url", s.db.DSN, "--network", "testnet", "--stellar-core-binary-path", "/test/core/bin/path", "--parallel-workers", "2", @@ -84,7 +87,7 @@ func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForCaptive() { func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForBuffered() { RootCmd.SetArgs([]string{ "db", "reingest", "range", - "--db-url", s.dsn, + "--db-url", s.db.DSN, "--network", "testnet", "--parallel-workers", "2", "--parallel-job-size", "5", diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 1f1d2277ec..79d86cefce 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -3,12 +3,16 @@ package integration import ( "context" "fmt" + "os" "path/filepath" "strconv" "testing" "time" + "github.com/fsouza/fake-gcs-server/fakestorage" + cp "github.com/otiai10/copy" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stellar/go/clients/horizonclient" "github.com/stellar/go/historyarchive" @@ -549,6 +553,67 @@ func TestReingestDB(t *testing.T) { tt.NoError(horizoncmd.RootCmd.Execute(), "Repeat the same reingest range against db, should not have errors.") } +func TestReingestDatastore(t *testing.T) { + os.Setenv("HORIZON_INTEGRATION_TESTS_ENABLED", "true") + + if os.Getenv("HORIZON_INTEGRATION_TESTS_ENABLED") == "" { + t.Skip("skipping integration test: HORIZON_INTEGRATION_TESTS_ENABLED not set") + } + + newDB := dbtest.Postgres(t) + defer newDB.Close() + horizoncmd.RootCmd.SetArgs([]string{ + "db", "migrate", "up", "--db-url", newDB.DSN}) + require.NoError(t, horizoncmd.RootCmd.Execute()) + + testTempDir := t.TempDir() + tempSeedDataPath := filepath.Join(testTempDir, "data") + tempSeedBucketPath := filepath.Join(tempSeedDataPath, "path", "to", "my", "bucket") + tempSeedBucketFolder := filepath.Join(tempSeedBucketPath, "FFFFFFFF--0-63999") + if err := os.MkdirAll(tempSeedBucketFolder, 0777); err != nil { + t.Fatalf("unable to create seed data in temp path, %v", err) + } + + err := cp.Copy("./testdata/testbucket", tempSeedBucketFolder) + if err != nil { + t.Fatalf("unable to copy seed data files for fake gcs, %v", err) + } + + testWriter := &testWriter{test: t} + opts := fakestorage.Options{ + Scheme: "http", + Host: "127.0.0.1", + Port: uint16(0), + Writer: testWriter, + Seed: tempSeedDataPath, + StorageRoot: filepath.Join(testTempDir, "bucket"), + PublicHost: "127.0.0.1", + } + + gcsServer, err := fakestorage.NewServerWithOptions(opts) + + if err != nil { + t.Fatalf("couldn't start the fake gcs http server %v", err) + } + + defer gcsServer.Stop() + t.Logf("fake gcs server started at %v", gcsServer.URL()) + t.Setenv("STORAGE_EMULATOR_HOST", gcsServer.URL()) + + horizoncmd.RootCmd.SetArgs([]string{"db", + "reingest", + "range", + "--db-url", newDB.DSN, + "--network", "testnet", + "--parallel-workers", "1", + "--ledgerbackend", "datastore", + "--datastore-config", "../../config.storagebackend.toml", + "997", + "999"}) + + require.NoError(t, horizoncmd.RootCmd.Execute()) +} + func TestReingestDBWithFilterRules(t *testing.T) { itest, _ := initializeDBIntegrationTest(t) tt := assert.New(t) @@ -905,3 +970,12 @@ func TestResumeFromInitializedDB(t *testing.T) { tt.Eventually(successfullyResumed, 1*time.Minute, 1*time.Second) } + +type testWriter struct { + test *testing.T +} + +func (w *testWriter) Write(p []byte) (n int, err error) { + w.test.Log(string(p)) + return len(p), nil +} From 3e8efb2328ac8af09b9f90359f2275ce4410930f Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Wed, 10 Jul 2024 17:29:26 -0700 Subject: [PATCH 02/13] #4911: added integration test for db reingest with --ledgerbacked datastore --- .../horizon/internal/integration/db_test.go | 71 ++++++++++++++++++ .../testbucket/FFFFFC18--999.xdr.zstd | Bin 0 -> 1547 bytes .../testbucket/FFFFFC19--998.xdr.zstd | Bin 0 -> 2791 bytes .../testbucket/FFFFFC1A--997.xdr.zstd | Bin 0 -> 2814 bytes 4 files changed, 71 insertions(+) create mode 100644 services/horizon/internal/integration/testdata/testbucket/FFFFFC18--999.xdr.zstd create mode 100644 services/horizon/internal/integration/testdata/testbucket/FFFFFC19--998.xdr.zstd create mode 100644 services/horizon/internal/integration/testdata/testbucket/FFFFFC1A--997.xdr.zstd diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 79d86cefce..25b41c8328 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -3,6 +3,7 @@ package integration import ( "context" "fmt" + "net" "os" "path/filepath" "strconv" @@ -11,12 +12,15 @@ import ( "github.com/fsouza/fake-gcs-server/fakestorage" cp "github.com/otiai10/copy" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stellar/go/clients/horizonclient" + sdk "github.com/stellar/go/clients/horizonclient" "github.com/stellar/go/historyarchive" "github.com/stellar/go/keypair" + "github.com/stellar/go/network" hProtocol "github.com/stellar/go/protocols/horizon" horizoncmd "github.com/stellar/go/services/horizon/cmd" horizon "github.com/stellar/go/services/horizon/internal" @@ -29,6 +33,7 @@ import ( "github.com/stellar/go/support/db/dbtest" "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" + "github.com/stellar/throttled" ) func submitLiquidityPoolOps(itest *integration.Test, tt *assert.Assertions) (submittedOperations []txnbuild.Operation, lastLedger int32) { @@ -612,6 +617,72 @@ func TestReingestDatastore(t *testing.T) { "999"}) require.NoError(t, horizoncmd.RootCmd.Execute()) + + listener, webApp, webPort, err := dynamicHorizonWeb(newDB.DSN) + if err != nil { + t.Fatalf("couldn't create and start horizon web app on dynamic port %v", err) + } + + webAppDone := make(chan struct{}) + go func() { + defer close(webAppDone) + if err = listener.Close(); err != nil { + return + } + webApp.Serve() + }() + + defer func() { + webApp.Close() + select { + case <-webAppDone: + return + default: + } + }() + + horizonClient := &sdk.Client{ + HorizonURL: fmt.Sprintf("http://localhost:%v", webPort), + } + + // wait until the web server is up before continuing to test requests + require.Eventually(t, func() bool { + if _, err := horizonClient.Root(); err != nil { + return false + } + return true + }, time.Second*15, time.Millisecond*100) + + _, err = horizonClient.LedgerDetail(998) + require.NoError(t, err) +} + +func dynamicHorizonWeb(dsn string) (net.Listener, *horizon.App, uint, error) { + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + return nil, nil, 0, err + } + webPort := uint(listener.Addr().(*net.TCPAddr).Port) + + webApp, err := horizon.NewApp(horizon.Config{ + DatabaseURL: dsn, + Port: webPort, + NetworkPassphrase: network.TestNetworkPassphrase, + LogLevel: logrus.InfoLevel, + DisableTxSub: true, + Ingest: false, + ConnectionTimeout: 10 * time.Second, + RateQuota: &throttled.RateQuota{ + MaxRate: throttled.PerHour(1000), + MaxBurst: 100, + }, + }) + if err != nil { + listener.Close() + return nil, nil, 0, err + } + + return listener, webApp, webPort, nil } func TestReingestDBWithFilterRules(t *testing.T) { diff --git a/services/horizon/internal/integration/testdata/testbucket/FFFFFC18--999.xdr.zstd b/services/horizon/internal/integration/testdata/testbucket/FFFFFC18--999.xdr.zstd new file mode 100644 index 0000000000000000000000000000000000000000..b2627e7fc1a95a8a8b50255573b00837a9139fe2 GIT binary patch literal 1547 zcmV+m2K4zTwJ-f-d>HL70F+1o00ZX$0RW*Kpt{u7lbA9>L6Ecqj14{q<4!*qZuzMh zP0G2{H#7hM02H>0jR*+|S#fT3INCscK`Rw*TZaIx?A)0;VAeka0*M=yO9G9GLnHah zP^Fg$Z+jGgxkxm!bJ+0wCgOL7yaE6K003pef#CoF00000)pBXsVo>)E?%)BC-?e*|Cc2Ut*|;5);s=4qnfnhdCe9!Po4Lk zqK=;@G4?uQ92zh!?$qQ>EM=9Ot%0I7lI}id!z0bFbBbkHEhZ|%4XTvg000B$4dAx3 zr(^&rm_q;p0001F08C3j0001K+>^52wkQEkIjEE&{m$XZ)I?UvZ z0096100IC211VD9PxAEIZ=qko>q8obZG~TL#Ji@V^J;a^e=|9(E&vZgKmY&$nE?=e z(Q7O?Gf09ifgbm&&lJ&^if;f^a#Yhk9&LtJ;GFH!OPLr*dqp@jAUQN`PR1){Gk*_o z*GGK$*%)$y*X{rS02fGh=mG!$K#<4UO%36E$PtF+i~Lo5dgNEzn2DcL>fYWwxQLg_ z{@?46%)TPmL~W8sKr&r}OMK@?Tk)Q+Zh&2W21dwn2|&G~StAvNt?Gc*&g*Kn4m_1* z9^Y)vkLy&~9K$3>^~6$tbtDr+1UCEiDWF4;P<|U~0}|eth9{&AdqGiFs}E+SJC=tU zS!h2gDSHvpIb;XuYu~v2iHUsP;P=;f=3LCuaVZNr-8PG4e1-(M2d5X_F$Jb_cs>CM zHcU#))F{XR06=?@pT1XownH^QSmiQm)6msx@ohNO(3R&m3xIna2RZ+LiIbW2)oeiV zMv!gx`K<^GZYf6{m4RBYY)DlZ*a0pTSX!~Bla9h=+MbGE)9?J1C7NRX-jyPPWHl+) zGI{jSfdd_wwVPu#dg+r72loo6_ZR)_TidAm8VW{gDFw>kGX5baq!4r)kF90Ho2Y)d z5CtZ5RFNRF?8eO%a1Lg$=00001R9aI258vo`qwkTqK}Y<4^3hWeao;6d zi&BJdQukz1Z!wN2fByhWMN(HrRZ~PoRZ>p@06=%We)E72A2sa7WzrbhJjaiBda00g z!J9qIp(4A`a$>uUm}HXGjpGe00U*hc9~ze0|Eo) z0b&yK$R!%U0001M00G-^=iGm;G627vdj?J9FctKR3_&ZvaoI*l$#oHx)vyl{5m8et zcgVra!Jg;EH#bUG@-=2JT4=MFJzf)}?*1qE(<^Ie5hPwrDq>e~B$t!{^aTb20008| z@PGjT0Pc(d0R;gA0RaF30RjR80|W#G00t&70000000000fUcm?O%e(KBJxr(-~s^v zDG~?TFbF0fDf}k?gVF+O2r94XOQ5WP8iL9@{qki6)DWl))5lGjQ-uuUfG2Qy+VMhX ze-Ps8($IM1*wJ4ue4QPC#)cDoF;Jg$t8ah8{Q#NQ$MxS3J9t?%gb_wR*#o>V8NhNV z^AR2kcy{uJ<*qrK`&e=QxRLuYqfBPN;bd}FAW)H-jO(IvnnL x4AGaJAPA6rf30ZM&bU4~TeL*V~Zd-={t?b;H zI$+j60|JQv000y}k=G?0L9C5#{HrD$2Yy=k5bTOm5mdK;ZbuH)x-vXc7V45iUA;Au z`-&pa2zjNUUJwB(2NaAejwr|SF(Wnr0001G!hzfX0RR910JU8Zvb%jKPSwCVN2TGl zkjW2MgH=Zy#87*_f&|h$ivR!sKt$Q9swJ(3i20m~KsPsBOdI(@sTtvf0JIqs$>piJ zs!wMvC&E@uIgAx3EOTn$bY8e(zp$+#@BIy2sjKtl1HK?|&N-brL-)Iq7JM%bFq)fG zVCA4qWlGwmTmSnn=T~gfcRD@0F>P!6A$-{D&+~H{0cGy>#k<$~4(HlS^Z)<@<_+Ms zv!`SLDV2u+0ssI2WB^P{KmY&$Y21^t-nJ+KX^&+AdPx^WL^_Z@UW|w(Y?ri4tUAo( zi~s=u0006200+nY+qWhPKzBWtLO&R`nu72p1Gu)m{=~mN+SVGNyh#DD0001bWx|2> z00GM1GX5baq!4r)kF90Ho2Y)d5CtZ5RFNRF?8eO%ah|CZ}Y&ZP>Jgb}7f0}e*L0Jt(6SUXO}CN^gfMlqmG7r?2RjhSZ}Ku6>a z-$nh)!0$q$FXJJ1Bw3$z{&#$8BG0xlSqCyA$7VTU01rYy0001)0T6xBYb-c3NP;eb z9`~xx6w#Q9Zva$sRMS2lZH89hoK@U5M@JmGj;91?Avjw&sq)_=%Gr(e-r+niE*;pu zO8@`>7f5#K0ssI&u%y%q7CB}%O+`<6NQdg*FRfzVo+NqgFV2Qoh5-t$q#j1gx3LAS z!OBJ9Ag-vGLVZ10u0WMSQ@Abh4JcX-KvH|Ng67ZftwPOqqMQ+ILe=1;hD^odedzlt-kD`Lx6d3Zm+2iJi7-@n;F0r~5vPl& zY6wOE0006(R8>Jp0001I%GP6enhtf1wfb)^lUfKy%5*sEcW4keGp;I`qt^2P00006 zj(31lL;yH-AOHYBm}cT^5`a3${Y&3BXk19Z7&<2I_&2-iDkuW>FCaG-{G?@siC7(i z4-_TyeTs>I4qT6wT$b<^1j%Xvu+MxCK+Y*TCCc0}q{T~tyK*5M5wO3UThzwtt?&=8 z>iwkzpiy~7SqAWBM4`Xk8h_4OypQ9f8|PIJCS3P(F>Q%83u|gC0bJM-J%_>9r#F4? zfEWS#5B2LMHc88j41AzOUjB47mT zZgWMSd@_=sfTZDq$Oeq$@$X1w7Q%Y`30J{EQXD%|N0eJVnc3fj$LG#Wgg=AOGKGf( zF)YCx#%I)k+{^3E$+L?W{yBv}I+LSP<{9`ZP2&002Pe2ZZGDmJw=y zB;ky1K2_sR`lVA%Ly{3lgT3&Pgu#yzJGmP4CU8d;4SsOx~*pdSM@!ZUHCQO&gL9+kj zH8Tj4g+B2OOd5s!*jQS8VwZ>MCHnhDqF>?h=bGxD?ru%jedUlXt}FtKn|T8&<%tN3 zJ2KpPv&4$>Z-k;gV(p3cqTV!5RF-{1G{b=l4?s|02&TuJ8bEe{rqZCxX1^c-%t`Nh z5j%OpBMwTEmjFEp000023Nte~1OSl9E5DJ1;54ic1#S!xpR(*)nKCC(`^po0B@wT6 zl;BO^002N!n40XQy5Ll4C`!hQ@@SQ&RPSg>iU7KI+k2 zlDOZrO21f>%tXaLgi2L$WuuH6S3tc_@tWEU%i;?F2NVpITlUSG){s8r#zj=r>P<(b zI5)LG|H!i1?yZf$-C;BIa|*e2Ol{pfjoB?l3xT`ABRI2 zkf)qQ&zYGhlKeE&00EE$*BpHM=G?a82*Cs&x=G>N)2a{xY(AaS>5gz?d&oqU2yKR_rmV#UKaner`rYGbj+yDRq3FOff z`sUmQCNKa100000004m#pz*bt89)F6@|MB?p#=s2L#)H~xcJhQEaOZP#YI#{!U&XI zFK`nsm7p+J!4-fG;l&>A-spsM29A^s7qg8l&4#UgEL`O#vmS@W#qPdLf zNEnSWt%Wm_7kPd<&hbnwI6BkrwD8#szlWy2odzLK0-$&oN1zzO!g-m9C@!Nq5=M)1 zgaz)@Wu*=Beu5a>fIk#-xK$N7D7rqFVV;T6^N)Qre9>YuI#7WiVNGw`5^Nt8D07UP zydVTWd{85~!-Y1p(NSDPb0mxwWd}`zd|{?8qVzkQ8W6iC*@nBz$n(|M$>W5mtOr5t zdL4HdzzQIL1Z;Qa(bdlC&8$vEZ?|8Wl92&N4%P8UAh2#L|anNx!P(FyDR}LL&^hAxE z0I_PQhLw2kSghO&WaFt_ct+~sr~K>$qR6}l5_t&_goF`PzBq05({pgz!_K%T!Wqa1 z5TRj0$QP%retHg0d)OKGL^uQa03tL@2>If))lbjCX%9Q&o(N}PKEVVgG#ovBvAGGq zg`)UTBVgxWnyUN~ikYOUv)CDy)>qjAOnqPB zB_@nJd!bw0NMgjl~0nzM5voQMj`#4>>q@t>tl=aI}IU{pcyOP7fGJOWafekS5|{0hFyrlL?Iom298OC?Z9(||rT_!I*QIA^Hj8yM^y z>XRmpV=8~mE`96#0^(13uSmcnF1|$~AAHF!Je^GlR(aQDho@ymmS)GNYx4`1r#Ex@ zu&CCN4jms!<}7@3OyY`c%JaJuwRnh?G=P~2i4+@-0tl<8y2l94xZmZZiLn&c%J9Xi z*r@rMIO{ayu{#>vI=trt=)62-K-(|u>hbTUDwOf-kv!l&a_5(r5@v|N9A2p7a9X*2 z`Ae#D5bR<8pyc^I)v$~mzOoBk&|I!h$N7MgqebyXdA%-udfj0wW?7lMx7MjNjP@@9cs>Hzs z$H@r1y57aN*3zN$*~{D7ZBLwLYHE)`gP*OWon{0p$23!Mssj;i7cYz}91ssgB^TZI zH`|Zrmzt`Lx7vsOq=!nN4G{xc*YI?kafUKwE<4b-aV?^j<)Fw5sln%7_qbJ+Tb z697aXAw)l-$DLkp_kJ}!04E#+M-m5+_qyY07iQ59D=V0nJd~Q$RSI~2nv!k3fv;6p zgKaIX{&n5)C}A%2VB)7yQPN-}O7Ua&{f0n7CvyKp+Py zm!Qnal<=%anmh^n$1L~A*n3WK%D7v*-$g>N*Z*TvvPz@f-^H%|oMhdJ|EMv(Dp}h* zIFu-ku13?VsNmKUbUmKN(b)GvH2z1a2;AzdxYP`jrLf9N))!2$x`@scttTnwPB~?L zGF;Pb-MhK2MH0P;S-UJkeq(Vm`q15%pG0ElUfQJC&lRX^pM+R{(-t5$y!-o6pffK; z*T_zJ=O6J!0I7wBV29c8qh1$O6bzzeqXc%VK5~v`Cp`cx{Vd0?33$h0=kqv!B;LC5 zou^XuN!6D3;}q0G!8{;H+Kav_xj1dDO^YA$;5G>QdHS+>99cZJWVg`B4*MAF#<5Cp zy%(O}d`+#8Zlau%*i`q3x}L6pV5_UW2q%5N7^u^0j})=_9=4!^aa!dV9%Mk@6ZdgV zWHYs2P!hS1JDKgjAMJrdkqQG<^}UpOQ$Dd>GCj7>zrbRtXnvOYVMqCHd&b!Lgp4#Y zBayX1Z4FBUu`o=h{dTh3E`K4hE|yqee)3RSF^qa)XAsMZ(n6`PjtM{_@GT^~s^Gu} zO%WQ`uArPduTJ|i4G5!I2Aw!Wtk0~SD!c%2#+SszmInih22e~TFlF5;>)h>uM3;2z z=N?33X526*x-D{CsjLhD`oWWvlY=FRPtZ@!0GzhtVfYICU}XvbLe3`YQp)^rF}^pP zHSZ)JWj*s}TbY`)V3mFxEGW=ui2XA3kaCv7Wn}Z9 z0@Y%W*!T8;&;B22Nyr{9<**O=R=eG_(?Hf^Hv16Cg%jCbtXSlQ3 zbg<^ap~pa4B0)C-P^M|7nZsU&Uw3cb27t^bbNT7?;!s!l{}rWMSci$`1wasI#GFA# z55!2mUISz8WMa#VdnJypd>|~`7WK1^cr>&t2Rv@&2;D6|!5^{&a{cTxCMnFpwo{Ez4GrnP<9dUWbAy z?Y|$qUJ<&t{)cVgiB8bUC@)iU!>Xh#d(I+5EW4>jNJehkEu6kjqSz;)o(BTLl-r8& zZpCbz+{xB@FQu*LZs6IeudMjt@QL;_27(9=^39IMrACkxzxyo^*vl%tFz7BX#3Y zPS47znHlzeC4n_|)GXMVpp0M5xc#=Z|BS3=%2lGjLm&QGwXuAX(I_9IXbHK#KetV1 zx+dM(@itqnQm%i^mvQ1t#r>3Lf6PUcBgA#zm-dw83TqfHo3u5yvp;kbV~|fxtBw;K z_BoVwo*!uuS2y~)j-n`R;Evo{G&pl4(l?_5>S4l-z5lo#$Y}dDtNRM?5lo}QnPuDWbms7xRyx5v(+7h{%pwSiFnJp5_ zlG7h)ajesF-H&sd#Gn~JE7V^v>~!yg!u;cjg<5i*&7E>(D(=r2o8L~~uF@DyQ2s1i z_1V~wrGnc0!@Kf@bmmj(*N+v0G3^NwAmQ5&_Mo_hM@%fVa%SDJnRo0WW6vSqshu;X zKJPYbTT9-&XiHywSZkm*c~F~h`QkkZlRO)T4;5~e;(u||+IWF{B41$9NspS4w^g=; z54b1FKsz;%)@mC>((vSx^t_chTu{B0XGA}IxspA!?I4Q%r|&DbK3+ko81WHDvV45# Nk6k%^FvPX4{(mc#_qG54 literal 0 HcmV?d00001 From 337006c00ee28b0e00db5168c423c3a88d8b62d8 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Wed, 10 Jul 2024 17:30:21 -0700 Subject: [PATCH 03/13] #4911: removed test debug lines --- services/horizon/internal/integration/db_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 25b41c8328..d9106bc48a 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -559,8 +559,6 @@ func TestReingestDB(t *testing.T) { } func TestReingestDatastore(t *testing.T) { - os.Setenv("HORIZON_INTEGRATION_TESTS_ENABLED", "true") - if os.Getenv("HORIZON_INTEGRATION_TESTS_ENABLED") == "" { t.Skip("skipping integration test: HORIZON_INTEGRATION_TESTS_ENABLED not set") } From a19e92952fd4039614333434f77734d28c3f8c81 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Wed, 10 Jul 2024 17:52:52 -0700 Subject: [PATCH 04/13] #4911: cleaned up go.mod tidy --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 26c78fd61b..ce6761d49e 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/docker/docker v27.0.3+incompatible github.com/docker/go-connections v0.5.0 github.com/fsouza/fake-gcs-server v1.49.2 + github.com/otiai10/copy v1.14.0 ) require ( @@ -99,7 +100,6 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/otiai10/copy v1.14.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect diff --git a/go.sum b/go.sum index 50c8ee05ef..4b3d6b186e 100644 --- a/go.sum +++ b/go.sum @@ -403,6 +403,8 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From ee50449e39df78a567e4108501231ef9c618d22b Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 11 Jul 2024 09:14:52 -0700 Subject: [PATCH 05/13] #4911: fixed govet warning --- services/horizon/internal/integration/db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index d9106bc48a..3c876d6f93 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -645,7 +645,7 @@ func TestReingestDatastore(t *testing.T) { // wait until the web server is up before continuing to test requests require.Eventually(t, func() bool { - if _, err := horizonClient.Root(); err != nil { + if _, horizonErr := horizonClient.Root(); horizonErr != nil { return false } return true From bbe931db3ce84cefbd717a231f90ccdc0a471a20 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 11 Jul 2024 14:18:07 -0700 Subject: [PATCH 06/13] #4911: fixed test reusage of commands, reset args --- services/horizon/cmd/db.go | 15 +++++++++++++++ services/horizon/cmd/db_test.go | 4 ++++ services/horizon/internal/integration/db_test.go | 4 ++++ 3 files changed, 23 insertions(+) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index a6ae341a07..647e94471a 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -608,6 +608,21 @@ func runDBDetectGapsInRange(config horizon.Config, start, end uint32) ([]history return q.GetLedgerGapsInRange(context.Background(), start, end) } +func ResetCommandArgs() { + + var commands []*cobra.Command + commands = append(commands, RootCmd) + + for len(commands) > 0 { + // deque the next + command := commands[0] + commands = commands[1:] + + command.SetArgs(nil) + commands = append(commands, command.Commands()...) + } +} + func init() { if err := dbReingestRangeCmdOpts.Init(dbReingestRangeCmd); err != nil { log.Fatal(err.Error()) diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index 2386cff509..baa5be1233 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -38,6 +38,10 @@ func (s *DBCommandsTestSuite) TearDownSuite() { s.db.Close() } +func (s *DBCommandsTestSuite) BeforeTest() { + ResetCommandArgs() +} + func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { RootCmd.SetArgs([]string{ "db", "reingest", "range", diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 3c876d6f93..bed535a473 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -492,6 +492,7 @@ func TestReingestDB(t *testing.T) { itest, reachedLedger := initializeDBIntegrationTest(t) tt := assert.New(t) + horizoncmd.ResetCommandArgs() horizonConfig := itest.GetHorizonIngestConfig() t.Run("validate parallel range", func(t *testing.T) { horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, @@ -563,6 +564,7 @@ func TestReingestDatastore(t *testing.T) { t.Skip("skipping integration test: HORIZON_INTEGRATION_TESTS_ENABLED not set") } + horizoncmd.ResetCommandArgs() newDB := dbtest.Postgres(t) defer newDB.Close() horizoncmd.RootCmd.SetArgs([]string{ @@ -687,6 +689,7 @@ func TestReingestDBWithFilterRules(t *testing.T) { itest, _ := initializeDBIntegrationTest(t) tt := assert.New(t) + horizoncmd.ResetCommandArgs() archive, err := historyarchive.Connect( itest.GetHorizonIngestConfig().HistoryArchiveURLs[0], historyarchive.ArchiveOptions{ @@ -907,6 +910,7 @@ func TestFillGaps(t *testing.T) { itest, reachedLedger := initializeDBIntegrationTest(t) tt := assert.New(t) + horizoncmd.ResetCommandArgs() // Create a fresh Horizon database newDB := dbtest.Postgres(t) freshHorizonPostgresURL := newDB.DSN From 5d89697d6a258f661ad420514a844d557175d36b Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 11 Jul 2024 15:38:48 -0700 Subject: [PATCH 07/13] #4911: reset command args after each cmd invoke in db cmd tests --- services/horizon/internal/integration/db_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index bed535a473..f5f9c0abe2 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -547,6 +547,7 @@ func TestReingestDB(t *testing.T) { "captive-core-reingest-range-integration-tests.cfg", ) + horizoncmd.ResetCommandArgs() horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "reingest", "range", @@ -605,6 +606,7 @@ func TestReingestDatastore(t *testing.T) { t.Logf("fake gcs server started at %v", gcsServer.URL()) t.Setenv("STORAGE_EMULATOR_HOST", gcsServer.URL()) + horizoncmd.ResetCommandArgs() horizoncmd.RootCmd.SetArgs([]string{"db", "reingest", "range", @@ -689,7 +691,6 @@ func TestReingestDBWithFilterRules(t *testing.T) { itest, _ := initializeDBIntegrationTest(t) tt := assert.New(t) - horizoncmd.ResetCommandArgs() archive, err := historyarchive.Connect( itest.GetHorizonIngestConfig().HistoryArchiveURLs[0], historyarchive.ArchiveOptions{ @@ -785,6 +786,7 @@ func TestReingestDBWithFilterRules(t *testing.T) { itest.StopHorizon() // clear the db with reaping all ledgers + horizoncmd.ResetCommandArgs() horizoncmd.RootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", "reap", "--history-retention-count=1", @@ -793,6 +795,7 @@ func TestReingestDBWithFilterRules(t *testing.T) { // repopulate the db with reingestion which should catchup using core reapply filter rules // correctly on reingestion ranged + horizoncmd.ResetCommandArgs() horizoncmd.RootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", "reingest", "range", From 07337a50e141e0d84bec882f04669216d169d67d Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 11 Jul 2024 19:14:59 -0700 Subject: [PATCH 08/13] #4911: added ResetCmds() for tests to redefine cobra commands to re execute them with clean args state --- services/horizon/cmd/db.go | 735 +++++++++--------- services/horizon/cmd/db_test.go | 2 +- services/horizon/cmd/root.go | 58 +- .../horizon/internal/integration/db_test.go | 14 +- 4 files changed, 414 insertions(+), 395 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 647e94471a..9636ebc061 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -27,17 +27,29 @@ import ( hlog "github.com/stellar/go/support/log" ) -var runDBReingestRangeFn = runDBReingestRange - -var dbCmd = &cobra.Command{ - Use: "db [command]", - Short: "commands to manage horizon's postgres db", -} - -var dbMigrateCmd = &cobra.Command{ - Use: "migrate [command]", - Short: "commands to run schema migrations on horizon's postgres db", -} +var ( + runDBReingestRangeFn = runDBReingestRange + dbCmd *cobra.Command + dbMigrateCmd *cobra.Command + dbInitCmd *cobra.Command + dbMigrateDownCmd *cobra.Command + dbMigrateRedoCmd *cobra.Command + dbMigrateStatusCmd *cobra.Command + dbMigrateUpCmd *cobra.Command + dbReapCmd *cobra.Command + dbReingestCmd *cobra.Command + dbReingestRangeCmd *cobra.Command + dbFillGapsCmd *cobra.Command + dbDetectGapsCmd *cobra.Command + reingestForce bool + parallelWorkers uint + parallelJobSize uint32 + retries uint + retryBackoffSeconds uint + ledgerBackendStr string + storageBackendConfigPath string + ledgerBackendType ingest.LedgerBackendType +) func requireAndSetFlags(names ...string) error { set := map[string]bool{} @@ -63,34 +75,6 @@ func requireAndSetFlags(names ...string) error { return fmt.Errorf("could not find %s flags", strings.Join(missing, ",")) } -var dbInitCmd = &cobra.Command{ - Use: "init", - Short: "install schema", - Long: "init initializes the postgres database used by horizon.", - RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { - return err - } - - db, err := sql.Open("postgres", globalConfig.DatabaseURL) - if err != nil { - return err - } - - numMigrationsRun, err := schema.Migrate(db, schema.MigrateUp, 0) - if err != nil { - return err - } - - if numMigrationsRun == 0 { - log.Println("No migrations applied.") - } else { - log.Printf("Successfully applied %d migrations.\n", numMigrationsRun) - } - return nil - }, -} - func migrate(dir schema.MigrateDir, count int) error { if !globalConfig.Ingest { log.Println("Skipping migrations because ingest flag is not enabled") @@ -116,163 +100,6 @@ func migrate(dir schema.MigrateDir, count int) error { return nil } -var dbMigrateDownCmd = &cobra.Command{ - Use: "down COUNT", - Short: "run downwards db schema migrations", - Long: "performs a downards schema migration command", - RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { - return err - } - - // Only allow invocations with 1 args. - if len(args) != 1 { - return ErrUsage{cmd} - } - - count, err := strconv.Atoi(args[0]) - if err != nil { - log.Println(err) - return ErrUsage{cmd} - } - - return migrate(schema.MigrateDown, count) - }, -} - -var dbMigrateRedoCmd = &cobra.Command{ - Use: "redo COUNT", - Short: "redo db schema migrations", - Long: "performs a redo schema migration command", - RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { - return err - } - - // Only allow invocations with 1 args. - if len(args) != 1 { - return ErrUsage{cmd} - } - - count, err := strconv.Atoi(args[0]) - if err != nil { - log.Println(err) - return ErrUsage{cmd} - } - - return migrate(schema.MigrateRedo, count) - }, -} - -var dbMigrateStatusCmd = &cobra.Command{ - Use: "status", - Short: "print current database migration status", - Long: "print current database migration status", - RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName); err != nil { - return err - } - - // Only allow invocations with 0 args. - if len(args) != 0 { - fmt.Println(args) - return ErrUsage{cmd} - } - - dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) - if err != nil { - return err - } - - status, err := schema.Status(dbConn.DB.DB) - if err != nil { - return err - } - - fmt.Println(status) - return nil - }, -} - -var dbMigrateUpCmd = &cobra.Command{ - Use: "up [COUNT]", - Short: "run upwards db schema migrations", - Long: "performs an upwards schema migration command", - RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { - return err - } - - // Only allow invocations with 0-1 args. - if len(args) > 1 { - return ErrUsage{cmd} - } - - count := 0 - if len(args) == 1 { - var err error - count, err = strconv.Atoi(args[0]) - if err != nil { - log.Println(err) - return ErrUsage{cmd} - } - } - - return migrate(schema.MigrateUp, count) - }, -} - -var dbReapCmd = &cobra.Command{ - Use: "reap", - Short: "reaps (i.e. removes) any reapable history data", - Long: "reap removes any historical data that is earlier than the configured retention cutoff", - RunE: func(cmd *cobra.Command, args []string) error { - - err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false}) - if err != nil { - return err - } - - session, err := db.Open("postgres", globalConfig.DatabaseURL) - if err != nil { - return fmt.Errorf("cannot open Horizon DB: %v", err) - } - defer session.Close() - - reaper := ingest.NewReaper( - ingest.ReapConfig{ - RetentionCount: uint32(globalConfig.HistoryRetentionCount), - BatchSize: uint32(globalConfig.HistoryRetentionReapCount), - }, - session, - ) - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) - defer cancel() - return reaper.DeleteUnretainedHistory(ctx) - }, -} - -var dbReingestCmd = &cobra.Command{ - Use: "reingest", - Short: "reingest commands", - Long: "reingest ingests historical data for every ledger or ledgers specified by subcommand", - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Println("Use one of the subcomands...") - return ErrUsage{cmd} - }, -} - -var ( - reingestForce bool - parallelWorkers uint - parallelJobSize uint32 - retries uint - retryBackoffSeconds uint - ledgerBackendStr string - storageBackendConfigPath string - ledgerBackendType ingest.LedgerBackendType -) - func ingestRangeCmdOpts() support.ConfigOptions { return support.ConfigOptions{ { @@ -356,138 +183,7 @@ func ingestRangeCmdOpts() support.ConfigOptions { } var dbReingestRangeCmdOpts = ingestRangeCmdOpts() -var dbReingestRangeCmd = &cobra.Command{ - Use: "range [Start sequence number] [End sequence number]", - Short: "reingests ledgers within a range", - Long: "reingests ledgers between X and Y sequence number (closed intervals)", - RunE: func(cmd *cobra.Command, args []string) error { - if err := dbReingestRangeCmdOpts.RequireE(); err != nil { - return err - } - if err := dbReingestRangeCmdOpts.SetValues(); err != nil { - return err - } - - if len(args) != 2 { - return ErrUsage{cmd} - } - - argsUInt32 := make([]uint32, 2) - for i, arg := range args { - if seq, err := strconv.ParseUint(arg, 10, 32); err != nil { - cmd.Usage() - return fmt.Errorf(`invalid sequence number "%s"`, arg) - } else { - argsUInt32[i] = uint32(seq) - } - } - - var storageBackendConfig ingest.StorageBackendConfig - options := horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false} - if ledgerBackendType == ingest.BufferedStorageBackend { - cfg, err := toml.LoadFile(storageBackendConfigPath) - if err != nil { - return fmt.Errorf("failed to load config file %v: %w", storageBackendConfigPath, err) - } - if err = cfg.Unmarshal(&storageBackendConfig); err != nil { - return fmt.Errorf("error unmarshalling TOML config: %w", err) - } - storageBackendConfig.BufferedStorageBackendFactory = ledgerbackend.NewBufferedStorageBackend - storageBackendConfig.DataStoreFactory = datastore.NewDataStore - // when using buffered storage, performance observations have noted optimal parallel batch size - // of 100, apply that as default if the flag was absent. - if !viper.IsSet("parallel-job-size") { - parallelJobSize = 100 - } - options.NoCaptiveCore = true - } - - err := horizon.ApplyFlags(globalConfig, globalFlags, options) - if err != nil { - return err - } - return runDBReingestRangeFn( - []history.LedgerRange{{StartSequence: argsUInt32[0], EndSequence: argsUInt32[1]}}, - reingestForce, - parallelWorkers, - *globalConfig, - storageBackendConfig, - ) - }, -} - var dbFillGapsCmdOpts = ingestRangeCmdOpts() -var dbFillGapsCmd = &cobra.Command{ - Use: "fill-gaps [Start sequence number] [End sequence number]", - Short: "Ingests any gaps found in the horizon db", - Long: "Ingests any gaps found in the horizon db. The command takes an optional start and end parameters which restrict the range of ledgers ingested.", - RunE: func(cmd *cobra.Command, args []string) error { - if err := dbFillGapsCmdOpts.RequireE(); err != nil { - return err - } - if err := dbFillGapsCmdOpts.SetValues(); err != nil { - return err - } - - if len(args) != 0 && len(args) != 2 { - hlog.Errorf("Expected either 0 arguments or 2 but found %v arguments", len(args)) - return ErrUsage{cmd} - } - - var start, end uint64 - var withRange bool - if len(args) == 2 { - var err error - start, err = strconv.ParseUint(args[0], 10, 32) - if err != nil { - cmd.Usage() - return fmt.Errorf(`invalid sequence number "%s"`, args[0]) - } - end, err = strconv.ParseUint(args[1], 10, 32) - if err != nil { - cmd.Usage() - return fmt.Errorf(`invalid sequence number "%s"`, args[1]) - } - withRange = true - } - - var storageBackendConfig ingest.StorageBackendConfig - options := horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false} - if ledgerBackendType == ingest.BufferedStorageBackend { - cfg, err := toml.LoadFile(storageBackendConfigPath) - if err != nil { - return fmt.Errorf("failed to load config file %v: %w", storageBackendConfigPath, err) - } - if err = cfg.Unmarshal(&storageBackendConfig); err != nil { - return fmt.Errorf("error unmarshalling TOML config: %w", err) - } - storageBackendConfig.BufferedStorageBackendFactory = ledgerbackend.NewBufferedStorageBackend - storageBackendConfig.DataStoreFactory = datastore.NewDataStore - options.NoCaptiveCore = true - } - - err := horizon.ApplyFlags(globalConfig, globalFlags, options) - if err != nil { - return err - } - var gaps []history.LedgerRange - if withRange { - gaps, err = runDBDetectGapsInRange(*globalConfig, uint32(start), uint32(end)) - if err != nil { - return err - } - hlog.Infof("found gaps %v within range [%v, %v]", gaps, start, end) - } else { - gaps, err = runDBDetectGaps(*globalConfig) - if err != nil { - return err - } - hlog.Infof("found gaps %v", gaps) - } - - return runDBReingestRangeFn(gaps, reingestForce, parallelWorkers, *globalConfig, storageBackendConfig) - }, -} func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, parallelWorkers uint, config horizon.Config, storageBackendConfig ingest.StorageBackendConfig) error { var err error @@ -559,35 +255,6 @@ the reingest command completes.`) return nil } -var dbDetectGapsCmd = &cobra.Command{ - Use: "detect-gaps", - Short: "detects ingestion gaps in Horizon's database", - Long: "detects ingestion gaps in Horizon's database and prints a list of reingest commands needed to fill the gaps", - RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName); err != nil { - return err - } - - if len(args) != 0 { - return ErrUsage{cmd} - } - gaps, err := runDBDetectGaps(*globalConfig) - if err != nil { - return err - } - if len(gaps) == 0 { - hlog.Info("No gaps found") - return nil - } - fmt.Println("Horizon commands to run in order to fill in the gaps:") - cmdname := os.Args[0] - for _, g := range gaps { - fmt.Printf("%s db reingest range %d %d\n", cmdname, g.StartSequence, g.EndSequence) - } - return nil - }, -} - func runDBDetectGaps(config horizon.Config) ([]history.LedgerRange, error) { horizonSession, err := db.Open("postgres", config.DatabaseURL) if err != nil { @@ -608,22 +275,352 @@ func runDBDetectGapsInRange(config horizon.Config, start, end uint32) ([]history return q.GetLedgerGapsInRange(context.Background(), start, end) } -func ResetCommandArgs() { +func DefineDBCommands() { + dbCmd = &cobra.Command{ + Use: "db [command]", + Short: "commands to manage horizon's postgres db", + } + + dbMigrateCmd = &cobra.Command{ + Use: "migrate [command]", + Short: "commands to run schema migrations on horizon's postgres db", + } + + dbInitCmd = &cobra.Command{ + Use: "init", + Short: "install schema", + Long: "init initializes the postgres database used by horizon.", + RunE: func(cmd *cobra.Command, args []string) error { + if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + return err + } - var commands []*cobra.Command - commands = append(commands, RootCmd) + db, err := sql.Open("postgres", globalConfig.DatabaseURL) + if err != nil { + return err + } - for len(commands) > 0 { - // deque the next - command := commands[0] - commands = commands[1:] + numMigrationsRun, err := schema.Migrate(db, schema.MigrateUp, 0) + if err != nil { + return err + } - command.SetArgs(nil) - commands = append(commands, command.Commands()...) + if numMigrationsRun == 0 { + log.Println("No migrations applied.") + } else { + log.Printf("Successfully applied %d migrations.\n", numMigrationsRun) + } + return nil + }, + } + + dbMigrateDownCmd = &cobra.Command{ + Use: "down COUNT", + Short: "run downwards db schema migrations", + Long: "performs a downards schema migration command", + RunE: func(cmd *cobra.Command, args []string) error { + if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + return err + } + + // Only allow invocations with 1 args. + if len(args) != 1 { + return ErrUsage{cmd} + } + + count, err := strconv.Atoi(args[0]) + if err != nil { + log.Println(err) + return ErrUsage{cmd} + } + + return migrate(schema.MigrateDown, count) + }, + } + + dbMigrateRedoCmd = &cobra.Command{ + Use: "redo COUNT", + Short: "redo db schema migrations", + Long: "performs a redo schema migration command", + RunE: func(cmd *cobra.Command, args []string) error { + if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + return err + } + + // Only allow invocations with 1 args. + if len(args) != 1 { + return ErrUsage{cmd} + } + + count, err := strconv.Atoi(args[0]) + if err != nil { + log.Println(err) + return ErrUsage{cmd} + } + + return migrate(schema.MigrateRedo, count) + }, + } + + dbMigrateStatusCmd = &cobra.Command{ + Use: "status", + Short: "print current database migration status", + Long: "print current database migration status", + RunE: func(cmd *cobra.Command, args []string) error { + if err := requireAndSetFlags(horizon.DatabaseURLFlagName); err != nil { + return err + } + + // Only allow invocations with 0 args. + if len(args) != 0 { + fmt.Println(args) + return ErrUsage{cmd} + } + + dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) + if err != nil { + return err + } + + status, err := schema.Status(dbConn.DB.DB) + if err != nil { + return err + } + + fmt.Println(status) + return nil + }, + } + + dbMigrateUpCmd = &cobra.Command{ + Use: "up [COUNT]", + Short: "run upwards db schema migrations", + Long: "performs an upwards schema migration command", + RunE: func(cmd *cobra.Command, args []string) error { + if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + return err + } + + // Only allow invocations with 0-1 args. + if len(args) > 1 { + return ErrUsage{cmd} + } + + count := 0 + if len(args) == 1 { + var err error + count, err = strconv.Atoi(args[0]) + if err != nil { + log.Println(err) + return ErrUsage{cmd} + } + } + + return migrate(schema.MigrateUp, count) + }, + } + + dbReapCmd = &cobra.Command{ + Use: "reap", + Short: "reaps (i.e. removes) any reapable history data", + Long: "reap removes any historical data that is earlier than the configured retention cutoff", + RunE: func(cmd *cobra.Command, args []string) error { + + err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false}) + if err != nil { + return err + } + + session, err := db.Open("postgres", globalConfig.DatabaseURL) + if err != nil { + return fmt.Errorf("cannot open Horizon DB: %v", err) + } + defer session.Close() + + reaper := ingest.NewReaper( + ingest.ReapConfig{ + RetentionCount: uint32(globalConfig.HistoryRetentionCount), + BatchSize: uint32(globalConfig.HistoryRetentionReapCount), + }, + session, + ) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer cancel() + return reaper.DeleteUnretainedHistory(ctx) + }, + } + + dbReingestCmd = &cobra.Command{ + Use: "reingest", + Short: "reingest commands", + Long: "reingest ingests historical data for every ledger or ledgers specified by subcommand", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Println("Use one of the subcomands...") + return ErrUsage{cmd} + }, + } + + dbReingestRangeCmd = &cobra.Command{ + Use: "range [Start sequence number] [End sequence number]", + Short: "reingests ledgers within a range", + Long: "reingests ledgers between X and Y sequence number (closed intervals)", + RunE: func(cmd *cobra.Command, args []string) error { + if err := dbReingestRangeCmdOpts.RequireE(); err != nil { + return err + } + if err := dbReingestRangeCmdOpts.SetValues(); err != nil { + return err + } + + if len(args) != 2 { + return ErrUsage{cmd} + } + + argsUInt32 := make([]uint32, 2) + for i, arg := range args { + if seq, err := strconv.ParseUint(arg, 10, 32); err != nil { + cmd.Usage() + return fmt.Errorf(`invalid sequence number "%s"`, arg) + } else { + argsUInt32[i] = uint32(seq) + } + } + + var storageBackendConfig ingest.StorageBackendConfig + options := horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false} + if ledgerBackendType == ingest.BufferedStorageBackend { + cfg, err := toml.LoadFile(storageBackendConfigPath) + if err != nil { + return fmt.Errorf("failed to load config file %v: %w", storageBackendConfigPath, err) + } + if err = cfg.Unmarshal(&storageBackendConfig); err != nil { + return fmt.Errorf("error unmarshalling TOML config: %w", err) + } + storageBackendConfig.BufferedStorageBackendFactory = ledgerbackend.NewBufferedStorageBackend + storageBackendConfig.DataStoreFactory = datastore.NewDataStore + // when using buffered storage, performance observations have noted optimal parallel batch size + // of 100, apply that as default if the flag was absent. + if !viper.IsSet("parallel-job-size") { + parallelJobSize = 100 + } + options.NoCaptiveCore = true + } + + err := horizon.ApplyFlags(globalConfig, globalFlags, options) + if err != nil { + return err + } + return runDBReingestRangeFn( + []history.LedgerRange{{StartSequence: argsUInt32[0], EndSequence: argsUInt32[1]}}, + reingestForce, + parallelWorkers, + *globalConfig, + storageBackendConfig, + ) + }, + } + + dbFillGapsCmd = &cobra.Command{ + Use: "fill-gaps [Start sequence number] [End sequence number]", + Short: "Ingests any gaps found in the horizon db", + Long: "Ingests any gaps found in the horizon db. The command takes an optional start and end parameters which restrict the range of ledgers ingested.", + RunE: func(cmd *cobra.Command, args []string) error { + if err := dbFillGapsCmdOpts.RequireE(); err != nil { + return err + } + if err := dbFillGapsCmdOpts.SetValues(); err != nil { + return err + } + + if len(args) != 0 && len(args) != 2 { + hlog.Errorf("Expected either 0 arguments or 2 but found %v arguments", len(args)) + return ErrUsage{cmd} + } + + var start, end uint64 + var withRange bool + if len(args) == 2 { + var err error + start, err = strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Usage() + return fmt.Errorf(`invalid sequence number "%s"`, args[0]) + } + end, err = strconv.ParseUint(args[1], 10, 32) + if err != nil { + cmd.Usage() + return fmt.Errorf(`invalid sequence number "%s"`, args[1]) + } + withRange = true + } + + var storageBackendConfig ingest.StorageBackendConfig + options := horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false} + if ledgerBackendType == ingest.BufferedStorageBackend { + cfg, err := toml.LoadFile(storageBackendConfigPath) + if err != nil { + return fmt.Errorf("failed to load config file %v: %w", storageBackendConfigPath, err) + } + if err = cfg.Unmarshal(&storageBackendConfig); err != nil { + return fmt.Errorf("error unmarshalling TOML config: %w", err) + } + storageBackendConfig.BufferedStorageBackendFactory = ledgerbackend.NewBufferedStorageBackend + storageBackendConfig.DataStoreFactory = datastore.NewDataStore + options.NoCaptiveCore = true + } + + err := horizon.ApplyFlags(globalConfig, globalFlags, options) + if err != nil { + return err + } + var gaps []history.LedgerRange + if withRange { + gaps, err = runDBDetectGapsInRange(*globalConfig, uint32(start), uint32(end)) + if err != nil { + return err + } + hlog.Infof("found gaps %v within range [%v, %v]", gaps, start, end) + } else { + gaps, err = runDBDetectGaps(*globalConfig) + if err != nil { + return err + } + hlog.Infof("found gaps %v", gaps) + } + + return runDBReingestRangeFn(gaps, reingestForce, parallelWorkers, *globalConfig, storageBackendConfig) + }, + } + + dbDetectGapsCmd = &cobra.Command{ + Use: "detect-gaps", + Short: "detects ingestion gaps in Horizon's database", + Long: "detects ingestion gaps in Horizon's database and prints a list of reingest commands needed to fill the gaps", + RunE: func(cmd *cobra.Command, args []string) error { + if err := requireAndSetFlags(horizon.DatabaseURLFlagName); err != nil { + return err + } + + if len(args) != 0 { + return ErrUsage{cmd} + } + gaps, err := runDBDetectGaps(*globalConfig) + if err != nil { + return err + } + if len(gaps) == 0 { + hlog.Info("No gaps found") + return nil + } + fmt.Println("Horizon commands to run in order to fill in the gaps:") + cmdname := os.Args[0] + for _, g := range gaps { + fmt.Printf("%s db reingest range %d %d\n", cmdname, g.StartSequence, g.EndSequence) + } + return nil + }, } -} -func init() { if err := dbReingestRangeCmdOpts.Init(dbReingestRangeCmd); err != nil { log.Fatal(err.Error()) } @@ -651,3 +648,7 @@ func init() { ) dbReingestCmd.AddCommand(dbReingestRangeCmd) } + +func init() { + DefineDBCommands() +} diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index baa5be1233..7b47306fc8 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -39,7 +39,7 @@ func (s *DBCommandsTestSuite) TearDownSuite() { } func (s *DBCommandsTestSuite) BeforeTest() { - ResetCommandArgs() + ResetCmds() } func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index d2900496d4..4e88e30310 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -12,7 +12,13 @@ import ( var ( globalConfig, globalFlags = horizon.Flags() - RootCmd = &cobra.Command{ + RootCmd = createRootCmd() + originalHelpFunc = RootCmd.HelpFunc() + originalUsageFunc = RootCmd.UsageFunc() +) + +func createRootCmd() *cobra.Command { + return &cobra.Command{ Use: "horizon", Short: "client-facing api server for the Stellar network", SilenceErrors: true, @@ -30,9 +36,37 @@ var ( return app.Serve() }, } - originalHelpFunc = RootCmd.HelpFunc() +} + +func initRootCmd() { + // override the default help output, apply further filtering on which global flags + // will be shown on the help outout dependent on the command help was issued upon. + RootCmd.SetHelpFunc(func(c *cobra.Command, args []string) { + enableGlobalOptionsInHelp(c, globalFlags) + originalHelpFunc(c, args) + }) + + RootCmd.SetUsageFunc(func(c *cobra.Command) error { + enableGlobalOptionsInHelp(c, globalFlags) + return originalUsageFunc(c) + }) + + err := globalFlags.Init(RootCmd) + if err != nil { + stdLog.Fatal(err.Error()) + } +} + +// intended for test purposes, recreates new instaces of whole command tree. +// allows tests top execute same command multiple times with different args +// in same process and avoid state carryover from prior command. +func ResetCmds() { + RootCmd = createRootCmd() + originalHelpFunc = RootCmd.HelpFunc() originalUsageFunc = RootCmd.UsageFunc() -) + initRootCmd() + DefineDBCommands() +} // ErrUsage indicates we should print the usage string and exit with code 1 type ErrUsage struct { @@ -51,23 +85,7 @@ func (e ErrExitCode) Error() string { } func init() { - - // override the default help output, apply further filtering on which global flags - // will be shown on the help outout dependent on the command help was issued upon. - RootCmd.SetHelpFunc(func(c *cobra.Command, args []string) { - enableGlobalOptionsInHelp(c, globalFlags) - originalHelpFunc(c, args) - }) - - RootCmd.SetUsageFunc(func(c *cobra.Command) error { - enableGlobalOptionsInHelp(c, globalFlags) - return originalUsageFunc(c) - }) - - err := globalFlags.Init(RootCmd) - if err != nil { - stdLog.Fatal(err.Error()) - } + initRootCmd() } func Execute() error { diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index f5f9c0abe2..8765d21837 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -492,7 +492,7 @@ func TestReingestDB(t *testing.T) { itest, reachedLedger := initializeDBIntegrationTest(t) tt := assert.New(t) - horizoncmd.ResetCommandArgs() + horizoncmd.ResetCmds() horizonConfig := itest.GetHorizonIngestConfig() t.Run("validate parallel range", func(t *testing.T) { horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, @@ -547,7 +547,7 @@ func TestReingestDB(t *testing.T) { "captive-core-reingest-range-integration-tests.cfg", ) - horizoncmd.ResetCommandArgs() + horizoncmd.ResetCmds() horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "reingest", "range", @@ -565,9 +565,9 @@ func TestReingestDatastore(t *testing.T) { t.Skip("skipping integration test: HORIZON_INTEGRATION_TESTS_ENABLED not set") } - horizoncmd.ResetCommandArgs() newDB := dbtest.Postgres(t) defer newDB.Close() + horizoncmd.ResetCmds() horizoncmd.RootCmd.SetArgs([]string{ "db", "migrate", "up", "--db-url", newDB.DSN}) require.NoError(t, horizoncmd.RootCmd.Execute()) @@ -606,7 +606,7 @@ func TestReingestDatastore(t *testing.T) { t.Logf("fake gcs server started at %v", gcsServer.URL()) t.Setenv("STORAGE_EMULATOR_HOST", gcsServer.URL()) - horizoncmd.ResetCommandArgs() + horizoncmd.ResetCmds() horizoncmd.RootCmd.SetArgs([]string{"db", "reingest", "range", @@ -786,7 +786,7 @@ func TestReingestDBWithFilterRules(t *testing.T) { itest.StopHorizon() // clear the db with reaping all ledgers - horizoncmd.ResetCommandArgs() + horizoncmd.ResetCmds() horizoncmd.RootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", "reap", "--history-retention-count=1", @@ -795,7 +795,7 @@ func TestReingestDBWithFilterRules(t *testing.T) { // repopulate the db with reingestion which should catchup using core reapply filter rules // correctly on reingestion ranged - horizoncmd.ResetCommandArgs() + horizoncmd.ResetCmds() horizoncmd.RootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", "reingest", "range", @@ -913,7 +913,6 @@ func TestFillGaps(t *testing.T) { itest, reachedLedger := initializeDBIntegrationTest(t) tt := assert.New(t) - horizoncmd.ResetCommandArgs() // Create a fresh Horizon database newDB := dbtest.Postgres(t) freshHorizonPostgresURL := newDB.DSN @@ -942,6 +941,7 @@ func TestFillGaps(t *testing.T) { }) tt.NoError(err) + horizoncmd.ResetCmds() t.Run("validate parallel range", func(t *testing.T) { horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", From cd15f6b22a2004b9d9849ebfdcf729cc5b710f3c Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 12 Jul 2024 11:46:07 -0700 Subject: [PATCH 09/13] #4911: avoid rootcmd arg/flag state collisions during test usages --- services/horizon/cmd/db.go | 49 ++++++++++--------- services/horizon/cmd/db_test.go | 24 ++++----- services/horizon/cmd/root.go | 42 ++++++++-------- .../horizon/internal/integration/db_test.go | 42 ++++++++-------- 4 files changed, 80 insertions(+), 77 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 9636ebc061..cd62cd1d3c 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -20,6 +20,7 @@ import ( "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/db2/schema" "github.com/stellar/go/services/horizon/internal/ingest" + "github.com/stellar/go/support/config" support "github.com/stellar/go/support/config" "github.com/stellar/go/support/datastore" "github.com/stellar/go/support/db" @@ -51,12 +52,12 @@ var ( ledgerBackendType ingest.LedgerBackendType ) -func requireAndSetFlags(names ...string) error { +func requireAndSetFlags(horizonFlags config.ConfigOptions, names ...string) error { set := map[string]bool{} for _, name := range names { set[name] = true } - for _, flag := range globalFlags { + for _, flag := range horizonFlags { if set[flag.Name] { flag.Require() if err := flag.SetValue(); err != nil { @@ -275,7 +276,7 @@ func runDBDetectGapsInRange(config horizon.Config, start, end uint32) ([]history return q.GetLedgerGapsInRange(context.Background(), start, end) } -func DefineDBCommands() { +func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, horizonFlags config.ConfigOptions) { dbCmd = &cobra.Command{ Use: "db [command]", Short: "commands to manage horizon's postgres db", @@ -291,11 +292,11 @@ func DefineDBCommands() { Short: "install schema", Long: "init initializes the postgres database used by horizon.", RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + if err := requireAndSetFlags(horizonFlags, horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { return err } - db, err := sql.Open("postgres", globalConfig.DatabaseURL) + db, err := sql.Open("postgres", horizonConfig.DatabaseURL) if err != nil { return err } @@ -319,7 +320,7 @@ func DefineDBCommands() { Short: "run downwards db schema migrations", Long: "performs a downards schema migration command", RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + if err := requireAndSetFlags(horizonFlags, horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { return err } @@ -343,7 +344,7 @@ func DefineDBCommands() { Short: "redo db schema migrations", Long: "performs a redo schema migration command", RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + if err := requireAndSetFlags(horizonFlags, horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { return err } @@ -367,7 +368,7 @@ func DefineDBCommands() { Short: "print current database migration status", Long: "print current database migration status", RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName); err != nil { + if err := requireAndSetFlags(horizonFlags, horizon.DatabaseURLFlagName); err != nil { return err } @@ -377,7 +378,7 @@ func DefineDBCommands() { return ErrUsage{cmd} } - dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) + dbConn, err := db.Open("postgres", horizonConfig.DatabaseURL) if err != nil { return err } @@ -397,7 +398,7 @@ func DefineDBCommands() { Short: "run upwards db schema migrations", Long: "performs an upwards schema migration command", RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { + if err := requireAndSetFlags(horizonFlags, horizon.DatabaseURLFlagName, horizon.IngestFlagName); err != nil { return err } @@ -426,12 +427,12 @@ func DefineDBCommands() { Long: "reap removes any historical data that is earlier than the configured retention cutoff", RunE: func(cmd *cobra.Command, args []string) error { - err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false}) + err := horizon.ApplyFlags(horizonConfig, horizonFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false}) if err != nil { return err } - session, err := db.Open("postgres", globalConfig.DatabaseURL) + session, err := db.Open("postgres", horizonConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -439,8 +440,8 @@ func DefineDBCommands() { reaper := ingest.NewReaper( ingest.ReapConfig{ - RetentionCount: uint32(globalConfig.HistoryRetentionCount), - BatchSize: uint32(globalConfig.HistoryRetentionReapCount), + RetentionCount: uint32(horizonConfig.HistoryRetentionCount), + BatchSize: uint32(horizonConfig.HistoryRetentionReapCount), }, session, ) @@ -506,7 +507,7 @@ func DefineDBCommands() { options.NoCaptiveCore = true } - err := horizon.ApplyFlags(globalConfig, globalFlags, options) + err := horizon.ApplyFlags(horizonConfig, horizonFlags, options) if err != nil { return err } @@ -514,7 +515,7 @@ func DefineDBCommands() { []history.LedgerRange{{StartSequence: argsUInt32[0], EndSequence: argsUInt32[1]}}, reingestForce, parallelWorkers, - *globalConfig, + *horizonConfig, storageBackendConfig, ) }, @@ -569,26 +570,26 @@ func DefineDBCommands() { options.NoCaptiveCore = true } - err := horizon.ApplyFlags(globalConfig, globalFlags, options) + err := horizon.ApplyFlags(horizonConfig, horizonFlags, options) if err != nil { return err } var gaps []history.LedgerRange if withRange { - gaps, err = runDBDetectGapsInRange(*globalConfig, uint32(start), uint32(end)) + gaps, err = runDBDetectGapsInRange(*horizonConfig, uint32(start), uint32(end)) if err != nil { return err } hlog.Infof("found gaps %v within range [%v, %v]", gaps, start, end) } else { - gaps, err = runDBDetectGaps(*globalConfig) + gaps, err = runDBDetectGaps(*horizonConfig) if err != nil { return err } hlog.Infof("found gaps %v", gaps) } - return runDBReingestRangeFn(gaps, reingestForce, parallelWorkers, *globalConfig, storageBackendConfig) + return runDBReingestRangeFn(gaps, reingestForce, parallelWorkers, *horizonConfig, storageBackendConfig) }, } @@ -597,14 +598,14 @@ func DefineDBCommands() { Short: "detects ingestion gaps in Horizon's database", Long: "detects ingestion gaps in Horizon's database and prints a list of reingest commands needed to fill the gaps", RunE: func(cmd *cobra.Command, args []string) error { - if err := requireAndSetFlags(horizon.DatabaseURLFlagName); err != nil { + if err := requireAndSetFlags(horizonFlags, horizon.DatabaseURLFlagName); err != nil { return err } if len(args) != 0 { return ErrUsage{cmd} } - gaps, err := runDBDetectGaps(*globalConfig) + gaps, err := runDBDetectGaps(*horizonConfig) if err != nil { return err } @@ -631,7 +632,7 @@ func DefineDBCommands() { viper.BindPFlags(dbReingestRangeCmd.PersistentFlags()) viper.BindPFlags(dbFillGapsCmd.PersistentFlags()) - RootCmd.AddCommand(dbCmd) + rootCmd.AddCommand(dbCmd) dbCmd.AddCommand( dbInitCmd, dbMigrateCmd, @@ -650,5 +651,5 @@ func DefineDBCommands() { } func init() { - DefineDBCommands() + DefineDBCommands(RootCmd, globalConfig, globalFlags) } diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index 7b47306fc8..845241da08 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -3,6 +3,7 @@ package cmd import ( "testing" + "github.com/spf13/cobra" horizon "github.com/stellar/go/services/horizon/internal" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest" @@ -18,7 +19,8 @@ func TestDBCommandsTestSuite(t *testing.T) { type DBCommandsTestSuite struct { suite.Suite - db *dbtest.DB + db *dbtest.DB + rootCmd *cobra.Command } func (s *DBCommandsTestSuite) SetupSuite() { @@ -38,12 +40,12 @@ func (s *DBCommandsTestSuite) TearDownSuite() { s.db.Close() } -func (s *DBCommandsTestSuite) BeforeTest() { - ResetCmds() +func (s *DBCommandsTestSuite) BeforeTest(suiteName string, testName string) { + s.rootCmd = NewRootCmd() } func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { - RootCmd.SetArgs([]string{ + s.rootCmd.SetArgs([]string{ "db", "reingest", "range", "--db-url", s.db.DSN, "--network", "testnet", @@ -53,12 +55,12 @@ func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { "2", "10"}) - require.NoError(s.T(), dbReingestRangeCmd.Execute()) + require.NoError(s.T(), s.rootCmd.Execute()) require.Equal(s.T(), parallelJobSize, uint32(100)) } func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForCaptiveBackend() { - RootCmd.SetArgs([]string{ + s.rootCmd.SetArgs([]string{ "db", "reingest", "range", "--db-url", s.db.DSN, "--network", "testnet", @@ -68,12 +70,12 @@ func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForCaptiveBackend() { "2", "10"}) - require.NoError(s.T(), RootCmd.Execute()) + require.NoError(s.T(), s.rootCmd.Execute()) require.Equal(s.T(), parallelJobSize, uint32(100_000)) } func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForCaptive() { - RootCmd.SetArgs([]string{ + s.rootCmd.SetArgs([]string{ "db", "reingest", "range", "--db-url", s.db.DSN, "--network", "testnet", @@ -84,12 +86,12 @@ func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForCaptive() { "2", "10"}) - require.NoError(s.T(), RootCmd.Execute()) + require.NoError(s.T(), s.rootCmd.Execute()) require.Equal(s.T(), parallelJobSize, uint32(5)) } func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForBuffered() { - RootCmd.SetArgs([]string{ + s.rootCmd.SetArgs([]string{ "db", "reingest", "range", "--db-url", s.db.DSN, "--network", "testnet", @@ -100,6 +102,6 @@ func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForBuffered() { "2", "10"}) - require.NoError(s.T(), RootCmd.Execute()) + require.NoError(s.T(), s.rootCmd.Execute()) require.Equal(s.T(), parallelJobSize, uint32(5)) } diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index 4e88e30310..099979c97b 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -12,12 +12,12 @@ import ( var ( globalConfig, globalFlags = horizon.Flags() - RootCmd = createRootCmd() + RootCmd = createRootCmd(globalConfig, globalFlags) originalHelpFunc = RootCmd.HelpFunc() originalUsageFunc = RootCmd.UsageFunc() ) -func createRootCmd() *cobra.Command { +func createRootCmd(horizonConfig *horizon.Config, configOptions config.ConfigOptions) *cobra.Command { return &cobra.Command{ Use: "horizon", Short: "client-facing api server for the Stellar network", @@ -29,7 +29,7 @@ func createRootCmd() *cobra.Command { "DEPRECATED - the use of command-line flags has been deprecated in favor of environment variables. Please" + "consult our Configuring section in the developer documentation on how to use them - https://developers.stellar.org/docs/run-api-server/configuring", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) + app, err := horizon.NewAppFromFlags(horizonConfig, configOptions) if err != nil { return err } @@ -38,34 +38,34 @@ func createRootCmd() *cobra.Command { } } -func initRootCmd() { +func initRootCmd(cmd *cobra.Command, + originalHelpFn func(*cobra.Command, []string), + originalUsageFn func(*cobra.Command) error, + horizonGlobalFlags config.ConfigOptions) { // override the default help output, apply further filtering on which global flags // will be shown on the help outout dependent on the command help was issued upon. - RootCmd.SetHelpFunc(func(c *cobra.Command, args []string) { - enableGlobalOptionsInHelp(c, globalFlags) - originalHelpFunc(c, args) + cmd.SetHelpFunc(func(c *cobra.Command, args []string) { + enableGlobalOptionsInHelp(c, horizonGlobalFlags) + originalHelpFn(c, args) }) - RootCmd.SetUsageFunc(func(c *cobra.Command) error { - enableGlobalOptionsInHelp(c, globalFlags) - return originalUsageFunc(c) + cmd.SetUsageFunc(func(c *cobra.Command) error { + enableGlobalOptionsInHelp(c, horizonGlobalFlags) + return originalUsageFn(c) }) - err := globalFlags.Init(RootCmd) + err := horizonGlobalFlags.Init(cmd) if err != nil { stdLog.Fatal(err.Error()) } } -// intended for test purposes, recreates new instaces of whole command tree. -// allows tests top execute same command multiple times with different args -// in same process and avoid state carryover from prior command. -func ResetCmds() { - RootCmd = createRootCmd() - originalHelpFunc = RootCmd.HelpFunc() - originalUsageFunc = RootCmd.UsageFunc() - initRootCmd() - DefineDBCommands() +func NewRootCmd() *cobra.Command { + horizonGlobalConfig, horizonGlobalFlags := horizon.Flags() + cmd := createRootCmd(horizonGlobalConfig, horizonGlobalFlags) + initRootCmd(cmd, cmd.HelpFunc(), cmd.UsageFunc(), horizonGlobalFlags) + DefineDBCommands(cmd, horizonGlobalConfig, horizonGlobalFlags) + return cmd } // ErrUsage indicates we should print the usage string and exit with code 1 @@ -85,7 +85,7 @@ func (e ErrExitCode) Error() string { } func init() { - initRootCmd() + initRootCmd(RootCmd, originalHelpFunc, originalUsageFunc, globalFlags) } func Execute() error { diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 8765d21837..c820ae7cd8 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -492,10 +492,10 @@ func TestReingestDB(t *testing.T) { itest, reachedLedger := initializeDBIntegrationTest(t) tt := assert.New(t) - horizoncmd.ResetCmds() horizonConfig := itest.GetHorizonIngestConfig() t.Run("validate parallel range", func(t *testing.T) { - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, + var rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "reingest", "range", @@ -504,7 +504,7 @@ func TestReingestDB(t *testing.T) { "2", )) - assert.EqualError(t, horizoncmd.RootCmd.Execute(), "Invalid range: {10 2} from > to") + assert.EqualError(t, rootCmd.Execute(), "Invalid range: {10 2} from > to") }) t.Logf("reached ledger is %v", reachedLedger) @@ -547,8 +547,8 @@ func TestReingestDB(t *testing.T) { "captive-core-reingest-range-integration-tests.cfg", ) - horizoncmd.ResetCmds() - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", + var rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "reingest", "range", "--parallel-workers=1", @@ -556,8 +556,8 @@ func TestReingestDB(t *testing.T) { fmt.Sprintf("%d", toLedger), )) - tt.NoError(horizoncmd.RootCmd.Execute()) - tt.NoError(horizoncmd.RootCmd.Execute(), "Repeat the same reingest range against db, should not have errors.") + tt.NoError(rootCmd.Execute()) + tt.NoError(rootCmd.Execute(), "Repeat the same reingest range against db, should not have errors.") } func TestReingestDatastore(t *testing.T) { @@ -567,10 +567,10 @@ func TestReingestDatastore(t *testing.T) { newDB := dbtest.Postgres(t) defer newDB.Close() - horizoncmd.ResetCmds() - horizoncmd.RootCmd.SetArgs([]string{ + var rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs([]string{ "db", "migrate", "up", "--db-url", newDB.DSN}) - require.NoError(t, horizoncmd.RootCmd.Execute()) + require.NoError(t, rootCmd.Execute()) testTempDir := t.TempDir() tempSeedDataPath := filepath.Join(testTempDir, "data") @@ -606,8 +606,8 @@ func TestReingestDatastore(t *testing.T) { t.Logf("fake gcs server started at %v", gcsServer.URL()) t.Setenv("STORAGE_EMULATOR_HOST", gcsServer.URL()) - horizoncmd.ResetCmds() - horizoncmd.RootCmd.SetArgs([]string{"db", + rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs([]string{"db", "reingest", "range", "--db-url", newDB.DSN, @@ -618,7 +618,7 @@ func TestReingestDatastore(t *testing.T) { "997", "999"}) - require.NoError(t, horizoncmd.RootCmd.Execute()) + require.NoError(t, rootCmd.Execute()) listener, webApp, webPort, err := dynamicHorizonWeb(newDB.DSN) if err != nil { @@ -786,17 +786,17 @@ func TestReingestDBWithFilterRules(t *testing.T) { itest.StopHorizon() // clear the db with reaping all ledgers - horizoncmd.ResetCmds() - horizoncmd.RootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", + var rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", "reap", "--history-retention-count=1", )) - tt.NoError(horizoncmd.RootCmd.Execute()) + tt.NoError(rootCmd.Execute()) // repopulate the db with reingestion which should catchup using core reapply filter rules // correctly on reingestion ranged - horizoncmd.ResetCmds() - horizoncmd.RootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", + rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, itest.GetHorizonIngestConfig(), "db", "reingest", "range", "1", @@ -941,9 +941,9 @@ func TestFillGaps(t *testing.T) { }) tt.NoError(err) - horizoncmd.ResetCmds() t.Run("validate parallel range", func(t *testing.T) { - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, + var rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "--parallel-workers=2", @@ -951,7 +951,7 @@ func TestFillGaps(t *testing.T) { "2", )) - assert.EqualError(t, horizoncmd.RootCmd.Execute(), "Invalid range: {10 2} from > to") + assert.EqualError(t, rootCmd.Execute(), "Invalid range: {10 2} from > to") }) // make sure a full checkpoint has elapsed otherwise there will be nothing to reingest From 9981bac99f5be78a5efb148d07f0fd156c69f130 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 12 Jul 2024 11:57:27 -0700 Subject: [PATCH 10/13] #4911: use rootcmd instance not package singleton in tests --- services/horizon/cmd/db_test.go | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index 463df6d447..d3fbcaf345 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -24,20 +24,6 @@ type DBCommandsTestSuite struct { rootCmd *cobra.Command } -func (s *DBCommandsTestSuite) SetupTest() { - resetFlags() -} - -func resetFlags() { - RootCmd.ResetFlags() - dbFillGapsCmd.ResetFlags() - dbReingestRangeCmd.ResetFlags() - - globalFlags.Init(RootCmd) - dbFillGapsCmdOpts.Init(dbFillGapsCmd) - dbReingestRangeCmdOpts.Init(dbReingestRangeCmd) -} - func (s *DBCommandsTestSuite) SetupSuite() { runDBReingestRangeFn = func([]history.LedgerRange, bool, uint, horizon.Config, ingest.StorageBackendConfig) error { @@ -258,21 +244,21 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() { for _, command := range commands { for _, tt := range tests { s.T().Run(tt.name+"_"+command.name, func(t *testing.T) { - resetFlags() + rootCmd := NewRootCmd() var args []string args = append(command.cmd, tt.args...) - RootCmd.SetArgs(append([]string{ - "--db-url", s.dsn, + rootCmd.SetArgs(append([]string{ + "--db-url", s.db.DSN, "--stellar-core-binary-path", "/test/core/bin/path", }, args...)) if tt.expectError { - err := RootCmd.Execute() + err := rootCmd.Execute() require.Error(t, err) require.Contains(t, err.Error(), tt.errorMessage) } else { - require.NoError(t, RootCmd.Execute()) + require.NoError(t, rootCmd.Execute()) } }) } From a3c452e1ab441afec77ccfcadba46f0e60dfebce Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 12 Jul 2024 13:21:34 -0700 Subject: [PATCH 11/13] #4911: use flag config instance not global for test root cmd execution --- services/horizon/cmd/db.go | 12 ++++++------ services/horizon/internal/integration/db_test.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index cd62cd1d3c..a07bc0ee10 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -76,13 +76,13 @@ func requireAndSetFlags(horizonFlags config.ConfigOptions, names ...string) erro return fmt.Errorf("could not find %s flags", strings.Join(missing, ",")) } -func migrate(dir schema.MigrateDir, count int) error { - if !globalConfig.Ingest { +func migrate(dir schema.MigrateDir, count int, horizonConfig *horizon.Config) error { + if !horizonConfig.Ingest { log.Println("Skipping migrations because ingest flag is not enabled") return nil } - dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) + dbConn, err := db.Open("postgres", horizonConfig.DatabaseURL) if err != nil { return err } @@ -335,7 +335,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor return ErrUsage{cmd} } - return migrate(schema.MigrateDown, count) + return migrate(schema.MigrateDown, count, horizonConfig) }, } @@ -359,7 +359,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor return ErrUsage{cmd} } - return migrate(schema.MigrateRedo, count) + return migrate(schema.MigrateRedo, count, horizonConfig) }, } @@ -417,7 +417,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor } } - return migrate(schema.MigrateUp, count) + return migrate(schema.MigrateUp, count, horizonConfig) }, } diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index c820ae7cd8..9dc211db29 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -803,7 +803,7 @@ func TestReingestDBWithFilterRules(t *testing.T) { fmt.Sprintf("%d", reachedLedger), )) - tt.NoError(horizoncmd.RootCmd.Execute()) + tt.NoError(rootCmd.Execute()) // bring up horizon, just the api server no ingestion, to query // for tx's that should have been repopulated on db from reingestion per From 50fef11e56726b0e14a36e59e23c6124a6560d74 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 12 Jul 2024 14:10:34 -0700 Subject: [PATCH 12/13] #4911: use root cmd instance not package var for reingest parallel test --- .../horizon/internal/integration/db_test.go | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 9dc211db29..b0a011f45b 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -983,21 +983,25 @@ func TestFillGaps(t *testing.T) { filepath.Dir(horizonConfig.CaptiveCoreConfigPath), "captive-core-reingest-range-integration-tests.cfg", ) - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "--parallel-workers=1")) - tt.NoError(horizoncmd.RootCmd.Execute()) + + rootCmd := horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "--parallel-workers=1")) + tt.NoError(rootCmd.Execute()) tt.NoError(historyQ.LatestLedger(context.Background(), &latestLedger)) tt.Equal(int64(0), latestLedger) - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "3", "4")) - tt.NoError(horizoncmd.RootCmd.Execute()) + rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "3", "4")) + tt.NoError(rootCmd.Execute()) tt.NoError(historyQ.LatestLedger(context.Background(), &latestLedger)) tt.NoError(historyQ.ElderLedger(context.Background(), &oldestLedger)) tt.Equal(int64(3), oldestLedger) tt.Equal(int64(4), latestLedger) - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "6", "7")) - tt.NoError(horizoncmd.RootCmd.Execute()) + rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "6", "7")) + tt.NoError(rootCmd.Execute()) tt.NoError(historyQ.LatestLedger(context.Background(), &latestLedger)) tt.NoError(historyQ.ElderLedger(context.Background(), &oldestLedger)) tt.Equal(int64(3), oldestLedger) @@ -1007,8 +1011,9 @@ func TestFillGaps(t *testing.T) { tt.NoError(err) tt.Equal([]history.LedgerRange{{StartSequence: 5, EndSequence: 5}}, gaps) - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps")) - tt.NoError(horizoncmd.RootCmd.Execute()) + rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps")) + tt.NoError(rootCmd.Execute()) tt.NoError(historyQ.LatestLedger(context.Background(), &latestLedger)) tt.NoError(historyQ.ElderLedger(context.Background(), &oldestLedger)) tt.Equal(int64(3), oldestLedger) @@ -1017,8 +1022,9 @@ func TestFillGaps(t *testing.T) { tt.NoError(err) tt.Empty(gaps) - horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "2", "8")) - tt.NoError(horizoncmd.RootCmd.Execute()) + rootCmd = horizoncmd.NewRootCmd() + rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "2", "8")) + tt.NoError(rootCmd.Execute()) tt.NoError(historyQ.LatestLedger(context.Background(), &latestLedger)) tt.NoError(historyQ.ElderLedger(context.Background(), &oldestLedger)) tt.Equal(int64(2), oldestLedger) From b1573d0d5a4fe5e8f70ac5fa6da700a7c8079e83 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 12 Jul 2024 14:14:23 -0700 Subject: [PATCH 13/13] #4911: use root cmd instance not package var for migrate tests --- services/horizon/internal/integration/db_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index b0a011f45b..86a86a8055 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -873,12 +873,13 @@ func TestMigrateIngestIsTrueByDefault(t *testing.T) { newDB := dbtest.Postgres(t) freshHorizonPostgresURL := newDB.DSN - horizoncmd.RootCmd.SetArgs([]string{ + rootCmd := horizoncmd.NewRootCmd() + rootCmd.SetArgs([]string{ // ingest is set to true by default "--db-url", freshHorizonPostgresURL, "db", "migrate", "up", }) - tt.NoError(horizoncmd.RootCmd.Execute()) + tt.NoError(rootCmd.Execute()) dbConn, err := db.Open("postgres", freshHorizonPostgresURL) tt.NoError(err) @@ -894,12 +895,13 @@ func TestMigrateChecksIngestFlag(t *testing.T) { newDB := dbtest.Postgres(t) freshHorizonPostgresURL := newDB.DSN - horizoncmd.RootCmd.SetArgs([]string{ + rootCmd := horizoncmd.NewRootCmd() + rootCmd.SetArgs([]string{ "--ingest=false", "--db-url", freshHorizonPostgresURL, "db", "migrate", "up", }) - tt.NoError(horizoncmd.RootCmd.Execute()) + tt.NoError(rootCmd.Execute()) dbConn, err := db.Open("postgres", freshHorizonPostgresURL) tt.NoError(err)