From 3ce1cea36919938d9ad21762c957100dc14580b3 Mon Sep 17 00:00:00 2001 From: shawn Date: Wed, 11 Oct 2023 11:49:10 -0700 Subject: [PATCH] cmd: limit global flags displayed on cli help output (#5077) --- services/horizon/CHANGELOG.md | 1 + services/horizon/cmd/db.go | 26 +- services/horizon/cmd/ingest.go | 118 ++--- services/horizon/cmd/record_metrics.go | 4 +- services/horizon/cmd/root.go | 64 ++- services/horizon/cmd/serve.go | 2 +- services/horizon/internal/flags.go | 437 ++++++++++-------- .../internal/integration/parameters_test.go | 141 ++++-- support/config/config_option.go | 9 +- 9 files changed, 515 insertions(+), 287 deletions(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 0a08373678..8bf83b5ad6 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -15,6 +15,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - The same slippage calculation from the [`v2.26.1`](#2261) hotfix now properly excludes spikes for smoother trade aggregation plots ([4999](https://github.com/stellar/go/pull/4999)). +- Limit the display of global flags on command line help `-h` output ([5077](https://github.com/stellar/go/pull/5077)). - Add a deprecation warning for using command-line flags when running Horizon ([5051](https://github.com/stellar/go/pull/5051)) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 59ffc8701c..e23c224012 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -38,7 +38,7 @@ func requireAndSetFlags(names ...string) error { for _, name := range names { set[name] = true } - for _, flag := range flags { + for _, flag := range globalFlags { if set[flag.Name] { flag.Require() if err := flag.SetValue(); err != nil { @@ -66,7 +66,7 @@ var dbInitCmd = &cobra.Command{ return err } - db, err := sql.Open("postgres", config.DatabaseURL) + db, err := sql.Open("postgres", globalConfig.DatabaseURL) if err != nil { return err } @@ -86,12 +86,12 @@ var dbInitCmd = &cobra.Command{ } func migrate(dir schema.MigrateDir, count int) error { - if !config.Ingest { + if !globalConfig.Ingest { log.Println("Skipping migrations because ingest flag is not enabled") return nil } - dbConn, err := db.Open("postgres", config.DatabaseURL) + dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return err } @@ -172,7 +172,7 @@ var dbMigrateStatusCmd = &cobra.Command{ return ErrUsage{cmd} } - dbConn, err := db.Open("postgres", config.DatabaseURL) + dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return err } @@ -220,7 +220,7 @@ var dbReapCmd = &cobra.Command{ Short: "reaps (i.e. removes) any reapable history data", Long: "reap removes any historical data that is earlier than the configured retention cutoff", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(config, flags) + app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) if err != nil { return err } @@ -321,7 +321,7 @@ var dbReingestRangeCmd = &cobra.Command{ } } - err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}) + err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}) if err != nil { return err } @@ -329,7 +329,7 @@ var dbReingestRangeCmd = &cobra.Command{ []history.LedgerRange{{StartSequence: argsUInt32[0], EndSequence: argsUInt32[1]}}, reingestForce, parallelWorkers, - *config, + *globalConfig, ) }, } @@ -369,26 +369,26 @@ var dbFillGapsCmd = &cobra.Command{ withRange = true } - err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}) + err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}) if err != nil { return err } var gaps []history.LedgerRange if withRange { - gaps, err = runDBDetectGapsInRange(*config, uint32(start), uint32(end)) + gaps, err = runDBDetectGapsInRange(*globalConfig, uint32(start), uint32(end)) if err != nil { return err } hlog.Infof("found gaps %v within range [%v, %v]", gaps, start, end) } else { - gaps, err = runDBDetectGaps(*config) + gaps, err = runDBDetectGaps(*globalConfig) if err != nil { return err } hlog.Infof("found gaps %v", gaps) } - return runDBReingestRange(gaps, reingestForce, parallelWorkers, *config) + return runDBReingestRange(gaps, reingestForce, parallelWorkers, *globalConfig) }, } @@ -479,7 +479,7 @@ var dbDetectGapsCmd = &cobra.Command{ if len(args) != 0 { return ErrUsage{cmd} } - gaps, err := runDBDetectGaps(*config) + gaps, err := runDBDetectGaps(*globalConfig) if err != nil { return err } diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index 42a62449d8..1c44e064d9 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -94,7 +94,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } @@ -111,11 +111,11 @@ var ingestVerifyRangeCmd = &cobra.Command{ }() } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } - mngr := historyarchive.NewCheckpointManager(config.CheckpointFrequency) + mngr := historyarchive.NewCheckpointManager(globalConfig.CheckpointFrequency) if !mngr.IsCheckpoint(ingestVerifyFrom) && ingestVerifyFrom != 1 { return fmt.Errorf("`--from` must be a checkpoint ledger") } @@ -125,26 +125,26 @@ var ingestVerifyRangeCmd = &cobra.Command{ } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath, - CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB, - RemoteCaptiveCoreURL: config.RemoteCaptiveCoreURL, - CheckpointFrequency: config.CheckpointFrequency, - CaptiveCoreToml: config.CaptiveCoreToml, - CaptiveCoreStoragePath: config.CaptiveCoreStoragePath, - RoundingSlippageFilter: config.RoundingSlippageFilter, - EnableIngestionFiltering: config.EnableIngestionFiltering, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, + CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, + RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, + CheckpointFrequency: globalConfig.CheckpointFrequency, + CaptiveCoreToml: globalConfig.CaptiveCoreToml, + CaptiveCoreStoragePath: globalConfig.CaptiveCoreStoragePath, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } if !ingestConfig.EnableCaptiveCore { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } @@ -203,11 +203,11 @@ var ingestStressTestCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -221,23 +221,23 @@ var ingestStressTestCmd = &cobra.Command{ } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - RoundingSlippageFilter: config.RoundingSlippageFilter, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, } - if config.EnableCaptiveCoreIngestion { - ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath - ingestConfig.RemoteCaptiveCoreURL = config.RemoteCaptiveCoreURL - ingestConfig.CaptiveCoreConfigUseDB = config.CaptiveCoreConfigUseDB + if globalConfig.EnableCaptiveCoreIngestion { + ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath + ingestConfig.RemoteCaptiveCoreURL = globalConfig.RemoteCaptiveCoreURL + ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB } else { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } @@ -267,11 +267,11 @@ var ingestTriggerStateRebuildCmd = &cobra.Command{ Short: "updates a database to trigger state rebuild, state will be rebuilt by a running Horizon instance, DO NOT RUN production DB, some endpoints will be unavailable until state is rebuilt", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -291,11 +291,11 @@ var ingestInitGenesisStateCmd = &cobra.Command{ Short: "ingests genesis state (ledger 1)", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -312,24 +312,24 @@ var ingestInitGenesisStateCmd = &cobra.Command{ } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - CheckpointFrequency: config.CheckpointFrequency, - RoundingSlippageFilter: config.RoundingSlippageFilter, - EnableIngestionFiltering: config.EnableIngestionFiltering, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + CheckpointFrequency: globalConfig.CheckpointFrequency, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } - if config.EnableCaptiveCoreIngestion { - ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath - ingestConfig.CaptiveCoreConfigUseDB = config.CaptiveCoreConfigUseDB + if globalConfig.EnableCaptiveCoreIngestion { + ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath + ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB } else { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } @@ -363,11 +363,11 @@ var ingestBuildStateCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -383,33 +383,33 @@ var ingestBuildStateCmd = &cobra.Command{ return fmt.Errorf("cannot run on non-empty DB") } - mngr := historyarchive.NewCheckpointManager(config.CheckpointFrequency) + mngr := historyarchive.NewCheckpointManager(globalConfig.CheckpointFrequency) if !mngr.IsCheckpoint(ingestBuildStateSequence) && ingestBuildStateSequence != 1 { return fmt.Errorf("`--sequence` must be a checkpoint ledger") } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath, - CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB, - RemoteCaptiveCoreURL: config.RemoteCaptiveCoreURL, - CheckpointFrequency: config.CheckpointFrequency, - CaptiveCoreToml: config.CaptiveCoreToml, - CaptiveCoreStoragePath: config.CaptiveCoreStoragePath, - RoundingSlippageFilter: config.RoundingSlippageFilter, - EnableIngestionFiltering: config.EnableIngestionFiltering, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, + CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, + RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, + CheckpointFrequency: globalConfig.CheckpointFrequency, + CaptiveCoreToml: globalConfig.CaptiveCoreToml, + CaptiveCoreStoragePath: globalConfig.CaptiveCoreStoragePath, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } if !ingestBuildStateSkipChecks { if !ingestConfig.EnableCaptiveCore { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } diff --git a/services/horizon/cmd/record_metrics.go b/services/horizon/cmd/record_metrics.go index cd4f335da1..fb6244b76b 100644 --- a/services/horizon/cmd/record_metrics.go +++ b/services/horizon/cmd/record_metrics.go @@ -19,7 +19,7 @@ var recordMetricsCmd = &cobra.Command{ Short: "records `/metrics` on admin port for debuging purposes", Long: "", RunE: func(cmd *cobra.Command, args []string) error { - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{}); err != nil { return err } @@ -50,7 +50,7 @@ var recordMetricsCmd = &cobra.Command{ time.Duration(time.Duration(scrapeIntervalSeconds*(scrapesCount-i))*time.Second), ) - metricsResponse, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", config.AdminPort)) + metricsResponse, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", globalConfig.AdminPort)) if err != nil { return errors.Wrap(err, "Error fetching metrics. Is admin server running?") } diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index ee8c18db43..d2900496d4 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -6,10 +6,11 @@ import ( "github.com/spf13/cobra" horizon "github.com/stellar/go/services/horizon/internal" + "github.com/stellar/go/support/config" ) var ( - config, flags = horizon.Flags() + globalConfig, globalFlags = horizon.Flags() RootCmd = &cobra.Command{ Use: "horizon", @@ -22,13 +23,15 @@ var ( "DEPRECATED - the use of command-line flags has been deprecated in favor of environment variables. Please" + "consult our Configuring section in the developer documentation on how to use them - https://developers.stellar.org/docs/run-api-server/configuring", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(config, flags) + app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) if err != nil { return err } return app.Serve() }, } + originalHelpFunc = RootCmd.HelpFunc() + originalUsageFunc = RootCmd.UsageFunc() ) // ErrUsage indicates we should print the usage string and exit with code 1 @@ -48,7 +51,20 @@ func (e ErrExitCode) Error() string { } func init() { - err := flags.Init(RootCmd) + + // override the default help output, apply further filtering on which global flags + // will be shown on the help outout dependent on the command help was issued upon. + RootCmd.SetHelpFunc(func(c *cobra.Command, args []string) { + enableGlobalOptionsInHelp(c, globalFlags) + originalHelpFunc(c, args) + }) + + RootCmd.SetUsageFunc(func(c *cobra.Command) error { + enableGlobalOptionsInHelp(c, globalFlags) + return originalUsageFunc(c) + }) + + err := globalFlags.Init(RootCmd) if err != nil { stdLog.Fatal(err.Error()) } @@ -57,3 +73,45 @@ func init() { func Execute() error { return RootCmd.Execute() } + +func enableGlobalOptionsInHelp(cmd *cobra.Command, cos config.ConfigOptions) { + for _, co := range cos { + if co.Hidden { + // this options was configured statically to be hidden + // Init() has already set that once, leave it as-is. + continue + } + + // we don't want' to display global flags in help output + // if no sub-command context given yet, i.e. just '-h' was used + // or there are subcomands required to use. + if cmd.Parent() == nil || cmd.HasAvailableSubCommands() { + co.ToggleHidden(true) + continue + } + + if len(co.UsedInCommands) > 0 && + !contains(co.UsedInCommands, cmd) { + co.ToggleHidden(true) + } else { + co.ToggleHidden(false) + } + } +} + +// check if this command or any of it's sub-level parents match +// supportedCommands +func contains(supportedCommands []string, cmd *cobra.Command) bool { + for _, supportedCommand := range supportedCommands { + if supportedCommand == cmd.Name() { + return true + } + } + + // don't do inheritance matching on the top most sub-commands. + // they are second level deep, the horizon itself is top level. + if cmd.Parent() != nil && cmd.Parent().Parent() != nil { + return contains(supportedCommands, cmd.Parent()) + } + return false +} diff --git a/services/horizon/cmd/serve.go b/services/horizon/cmd/serve.go index 8e06855d3c..21a34da3ad 100644 --- a/services/horizon/cmd/serve.go +++ b/services/horizon/cmd/serve.go @@ -10,7 +10,7 @@ var serveCmd = &cobra.Command{ Short: "run horizon server", Long: "serve initializes then starts the horizon HTTP server", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(config, flags) + app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) if err != nil { return err } diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index e980873aaf..4f18aacc63 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -67,6 +67,32 @@ const ( defaultMaxHTTPRequestSize = uint(200 * 1024) ) +var ( + IngestCmd = "ingest" + RecordMetricsCmd = "record-metrics" + DbCmd = "db" + ServeCmd = "serve" + HorizonCmd = "horizon" + + DbFillGapsCmd = "fill-gaps" + DbReingestCmd = "reingest" + IngestTriggerStateRebuild = "trigger-state-rebuild" + IngestInitGenesisStateCmd = "init-genesis-state" + IngestBuildStateCmd = "build-state" + IngestStressTestCmd = "stress-test" + IngestVerifyRangeCmd = "verify-range" + + ApiServerCommands = []string{HorizonCmd, ServeCmd} + IngestionCommands = append(ApiServerCommands, + IngestInitGenesisStateCmd, + IngestBuildStateCmd, + IngestStressTestCmd, + IngestVerifyRangeCmd, + DbFillGapsCmd, + DbReingestCmd) + DatabaseBoundCommands = append(ApiServerCommands, DbCmd, IngestCmd) +) + // validateBothOrNeither ensures that both options are provided, if either is provided. func validateBothOrNeither(option1, option2 string) error { arg1, arg2 := viper.GetString(option1), viper.GetString(option2) @@ -130,36 +156,40 @@ func Flags() (*Config, support.ConfigOptions) { // Add a new entry here to connect a new field in the horizon.Config struct var flags = support.ConfigOptions{ &support.ConfigOption{ - Name: DatabaseURLFlagName, - EnvVar: "DATABASE_URL", - ConfigKey: &config.DatabaseURL, - OptType: types.String, - Required: true, - Usage: "horizon postgres database to connect with", + Name: DatabaseURLFlagName, + EnvVar: "DATABASE_URL", + ConfigKey: &config.DatabaseURL, + OptType: types.String, + Required: true, + Usage: "horizon postgres database to connect with", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "ro-database-url", - ConfigKey: &config.RoDatabaseURL, - OptType: types.String, - Required: false, - Usage: "horizon postgres read-replica to connect with, when set it will return stale history error when replica is behind primary", + Name: "ro-database-url", + ConfigKey: &config.RoDatabaseURL, + OptType: types.String, + Required: false, + Usage: "horizon postgres read-replica to connect with, when set it will return stale history error when replica is behind primary", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: StellarCoreBinaryPathName, - OptType: types.String, - FlagDefault: "", - Required: false, - Usage: "path to stellar core binary, look for the stellar-core binary in $PATH by default.", - ConfigKey: &config.CaptiveCoreBinaryPath, + Name: StellarCoreBinaryPathName, + OptType: types.String, + FlagDefault: "", + Required: false, + Usage: "path to stellar core binary, look for the stellar-core binary in $PATH by default.", + ConfigKey: &config.CaptiveCoreBinaryPath, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: DisableTxSubFlagName, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "disables the transaction submission functionality of Horizon.", - ConfigKey: &config.DisableTxSub, - Hidden: false, + Name: DisableTxSubFlagName, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "disables the transaction submission functionality of Horizon.", + ConfigKey: &config.DisableTxSub, + Hidden: false, + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: captiveCoreConfigAppendPathName, @@ -183,6 +213,7 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: CaptiveCoreConfigPathName, @@ -197,6 +228,7 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: CaptiveCoreConfigUseDB, @@ -215,15 +247,17 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, - ConfigKey: &config.CaptiveCoreConfigUseDB, + ConfigKey: &config.CaptiveCoreConfigUseDB, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "enable-captive-core-ingestion", - OptType: types.Bool, - FlagDefault: true, - Required: false, - Usage: "causes Horizon to ingest from a Captive Stellar Core process instead of a persistent Stellar Core database", - ConfigKey: &config.EnableCaptiveCoreIngestion, + Name: "enable-captive-core-ingestion", + OptType: types.Bool, + FlagDefault: true, + Required: false, + Usage: "causes Horizon to ingest from a Captive Stellar Core process instead of a persistent Stellar Core database", + ConfigKey: &config.EnableCaptiveCoreIngestion, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: EnableIngestionFilteringFlagName, @@ -256,6 +290,7 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: uint(0), Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", ConfigKey: &config.CaptiveCoreTomlParams.HTTPPort, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "captive-core-storage-path", @@ -272,9 +307,10 @@ func Flags() (*Config, support.ConfigOptions) { *opt.ConfigKey.(*string) = existingValue return nil }, - Required: false, - Usage: "Storage location for Captive Core bucket data. If not set, the current working directory is used as the default location.", - ConfigKey: &config.CaptiveCoreStoragePath, + Required: false, + Usage: "Storage location for Captive Core bucket data. If not set, the current working directory is used as the default location.", + ConfigKey: &config.CaptiveCoreStoragePath, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "captive-core-peer-port", @@ -284,20 +320,23 @@ func Flags() (*Config, support.ConfigOptions) { Required: false, Usage: "port for Captive Core to bind to for connecting to the Stellar swarm (0 uses Stellar Core's default)", ConfigKey: &config.CaptiveCoreTomlParams.PeerPort, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: StellarCoreDBURLFlagName, - EnvVar: "STELLAR_CORE_DATABASE_URL", - ConfigKey: &config.StellarCoreDatabaseURL, - OptType: types.String, - Required: false, - Usage: "stellar-core postgres database to connect with", + Name: StellarCoreDBURLFlagName, + EnvVar: "STELLAR_CORE_DATABASE_URL", + ConfigKey: &config.StellarCoreDatabaseURL, + OptType: types.String, + Required: false, + Usage: "stellar-core postgres database to connect with", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: StellarCoreURLFlagName, - ConfigKey: &config.StellarCoreURL, - OptType: types.String, - Usage: "stellar-core to connect with (for http commands). If unset and the local Captive core is enabled, it will use http://localhost:", + Name: StellarCoreURLFlagName, + ConfigKey: &config.StellarCoreURL, + OptType: types.String, + Usage: "stellar-core to connect with (for http commands). If unset and the local Captive core is enabled, it will use http://localhost:", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: HistoryArchiveURLsFlagName, @@ -315,42 +354,48 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, - Usage: "comma-separated list of stellar history archives to connect with", + Usage: "comma-separated list of stellar history archives to connect with", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "port", - ConfigKey: &config.Port, - OptType: types.Uint, - FlagDefault: uint(8000), - Usage: "tcp port to listen on for http requests", + Name: "port", + ConfigKey: &config.Port, + OptType: types.Uint, + FlagDefault: uint(8000), + Usage: "tcp port to listen on for http requests", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "admin-port", - ConfigKey: &config.AdminPort, - OptType: types.Uint, - FlagDefault: uint(0), - Usage: "WARNING: this should not be accessible from the Internet and does not use TLS, tcp port to listen on for admin http requests, 0 (default) disables the admin server", + Name: "admin-port", + ConfigKey: &config.AdminPort, + OptType: types.Uint, + FlagDefault: uint(0), + Usage: "WARNING: this should not be accessible from the Internet and does not use TLS, tcp port to listen on for admin http requests, 0 (default) disables the admin server", + UsedInCommands: append(ApiServerCommands, RecordMetricsCmd), }, &support.ConfigOption{ - Name: "max-db-connections", - ConfigKey: &config.MaxDBConnections, - OptType: types.Int, - FlagDefault: 0, - Usage: "when set has a priority over horizon-db-max-open-connections, horizon-db-max-idle-connections. max horizon database open connections may need to be increased when responses are slow but DB CPU is normal", + Name: "max-db-connections", + ConfigKey: &config.MaxDBConnections, + OptType: types.Int, + FlagDefault: 0, + Usage: "when set has a priority over horizon-db-max-open-connections, horizon-db-max-idle-connections. max horizon database open connections may need to be increased when responses are slow but DB CPU is normal", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "horizon-db-max-open-connections", - ConfigKey: &config.HorizonDBMaxOpenConnections, - OptType: types.Int, - FlagDefault: 20, - Usage: "max horizon database open connections. may need to be increased when responses are slow but DB CPU is normal", + Name: "horizon-db-max-open-connections", + ConfigKey: &config.HorizonDBMaxOpenConnections, + OptType: types.Int, + FlagDefault: 20, + Usage: "max horizon database open connections. may need to be increased when responses are slow but DB CPU is normal", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "horizon-db-max-idle-connections", - ConfigKey: &config.HorizonDBMaxIdleConnections, - OptType: types.Int, - FlagDefault: 20, - Usage: "max horizon database idle connections. may need to be set to the same value as horizon-db-max-open-connections when responses are slow and DB CPU is normal, because it may indicate that a lot of time is spent closing/opening idle connections. This can happen in case of high variance in number of requests. must be equal or lower than max open connections", + Name: "horizon-db-max-idle-connections", + ConfigKey: &config.HorizonDBMaxIdleConnections, + OptType: types.Int, + FlagDefault: 20, + Usage: "max horizon database idle connections. may need to be set to the same value as horizon-db-max-open-connections when responses are slow and DB CPU is normal, because it may indicate that a lot of time is spent closing/opening idle connections. This can happen in case of high variance in number of requests. must be equal or lower than max open connections", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ Name: "sse-update-frequency", @@ -359,6 +404,7 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: 5, CustomSetValue: support.SetDuration, Usage: "defines how often streams should check if there's a new ledger (in seconds), may need to increase in case of big number of streams", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "connection-timeout", @@ -367,13 +413,15 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: 55, CustomSetValue: support.SetDuration, Usage: "defines the timeout of connection after which 504 response will be sent or stream will be closed, if Horizon is behind a load balancer with idle connection timeout, this should be set to a few seconds less that idle timeout, does not apply to POST /transactions", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "max-http-request-size", - ConfigKey: &config.MaxHTTPRequestSize, - OptType: types.Uint, - FlagDefault: defaultMaxHTTPRequestSize, - Usage: "sets the limit on the maximum allowed http request payload size, default is 200kb, to disable the limit check, set to 0, only do so if you acknowledge the implications of accepting unbounded http request payload sizes.", + Name: "max-http-request-size", + ConfigKey: &config.MaxHTTPRequestSize, + OptType: types.Uint, + FlagDefault: defaultMaxHTTPRequestSize, + Usage: "sets the limit on the maximum allowed http request payload size, default is 200kb, to disable the limit check, set to 0, only do so if you acknowledge the implications of accepting unbounded http request payload sizes.", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "per-hour-rate-limit", @@ -392,7 +440,8 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, - Usage: "max count of requests allowed in a one hour period, by remote ip address", + Usage: "max count of requests allowed in a one hour period, by remote ip address", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "friendbot-url", @@ -400,6 +449,7 @@ func Flags() (*Config, support.ConfigOptions) { OptType: types.String, CustomSetValue: support.SetURL, Usage: "friendbot service to redirect to", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "log-level", @@ -429,36 +479,41 @@ func Flags() (*Config, support.ConfigOptions) { CustomSetValue: support.SetOptionalString, Required: false, Usage: "name of the path for Core logs (leave empty to log w/ Horizon only)", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "max-path-length", - ConfigKey: &config.MaxPathLength, - OptType: types.Uint, - FlagDefault: uint(3), - Usage: "the maximum number of assets on the path in `/paths` endpoint, warning: increasing this value will increase /paths response time", + Name: "max-path-length", + ConfigKey: &config.MaxPathLength, + OptType: types.Uint, + FlagDefault: uint(3), + Usage: "the maximum number of assets on the path in `/paths` endpoint, warning: increasing this value will increase /paths response time", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "max-assets-per-path-request", - ConfigKey: &config.MaxAssetsPerPathRequest, - OptType: types.Int, - FlagDefault: int(15), - Usage: "the maximum number of assets in '/paths/strict-send' and '/paths/strict-receive' endpoints", + Name: "max-assets-per-path-request", + ConfigKey: &config.MaxAssetsPerPathRequest, + OptType: types.Int, + FlagDefault: int(15), + Usage: "the maximum number of assets in '/paths/strict-send' and '/paths/strict-receive' endpoints", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "disable-pool-path-finding", - ConfigKey: &config.DisablePoolPathFinding, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "excludes liquidity pools from consideration in the `/paths` endpoint", + Name: "disable-pool-path-finding", + ConfigKey: &config.DisablePoolPathFinding, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "excludes liquidity pools from consideration in the `/paths` endpoint", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "disable-path-finding", - ConfigKey: &config.DisablePathFinding, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "disables the path finding endpoints", + Name: "disable-path-finding", + ConfigKey: &config.DisablePathFinding, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "disables the path finding endpoints", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "max-path-finding-requests", @@ -468,19 +523,22 @@ func Flags() (*Config, support.ConfigOptions) { Required: false, Usage: "The maximum number of path finding requests per second horizon will allow." + " A value of zero (the default) disables the limit.", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: NetworkPassphraseFlagName, - ConfigKey: &config.NetworkPassphrase, - OptType: types.String, - Required: false, - Usage: "Override the network passphrase", + Name: NetworkPassphraseFlagName, + ConfigKey: &config.NetworkPassphrase, + OptType: types.String, + Required: false, + Usage: "Override the network passphrase", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "sentry-dsn", - ConfigKey: &config.SentryDSN, - OptType: types.String, - Usage: "Sentry URL to which panics and errors should be reported", + Name: "sentry-dsn", + ConfigKey: &config.SentryDSN, + OptType: types.String, + Usage: "Sentry URL to which panics and errors should be reported", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "loggly-token", @@ -496,59 +554,67 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "Tag to be added to every loggly log event", }, &support.ConfigOption{ - Name: "tls-cert", - ConfigKey: &config.TLSCert, - OptType: types.String, - Usage: "TLS certificate file to use for securing connections to horizon", + Name: "tls-cert", + ConfigKey: &config.TLSCert, + OptType: types.String, + Usage: "TLS certificate file to use for securing connections to horizon", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "tls-key", - ConfigKey: &config.TLSKey, - OptType: types.String, - Usage: "TLS private key file to use for securing connections to horizon", + Name: "tls-key", + ConfigKey: &config.TLSKey, + OptType: types.String, + Usage: "TLS private key file to use for securing connections to horizon", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: IngestFlagName, - ConfigKey: &config.Ingest, - OptType: types.Bool, - FlagDefault: true, - Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", + Name: IngestFlagName, + ConfigKey: &config.Ingest, + OptType: types.Bool, + FlagDefault: true, + Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "cursor-name", - EnvVar: "CURSOR_NAME", - ConfigKey: &config.CursorName, - OptType: types.String, - FlagDefault: "HORIZON", - Usage: "ingestor cursor used by horizon to ingest from stellar core. must be uppercase and unique for each horizon instance ingesting from that core instance.", + Name: "cursor-name", + EnvVar: "CURSOR_NAME", + ConfigKey: &config.CursorName, + OptType: types.String, + FlagDefault: "HORIZON", + Usage: "ingestor cursor used by horizon to ingest from stellar core. must be uppercase and unique for each horizon instance ingesting from that core instance.", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "history-retention-count", - ConfigKey: &config.HistoryRetentionCount, - OptType: types.Uint, - FlagDefault: uint(0), - Usage: "the minimum number of ledgers to maintain within horizon's history tables. 0 signifies an unlimited number of ledgers will be retained", + Name: "history-retention-count", + ConfigKey: &config.HistoryRetentionCount, + OptType: types.Uint, + FlagDefault: uint(0), + Usage: "the minimum number of ledgers to maintain within horizon's history tables. 0 signifies an unlimited number of ledgers will be retained", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "history-stale-threshold", - ConfigKey: &config.StaleThreshold, - OptType: types.Uint, - FlagDefault: uint(0), - Usage: "the maximum number of ledgers the history db is allowed to be out of date from the connected stellar-core db before horizon considers history stale", + Name: "history-stale-threshold", + ConfigKey: &config.StaleThreshold, + OptType: types.Uint, + FlagDefault: uint(0), + Usage: "the maximum number of ledgers the history db is allowed to be out of date from the connected stellar-core db before horizon considers history stale", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "skip-cursor-update", - ConfigKey: &config.SkipCursorUpdate, - OptType: types.Bool, - FlagDefault: false, - Usage: "causes the ingester to skip reporting the last imported ledger state to stellar-core", + Name: "skip-cursor-update", + ConfigKey: &config.SkipCursorUpdate, + OptType: types.Bool, + FlagDefault: false, + Usage: "causes the ingester to skip reporting the last imported ledger state to stellar-core", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "ingest-disable-state-verification", - ConfigKey: &config.IngestDisableStateVerification, - OptType: types.Bool, - FlagDefault: false, - Usage: "ingestion system runs a verification routing to compare state in local database with history buckets, this can be disabled however it's not recommended", + Name: "ingest-disable-state-verification", + ConfigKey: &config.IngestDisableStateVerification, + OptType: types.Bool, + FlagDefault: false, + Usage: "ingestion system runs a verification routing to compare state in local database with history buckets, this can be disabled however it's not recommended", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "ingest-state-verification-checkpoint-frequency", @@ -558,6 +624,7 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "the frequency in units per checkpoint for how often state verification is executed. " + "A value of 1 implies running state verification on every checkpoint. " + "A value of 2 implies running state verification on every second checkpoint.", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "ingest-state-verification-timeout", @@ -567,53 +634,60 @@ func Flags() (*Config, support.ConfigOptions) { CustomSetValue: support.SetDurationMinutes, Usage: "defines an upper bound in minutes for on how long state verification is allowed to run. " + "A value of 0 disables the timeout.", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "ingest-enable-extended-log-ledger-stats", - ConfigKey: &config.IngestEnableExtendedLogLedgerStats, - OptType: types.Bool, - FlagDefault: false, - Usage: "enables extended ledger stats in the log (ledger entry changes and operations stats)", + Name: "ingest-enable-extended-log-ledger-stats", + ConfigKey: &config.IngestEnableExtendedLogLedgerStats, + OptType: types.Bool, + FlagDefault: false, + Usage: "enables extended ledger stats in the log (ledger entry changes and operations stats)", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "apply-migrations", - ConfigKey: &config.ApplyMigrations, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "applies pending migrations before starting horizon", + Name: "apply-migrations", + ConfigKey: &config.ApplyMigrations, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "applies pending migrations before starting horizon", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "checkpoint-frequency", - ConfigKey: &config.CheckpointFrequency, - OptType: types.Uint32, - FlagDefault: uint32(64), - Required: false, - Usage: "establishes how many ledgers exist between checkpoints, do NOT change this unless you really know what you are doing", + Name: "checkpoint-frequency", + ConfigKey: &config.CheckpointFrequency, + OptType: types.Uint32, + FlagDefault: uint32(64), + Required: false, + Usage: "establishes how many ledgers exist between checkpoints, do NOT change this unless you really know what you are doing", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "behind-cloudflare", - ConfigKey: &config.BehindCloudflare, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "determines if Horizon instance is behind Cloudflare, in such case client IP in the logs will be replaced with Cloudflare header (cannot be used with --behind-aws-load-balancer)", + Name: "behind-cloudflare", + ConfigKey: &config.BehindCloudflare, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "determines if Horizon instance is behind Cloudflare, in such case client IP in the logs will be replaced with Cloudflare header (cannot be used with --behind-aws-load-balancer)", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "behind-aws-load-balancer", - ConfigKey: &config.BehindAWSLoadBalancer, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "determines if Horizon instance is behind AWS load balances like ELB or ALB, in such case client IP in the logs will be replaced with the last IP in X-Forwarded-For header (cannot be used with --behind-cloudflare)", + Name: "behind-aws-load-balancer", + ConfigKey: &config.BehindAWSLoadBalancer, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "determines if Horizon instance is behind AWS load balances like ELB or ALB, in such case client IP in the logs will be replaced with the last IP in X-Forwarded-For header (cannot be used with --behind-cloudflare)", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "rounding-slippage-filter", - ConfigKey: &config.RoundingSlippageFilter, - OptType: types.Int, - FlagDefault: 1000, - Required: false, - Usage: "excludes trades from /trade_aggregations unless their rounding slippage is