From a7ef95c18c66d25ea920ccd8efce7b82b9058f5f Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 5 Oct 2023 21:00:35 -0700 Subject: [PATCH 1/4] #5022: limit the global hel[p flags display to only when relevant for the command --- services/horizon/cmd/db.go | 26 +- services/horizon/cmd/ingest.go | 118 ++--- services/horizon/cmd/record_metrics.go | 4 +- services/horizon/cmd/root.go | 58 ++- services/horizon/cmd/serve.go | 2 +- services/horizon/internal/flags.go | 437 ++++++++++-------- .../internal/integration/parameters_test.go | 112 +++-- support/config/config_option.go | 9 +- 8 files changed, 479 insertions(+), 287 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 37a166533f..5a92124095 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -38,7 +38,7 @@ func requireAndSetFlags(names ...string) error { for _, name := range names { set[name] = true } - for _, flag := range flags { + for _, flag := range globalFlags { if set[flag.Name] { flag.Require() if err := flag.SetValue(); err != nil { @@ -66,7 +66,7 @@ var dbInitCmd = &cobra.Command{ return err } - db, err := sql.Open("postgres", config.DatabaseURL) + db, err := sql.Open("postgres", globalConfig.DatabaseURL) if err != nil { return err } @@ -86,12 +86,12 @@ var dbInitCmd = &cobra.Command{ } func migrate(dir schema.MigrateDir, count int) error { - if !config.Ingest { + if !globalConfig.Ingest { log.Println("Skipping migrations because ingest flag is not enabled") return nil } - dbConn, err := db.Open("postgres", config.DatabaseURL) + dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return err } @@ -172,7 +172,7 @@ var dbMigrateStatusCmd = &cobra.Command{ return ErrUsage{cmd} } - dbConn, err := db.Open("postgres", config.DatabaseURL) + dbConn, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return err } @@ -220,7 +220,7 @@ var dbReapCmd = &cobra.Command{ Short: "reaps (i.e. removes) any reapable history data", Long: "reap removes any historical data that is earlier than the configured retention cutoff", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(config, flags) + app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) if err != nil { return err } @@ -321,7 +321,7 @@ var dbReingestRangeCmd = &cobra.Command{ } } - err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}) + err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}) if err != nil { return err } @@ -329,7 +329,7 @@ var dbReingestRangeCmd = &cobra.Command{ []history.LedgerRange{{StartSequence: argsUInt32[0], EndSequence: argsUInt32[1]}}, reingestForce, parallelWorkers, - *config, + *globalConfig, ) }, } @@ -369,26 +369,26 @@ var dbFillGapsCmd = &cobra.Command{ withRange = true } - err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}) + err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}) if err != nil { return err } var gaps []history.LedgerRange if withRange { - gaps, err = runDBDetectGapsInRange(*config, uint32(start), uint32(end)) + gaps, err = runDBDetectGapsInRange(*globalConfig, uint32(start), uint32(end)) if err != nil { return err } hlog.Infof("found gaps %v within range [%v, %v]", gaps, start, end) } else { - gaps, err = runDBDetectGaps(*config) + gaps, err = runDBDetectGaps(*globalConfig) if err != nil { return err } hlog.Infof("found gaps %v", gaps) } - return runDBReingestRange(gaps, reingestForce, parallelWorkers, *config) + return runDBReingestRange(gaps, reingestForce, parallelWorkers, *globalConfig) }, } @@ -479,7 +479,7 @@ var dbDetectGapsCmd = &cobra.Command{ if len(args) != 0 { return ErrUsage{cmd} } - gaps, err := runDBDetectGaps(*config) + gaps, err := runDBDetectGaps(*globalConfig) if err != nil { return err } diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index db9eccea30..eed1f51f2f 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -94,7 +94,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { return err } @@ -111,11 +111,11 @@ var ingestVerifyRangeCmd = &cobra.Command{ }() } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } - mngr := historyarchive.NewCheckpointManager(config.CheckpointFrequency) + mngr := historyarchive.NewCheckpointManager(globalConfig.CheckpointFrequency) if !mngr.IsCheckpoint(ingestVerifyFrom) && ingestVerifyFrom != 1 { return fmt.Errorf("`--from` must be a checkpoint ledger") } @@ -125,26 +125,26 @@ var ingestVerifyRangeCmd = &cobra.Command{ } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath, - CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB, - RemoteCaptiveCoreURL: config.RemoteCaptiveCoreURL, - CheckpointFrequency: config.CheckpointFrequency, - CaptiveCoreToml: config.CaptiveCoreToml, - CaptiveCoreStoragePath: config.CaptiveCoreStoragePath, - RoundingSlippageFilter: config.RoundingSlippageFilter, - EnableIngestionFiltering: config.EnableIngestionFiltering, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, + CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, + RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, + CheckpointFrequency: globalConfig.CheckpointFrequency, + CaptiveCoreToml: globalConfig.CaptiveCoreToml, + CaptiveCoreStoragePath: globalConfig.CaptiveCoreStoragePath, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } if !ingestConfig.EnableCaptiveCore { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } @@ -203,11 +203,11 @@ var ingestStressTestCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -221,23 +221,23 @@ var ingestStressTestCmd = &cobra.Command{ } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - RoundingSlippageFilter: config.RoundingSlippageFilter, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, } - if config.EnableCaptiveCoreIngestion { - ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath - ingestConfig.RemoteCaptiveCoreURL = config.RemoteCaptiveCoreURL - ingestConfig.CaptiveCoreConfigUseDB = config.CaptiveCoreConfigUseDB + if globalConfig.EnableCaptiveCoreIngestion { + ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath + ingestConfig.RemoteCaptiveCoreURL = globalConfig.RemoteCaptiveCoreURL + ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB } else { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } @@ -267,11 +267,11 @@ var ingestTriggerStateRebuildCmd = &cobra.Command{ Short: "updates a database to trigger state rebuild, state will be rebuilt by a running Horizon instance, DO NOT RUN production DB, some endpoints will be unavailable until state is rebuilt", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -291,11 +291,11 @@ var ingestInitGenesisStateCmd = &cobra.Command{ Short: "ingests genesis state (ledger 1)", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -312,24 +312,24 @@ var ingestInitGenesisStateCmd = &cobra.Command{ } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - CheckpointFrequency: config.CheckpointFrequency, - RoundingSlippageFilter: config.RoundingSlippageFilter, - EnableIngestionFiltering: config.EnableIngestionFiltering, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + CheckpointFrequency: globalConfig.CheckpointFrequency, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } - if config.EnableCaptiveCoreIngestion { - ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath - ingestConfig.CaptiveCoreConfigUseDB = config.CaptiveCoreConfigUseDB + if globalConfig.EnableCaptiveCoreIngestion { + ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath + ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB } else { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } @@ -363,11 +363,11 @@ var ingestBuildStateCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { return err } - horizonSession, err := db.Open("postgres", config.DatabaseURL) + horizonSession, err := db.Open("postgres", globalConfig.DatabaseURL) if err != nil { return fmt.Errorf("cannot open Horizon DB: %v", err) } @@ -383,33 +383,33 @@ var ingestBuildStateCmd = &cobra.Command{ return fmt.Errorf("cannot run on non-empty DB") } - mngr := historyarchive.NewCheckpointManager(config.CheckpointFrequency) + mngr := historyarchive.NewCheckpointManager(globalConfig.CheckpointFrequency) if !mngr.IsCheckpoint(ingestBuildStateSequence) && ingestBuildStateSequence != 1 { return fmt.Errorf("`--sequence` must be a checkpoint ledger") } ingestConfig := ingest.Config{ - NetworkPassphrase: config.NetworkPassphrase, + NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, - HistoryArchiveURLs: config.HistoryArchiveURLs, - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, - CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath, - CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB, - RemoteCaptiveCoreURL: config.RemoteCaptiveCoreURL, - CheckpointFrequency: config.CheckpointFrequency, - CaptiveCoreToml: config.CaptiveCoreToml, - CaptiveCoreStoragePath: config.CaptiveCoreStoragePath, - RoundingSlippageFilter: config.RoundingSlippageFilter, - EnableIngestionFiltering: config.EnableIngestionFiltering, + HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, + EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, + CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, + CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, + RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, + CheckpointFrequency: globalConfig.CheckpointFrequency, + CaptiveCoreToml: globalConfig.CaptiveCoreToml, + CaptiveCoreStoragePath: globalConfig.CaptiveCoreStoragePath, + RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } if !ingestBuildStateSkipChecks { if !ingestConfig.EnableCaptiveCore { - if config.StellarCoreDatabaseURL == "" { + if globalConfig.StellarCoreDatabaseURL == "" { return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) } - coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) } diff --git a/services/horizon/cmd/record_metrics.go b/services/horizon/cmd/record_metrics.go index cd4f335da1..fb6244b76b 100644 --- a/services/horizon/cmd/record_metrics.go +++ b/services/horizon/cmd/record_metrics.go @@ -19,7 +19,7 @@ var recordMetricsCmd = &cobra.Command{ Short: "records `/metrics` on admin port for debuging purposes", Long: "", RunE: func(cmd *cobra.Command, args []string) error { - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{}); err != nil { + if err := horizon.ApplyFlags(globalConfig, globalFlags, horizon.ApplyOptions{}); err != nil { return err } @@ -50,7 +50,7 @@ var recordMetricsCmd = &cobra.Command{ time.Duration(time.Duration(scrapeIntervalSeconds*(scrapesCount-i))*time.Second), ) - metricsResponse, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", config.AdminPort)) + metricsResponse, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", globalConfig.AdminPort)) if err != nil { return errors.Wrap(err, "Error fetching metrics. Is admin server running?") } diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index af63675d0c..37a60cfc46 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -6,10 +6,11 @@ import ( "github.com/spf13/cobra" horizon "github.com/stellar/go/services/horizon/internal" + "github.com/stellar/go/support/config" ) var ( - config, flags = horizon.Flags() + globalConfig, globalFlags = horizon.Flags() RootCmd = &cobra.Command{ Use: "horizon", @@ -18,13 +19,14 @@ var ( SilenceUsage: true, Long: "Client-facing API server for the Stellar network. It acts as the interface between Stellar Core and applications that want to access the Stellar network. It allows you to submit transactions to the network, check the status of accounts, subscribe to event streams and more.", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(config, flags) + app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) if err != nil { return err } return app.Serve() }, } + originalHelpFunc = RootCmd.HelpFunc() ) // ErrUsage indicates we should print the usage string and exit with code 1 @@ -44,7 +46,15 @@ func (e ErrExitCode) Error() string { } func init() { - err := flags.Init(RootCmd) + + // override the default help output, apply further filtering on which global flags + // will be shown on the help outout dependent on the command help was issued upon. + RootCmd.SetHelpFunc(func(c *cobra.Command, args []string) { + enableGlobalOptionsInHelp(c, globalFlags) + originalHelpFunc(c, args) + }) + + err := globalFlags.Init(RootCmd) if err != nil { stdLog.Fatal(err.Error()) } @@ -53,3 +63,45 @@ func init() { func Execute() error { return RootCmd.Execute() } + +func enableGlobalOptionsInHelp(cmd *cobra.Command, cos config.ConfigOptions) { + for _, co := range cos { + if co.Hidden { + // this options was configured statically to be hidden + // Init() has already set that once, leave it as-is. + continue + } + + // we don't want' to display global flags in help output + // if no sub-command context given yet, i.e. just '-h' was used + // or there are subcomands required to use. + if cmd.Parent() == nil || cmd.HasAvailableSubCommands() { + co.ToggleHidden(true) + continue + } + + if len(co.UsedInCommands) > 0 && + !contains(co.UsedInCommands, cmd) { + co.ToggleHidden(true) + } else { + co.ToggleHidden(false) + } + } +} + +// check if this command or any of it's sub-level parents match +// supportedCommands +func contains(supportedCommands []string, cmd *cobra.Command) bool { + for _, supportedCommand := range supportedCommands { + if supportedCommand == cmd.Name() { + return true + } + } + + // don't do inheritance matching on the top most sub-commands. + // they are second level deep, the horizon itself is top level. + if cmd.Parent() != nil && cmd.Parent().Parent() != nil { + return contains(supportedCommands, cmd.Parent()) + } + return false +} diff --git a/services/horizon/cmd/serve.go b/services/horizon/cmd/serve.go index 8e06855d3c..21a34da3ad 100644 --- a/services/horizon/cmd/serve.go +++ b/services/horizon/cmd/serve.go @@ -10,7 +10,7 @@ var serveCmd = &cobra.Command{ Short: "run horizon server", Long: "serve initializes then starts the horizon HTTP server", RunE: func(cmd *cobra.Command, args []string) error { - app, err := horizon.NewAppFromFlags(config, flags) + app, err := horizon.NewAppFromFlags(globalConfig, globalFlags) if err != nil { return err } diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index d2cc055ef8..6c3d339b51 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -67,6 +67,32 @@ const ( defaultMaxHTTPRequestSize = uint(200 * 1024) ) +var ( + IngestCmd = "ingest" + RecordMetricsCmd = "record-metrics" + DbCmd = "db" + ServeCmd = "serve" + HorizonCmd = "horizon" + + DbFillGapsCmd = "fill-gaps" + DbReingestCmd = "reingest" + IngestTriggerStateRebuild = "trigger-state-rebuild" + IngestInitGenesisStateCmd = "init-genesis-state" + IngestBuildStateCmd = "build-state" + IngestStressTestCmd = "stress-test" + IngestVerifyRangeCmd = "verify-range" + + ApiServerCommands = []string{HorizonCmd, ServeCmd} + IngestionCommands = append(ApiServerCommands, + IngestInitGenesisStateCmd, + IngestBuildStateCmd, + IngestStressTestCmd, + IngestVerifyRangeCmd, + DbFillGapsCmd, + DbReingestCmd) + DatabaseBoundCommands = append(ApiServerCommands, DbCmd, IngestCmd) +) + // validateBothOrNeither ensures that both options are provided, if either is provided. func validateBothOrNeither(option1, option2 string) error { arg1, arg2 := viper.GetString(option1), viper.GetString(option2) @@ -130,36 +156,40 @@ func Flags() (*Config, support.ConfigOptions) { // Add a new entry here to connect a new field in the horizon.Config struct var flags = support.ConfigOptions{ &support.ConfigOption{ - Name: DatabaseURLFlagName, - EnvVar: "DATABASE_URL", - ConfigKey: &config.DatabaseURL, - OptType: types.String, - Required: true, - Usage: "horizon postgres database to connect with", + Name: DatabaseURLFlagName, + EnvVar: "DATABASE_URL", + ConfigKey: &config.DatabaseURL, + OptType: types.String, + Required: true, + Usage: "horizon postgres database to connect with", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "ro-database-url", - ConfigKey: &config.RoDatabaseURL, - OptType: types.String, - Required: false, - Usage: "horizon postgres read-replica to connect with, when set it will return stale history error when replica is behind primary", + Name: "ro-database-url", + ConfigKey: &config.RoDatabaseURL, + OptType: types.String, + Required: false, + Usage: "horizon postgres read-replica to connect with, when set it will return stale history error when replica is behind primary", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: StellarCoreBinaryPathName, - OptType: types.String, - FlagDefault: "", - Required: false, - Usage: "path to stellar core binary, look for the stellar-core binary in $PATH by default.", - ConfigKey: &config.CaptiveCoreBinaryPath, + Name: StellarCoreBinaryPathName, + OptType: types.String, + FlagDefault: "", + Required: false, + Usage: "path to stellar core binary, look for the stellar-core binary in $PATH by default.", + ConfigKey: &config.CaptiveCoreBinaryPath, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: DisableTxSubFlagName, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "disables the transaction submission functionality of Horizon.", - ConfigKey: &config.DisableTxSub, - Hidden: false, + Name: DisableTxSubFlagName, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "disables the transaction submission functionality of Horizon.", + ConfigKey: &config.DisableTxSub, + Hidden: false, + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: captiveCoreConfigAppendPathName, @@ -183,6 +213,7 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: CaptiveCoreConfigPathName, @@ -197,6 +228,7 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: CaptiveCoreConfigUseDB, @@ -215,15 +247,17 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, - ConfigKey: &config.CaptiveCoreConfigUseDB, + ConfigKey: &config.CaptiveCoreConfigUseDB, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "enable-captive-core-ingestion", - OptType: types.Bool, - FlagDefault: true, - Required: false, - Usage: "causes Horizon to ingest from a Captive Stellar Core process instead of a persistent Stellar Core database", - ConfigKey: &config.EnableCaptiveCoreIngestion, + Name: "enable-captive-core-ingestion", + OptType: types.Bool, + FlagDefault: true, + Required: false, + Usage: "causes Horizon to ingest from a Captive Stellar Core process instead of a persistent Stellar Core database", + ConfigKey: &config.EnableCaptiveCoreIngestion, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: EnableIngestionFilteringFlagName, @@ -257,6 +291,7 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: uint(0), Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", ConfigKey: &config.CaptiveCoreTomlParams.HTTPPort, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "captive-core-storage-path", @@ -273,9 +308,10 @@ func Flags() (*Config, support.ConfigOptions) { *opt.ConfigKey.(*string) = existingValue return nil }, - Required: false, - Usage: "Storage location for Captive Core bucket data. If not set, the current working directory is used as the default location.", - ConfigKey: &config.CaptiveCoreStoragePath, + Required: false, + Usage: "Storage location for Captive Core bucket data. If not set, the current working directory is used as the default location.", + ConfigKey: &config.CaptiveCoreStoragePath, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "captive-core-peer-port", @@ -285,20 +321,23 @@ func Flags() (*Config, support.ConfigOptions) { Required: false, Usage: "port for Captive Core to bind to for connecting to the Stellar swarm (0 uses Stellar Core's default)", ConfigKey: &config.CaptiveCoreTomlParams.PeerPort, + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: StellarCoreDBURLFlagName, - EnvVar: "STELLAR_CORE_DATABASE_URL", - ConfigKey: &config.StellarCoreDatabaseURL, - OptType: types.String, - Required: false, - Usage: "stellar-core postgres database to connect with", + Name: StellarCoreDBURLFlagName, + EnvVar: "STELLAR_CORE_DATABASE_URL", + ConfigKey: &config.StellarCoreDatabaseURL, + OptType: types.String, + Required: false, + Usage: "stellar-core postgres database to connect with", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: StellarCoreURLFlagName, - ConfigKey: &config.StellarCoreURL, - OptType: types.String, - Usage: "stellar-core to connect with (for http commands). If unset and the local Captive core is enabled, it will use http://localhost:", + Name: StellarCoreURLFlagName, + ConfigKey: &config.StellarCoreURL, + OptType: types.String, + Usage: "stellar-core to connect with (for http commands). If unset and the local Captive core is enabled, it will use http://localhost:", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: HistoryArchiveURLsFlagName, @@ -316,42 +355,48 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, - Usage: "comma-separated list of stellar history archives to connect with", + Usage: "comma-separated list of stellar history archives to connect with", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "port", - ConfigKey: &config.Port, - OptType: types.Uint, - FlagDefault: uint(8000), - Usage: "tcp port to listen on for http requests", + Name: "port", + ConfigKey: &config.Port, + OptType: types.Uint, + FlagDefault: uint(8000), + Usage: "tcp port to listen on for http requests", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "admin-port", - ConfigKey: &config.AdminPort, - OptType: types.Uint, - FlagDefault: uint(0), - Usage: "WARNING: this should not be accessible from the Internet and does not use TLS, tcp port to listen on for admin http requests, 0 (default) disables the admin server", + Name: "admin-port", + ConfigKey: &config.AdminPort, + OptType: types.Uint, + FlagDefault: uint(0), + Usage: "WARNING: this should not be accessible from the Internet and does not use TLS, tcp port to listen on for admin http requests, 0 (default) disables the admin server", + UsedInCommands: append(ApiServerCommands, RecordMetricsCmd), }, &support.ConfigOption{ - Name: "max-db-connections", - ConfigKey: &config.MaxDBConnections, - OptType: types.Int, - FlagDefault: 0, - Usage: "when set has a priority over horizon-db-max-open-connections, horizon-db-max-idle-connections. max horizon database open connections may need to be increased when responses are slow but DB CPU is normal", + Name: "max-db-connections", + ConfigKey: &config.MaxDBConnections, + OptType: types.Int, + FlagDefault: 0, + Usage: "when set has a priority over horizon-db-max-open-connections, horizon-db-max-idle-connections. max horizon database open connections may need to be increased when responses are slow but DB CPU is normal", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "horizon-db-max-open-connections", - ConfigKey: &config.HorizonDBMaxOpenConnections, - OptType: types.Int, - FlagDefault: 20, - Usage: "max horizon database open connections. may need to be increased when responses are slow but DB CPU is normal", + Name: "horizon-db-max-open-connections", + ConfigKey: &config.HorizonDBMaxOpenConnections, + OptType: types.Int, + FlagDefault: 20, + Usage: "max horizon database open connections. may need to be increased when responses are slow but DB CPU is normal", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "horizon-db-max-idle-connections", - ConfigKey: &config.HorizonDBMaxIdleConnections, - OptType: types.Int, - FlagDefault: 20, - Usage: "max horizon database idle connections. may need to be set to the same value as horizon-db-max-open-connections when responses are slow and DB CPU is normal, because it may indicate that a lot of time is spent closing/opening idle connections. This can happen in case of high variance in number of requests. must be equal or lower than max open connections", + Name: "horizon-db-max-idle-connections", + ConfigKey: &config.HorizonDBMaxIdleConnections, + OptType: types.Int, + FlagDefault: 20, + Usage: "max horizon database idle connections. may need to be set to the same value as horizon-db-max-open-connections when responses are slow and DB CPU is normal, because it may indicate that a lot of time is spent closing/opening idle connections. This can happen in case of high variance in number of requests. must be equal or lower than max open connections", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ Name: "sse-update-frequency", @@ -360,6 +405,7 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: 5, CustomSetValue: support.SetDuration, Usage: "defines how often streams should check if there's a new ledger (in seconds), may need to increase in case of big number of streams", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "connection-timeout", @@ -368,13 +414,15 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: 55, CustomSetValue: support.SetDuration, Usage: "defines the timeout of connection after which 504 response will be sent or stream will be closed, if Horizon is behind a load balancer with idle connection timeout, this should be set to a few seconds less that idle timeout, does not apply to POST /transactions", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "max-http-request-size", - ConfigKey: &config.MaxHTTPRequestSize, - OptType: types.Uint, - FlagDefault: defaultMaxHTTPRequestSize, - Usage: "sets the limit on the maximum allowed http request payload size, default is 200kb, to disable the limit check, set to 0, only do so if you acknowledge the implications of accepting unbounded http request payload sizes.", + Name: "max-http-request-size", + ConfigKey: &config.MaxHTTPRequestSize, + OptType: types.Uint, + FlagDefault: defaultMaxHTTPRequestSize, + Usage: "sets the limit on the maximum allowed http request payload size, default is 200kb, to disable the limit check, set to 0, only do so if you acknowledge the implications of accepting unbounded http request payload sizes.", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "per-hour-rate-limit", @@ -393,7 +441,8 @@ func Flags() (*Config, support.ConfigOptions) { } return nil }, - Usage: "max count of requests allowed in a one hour period, by remote ip address", + Usage: "max count of requests allowed in a one hour period, by remote ip address", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "friendbot-url", @@ -401,6 +450,7 @@ func Flags() (*Config, support.ConfigOptions) { OptType: types.String, CustomSetValue: support.SetURL, Usage: "friendbot service to redirect to", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "log-level", @@ -430,36 +480,41 @@ func Flags() (*Config, support.ConfigOptions) { CustomSetValue: support.SetOptionalString, Required: false, Usage: "name of the path for Core logs (leave empty to log w/ Horizon only)", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "max-path-length", - ConfigKey: &config.MaxPathLength, - OptType: types.Uint, - FlagDefault: uint(3), - Usage: "the maximum number of assets on the path in `/paths` endpoint, warning: increasing this value will increase /paths response time", + Name: "max-path-length", + ConfigKey: &config.MaxPathLength, + OptType: types.Uint, + FlagDefault: uint(3), + Usage: "the maximum number of assets on the path in `/paths` endpoint, warning: increasing this value will increase /paths response time", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "max-assets-per-path-request", - ConfigKey: &config.MaxAssetsPerPathRequest, - OptType: types.Int, - FlagDefault: int(15), - Usage: "the maximum number of assets in '/paths/strict-send' and '/paths/strict-receive' endpoints", + Name: "max-assets-per-path-request", + ConfigKey: &config.MaxAssetsPerPathRequest, + OptType: types.Int, + FlagDefault: int(15), + Usage: "the maximum number of assets in '/paths/strict-send' and '/paths/strict-receive' endpoints", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "disable-pool-path-finding", - ConfigKey: &config.DisablePoolPathFinding, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "excludes liquidity pools from consideration in the `/paths` endpoint", + Name: "disable-pool-path-finding", + ConfigKey: &config.DisablePoolPathFinding, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "excludes liquidity pools from consideration in the `/paths` endpoint", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "disable-path-finding", - ConfigKey: &config.DisablePathFinding, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "disables the path finding endpoints", + Name: "disable-path-finding", + ConfigKey: &config.DisablePathFinding, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "disables the path finding endpoints", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "max-path-finding-requests", @@ -469,19 +524,22 @@ func Flags() (*Config, support.ConfigOptions) { Required: false, Usage: "The maximum number of path finding requests per second horizon will allow." + " A value of zero (the default) disables the limit.", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: NetworkPassphraseFlagName, - ConfigKey: &config.NetworkPassphrase, - OptType: types.String, - Required: false, - Usage: "Override the network passphrase", + Name: NetworkPassphraseFlagName, + ConfigKey: &config.NetworkPassphrase, + OptType: types.String, + Required: false, + Usage: "Override the network passphrase", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "sentry-dsn", - ConfigKey: &config.SentryDSN, - OptType: types.String, - Usage: "Sentry URL to which panics and errors should be reported", + Name: "sentry-dsn", + ConfigKey: &config.SentryDSN, + OptType: types.String, + Usage: "Sentry URL to which panics and errors should be reported", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ Name: "loggly-token", @@ -497,59 +555,67 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "Tag to be added to every loggly log event", }, &support.ConfigOption{ - Name: "tls-cert", - ConfigKey: &config.TLSCert, - OptType: types.String, - Usage: "TLS certificate file to use for securing connections to horizon", + Name: "tls-cert", + ConfigKey: &config.TLSCert, + OptType: types.String, + Usage: "TLS certificate file to use for securing connections to horizon", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "tls-key", - ConfigKey: &config.TLSKey, - OptType: types.String, - Usage: "TLS private key file to use for securing connections to horizon", + Name: "tls-key", + ConfigKey: &config.TLSKey, + OptType: types.String, + Usage: "TLS private key file to use for securing connections to horizon", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: IngestFlagName, - ConfigKey: &config.Ingest, - OptType: types.Bool, - FlagDefault: true, - Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", + Name: IngestFlagName, + ConfigKey: &config.Ingest, + OptType: types.Bool, + FlagDefault: true, + Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "cursor-name", - EnvVar: "CURSOR_NAME", - ConfigKey: &config.CursorName, - OptType: types.String, - FlagDefault: "HORIZON", - Usage: "ingestor cursor used by horizon to ingest from stellar core. must be uppercase and unique for each horizon instance ingesting from that core instance.", + Name: "cursor-name", + EnvVar: "CURSOR_NAME", + ConfigKey: &config.CursorName, + OptType: types.String, + FlagDefault: "HORIZON", + Usage: "ingestor cursor used by horizon to ingest from stellar core. must be uppercase and unique for each horizon instance ingesting from that core instance.", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "history-retention-count", - ConfigKey: &config.HistoryRetentionCount, - OptType: types.Uint, - FlagDefault: uint(0), - Usage: "the minimum number of ledgers to maintain within horizon's history tables. 0 signifies an unlimited number of ledgers will be retained", + Name: "history-retention-count", + ConfigKey: &config.HistoryRetentionCount, + OptType: types.Uint, + FlagDefault: uint(0), + Usage: "the minimum number of ledgers to maintain within horizon's history tables. 0 signifies an unlimited number of ledgers will be retained", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "history-stale-threshold", - ConfigKey: &config.StaleThreshold, - OptType: types.Uint, - FlagDefault: uint(0), - Usage: "the maximum number of ledgers the history db is allowed to be out of date from the connected stellar-core db before horizon considers history stale", + Name: "history-stale-threshold", + ConfigKey: &config.StaleThreshold, + OptType: types.Uint, + FlagDefault: uint(0), + Usage: "the maximum number of ledgers the history db is allowed to be out of date from the connected stellar-core db before horizon considers history stale", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "skip-cursor-update", - ConfigKey: &config.SkipCursorUpdate, - OptType: types.Bool, - FlagDefault: false, - Usage: "causes the ingester to skip reporting the last imported ledger state to stellar-core", + Name: "skip-cursor-update", + ConfigKey: &config.SkipCursorUpdate, + OptType: types.Bool, + FlagDefault: false, + Usage: "causes the ingester to skip reporting the last imported ledger state to stellar-core", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "ingest-disable-state-verification", - ConfigKey: &config.IngestDisableStateVerification, - OptType: types.Bool, - FlagDefault: false, - Usage: "ingestion system runs a verification routing to compare state in local database with history buckets, this can be disabled however it's not recommended", + Name: "ingest-disable-state-verification", + ConfigKey: &config.IngestDisableStateVerification, + OptType: types.Bool, + FlagDefault: false, + Usage: "ingestion system runs a verification routing to compare state in local database with history buckets, this can be disabled however it's not recommended", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "ingest-state-verification-checkpoint-frequency", @@ -559,6 +625,7 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "the frequency in units per checkpoint for how often state verification is executed. " + "A value of 1 implies running state verification on every checkpoint. " + "A value of 2 implies running state verification on every second checkpoint.", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ Name: "ingest-state-verification-timeout", @@ -568,53 +635,60 @@ func Flags() (*Config, support.ConfigOptions) { CustomSetValue: support.SetDurationMinutes, Usage: "defines an upper bound in minutes for on how long state verification is allowed to run. " + "A value of 0 disables the timeout.", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "ingest-enable-extended-log-ledger-stats", - ConfigKey: &config.IngestEnableExtendedLogLedgerStats, - OptType: types.Bool, - FlagDefault: false, - Usage: "enables extended ledger stats in the log (ledger entry changes and operations stats)", + Name: "ingest-enable-extended-log-ledger-stats", + ConfigKey: &config.IngestEnableExtendedLogLedgerStats, + OptType: types.Bool, + FlagDefault: false, + Usage: "enables extended ledger stats in the log (ledger entry changes and operations stats)", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "apply-migrations", - ConfigKey: &config.ApplyMigrations, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "applies pending migrations before starting horizon", + Name: "apply-migrations", + ConfigKey: &config.ApplyMigrations, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "applies pending migrations before starting horizon", + UsedInCommands: DatabaseBoundCommands, }, &support.ConfigOption{ - Name: "checkpoint-frequency", - ConfigKey: &config.CheckpointFrequency, - OptType: types.Uint32, - FlagDefault: uint32(64), - Required: false, - Usage: "establishes how many ledgers exist between checkpoints, do NOT change this unless you really know what you are doing", + Name: "checkpoint-frequency", + ConfigKey: &config.CheckpointFrequency, + OptType: types.Uint32, + FlagDefault: uint32(64), + Required: false, + Usage: "establishes how many ledgers exist between checkpoints, do NOT change this unless you really know what you are doing", + UsedInCommands: IngestionCommands, }, &support.ConfigOption{ - Name: "behind-cloudflare", - ConfigKey: &config.BehindCloudflare, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "determines if Horizon instance is behind Cloudflare, in such case client IP in the logs will be replaced with Cloudflare header (cannot be used with --behind-aws-load-balancer)", + Name: "behind-cloudflare", + ConfigKey: &config.BehindCloudflare, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "determines if Horizon instance is behind Cloudflare, in such case client IP in the logs will be replaced with Cloudflare header (cannot be used with --behind-aws-load-balancer)", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "behind-aws-load-balancer", - ConfigKey: &config.BehindAWSLoadBalancer, - OptType: types.Bool, - FlagDefault: false, - Required: false, - Usage: "determines if Horizon instance is behind AWS load balances like ELB or ALB, in such case client IP in the logs will be replaced with the last IP in X-Forwarded-For header (cannot be used with --behind-cloudflare)", + Name: "behind-aws-load-balancer", + ConfigKey: &config.BehindAWSLoadBalancer, + OptType: types.Bool, + FlagDefault: false, + Required: false, + Usage: "determines if Horizon instance is behind AWS load balances like ELB or ALB, in such case client IP in the logs will be replaced with the last IP in X-Forwarded-For header (cannot be used with --behind-cloudflare)", + UsedInCommands: ApiServerCommands, }, &support.ConfigOption{ - Name: "rounding-slippage-filter", - ConfigKey: &config.RoundingSlippageFilter, - OptType: types.Int, - FlagDefault: 1000, - Required: false, - Usage: "excludes trades from /trade_aggregations unless their rounding slippage is