simplify manager init, fix tests, update docs

auditbeat/auditbeat.reference.yml

@@ -1346,11 +1346,12 @@ setup.template.settings:
 #setup.dsl.enabled: true
 
 # Set the lifecycle policy name. The default policy name is
-# 'beatname'.
+# 'auditbeat'.
 #setup.dsl.policy_name: "auditbeat"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
 # to load your own lifecycle policy.
+# If no custom policy is specified, a default policy with a lifetime of 7 days will be created.
 #setup.dsl.policy_file:
 
 # Disable the check for an existing lifecycle policy. The default is true. If

filebeat/filebeat.reference.yml

@@ -2442,11 +2442,12 @@ setup.template.settings:
 #setup.dsl.enabled: true
 
 # Set the lifecycle policy name. The default policy name is
-# 'beatname'.
+# 'filebeat'.
 #setup.dsl.policy_name: "filebeat"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
 # to load your own lifecycle policy.
+# If no custom policy is specified, a default policy with a lifetime of 7 days will be created.
 #setup.dsl.policy_file:
 
 # Disable the check for an existing lifecycle policy. The default is true. If

heartbeat/heartbeat.reference.yml

@@ -1438,11 +1438,12 @@ setup.template.settings:
 #setup.dsl.enabled: true
 
 # Set the lifecycle policy name. The default policy name is
-# 'beatname'.
+# 'heartbeat'.
 #setup.dsl.policy_name: "heartbeat"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
 # to load your own lifecycle policy.
+# If no custom policy is specified, a default policy with a lifetime of 7 days will be created.
 #setup.dsl.policy_file:
 
 # Disable the check for an existing lifecycle policy. The default is true. If

libbeat/_meta/config/setup.dsl.reference.yml.tmpl

@@ -7,11 +7,12 @@
 #setup.dsl.enabled: true
 
 # Set the lifecycle policy name. The default policy name is
-# 'beatname'.
+# '{{.BeatName}}'.
 #setup.dsl.policy_name: "{{.BeatName}}"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
 # to load your own lifecycle policy.
+# If no custom policy is specified, a default policy with a lifetime of 7 days will be created.
 #setup.dsl.policy_file:
 
 # Disable the check for an existing lifecycle policy. The default is true. If

libbeat/idxmgmt/idxmgmt.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
-	"github.com/elastic/beats/v7/libbeat/esleg/eslegclient"
 	"github.com/elastic/beats/v7/libbeat/idxmgmt/lifecycle"
 	"github.com/elastic/beats/v7/libbeat/outputs"
 	"github.com/elastic/beats/v7/libbeat/template"
@@ -113,54 +112,34 @@ func MakeDefaultSupport(ilmSupport lifecycle.SupportFactory) SupportFactory {
 			log = log.Named(logName)
 		}
 
-		// first fetch the ES output, check if we're running against serverless, use that to set a default config.
-		// the Supporter only cares about lifecycle config for checking if ILM/DSL is enabled or disabled.
-		outCfg := struct {
-			Output config.Namespace `config:"output"`
-		}{}
-		if configRoot != nil {
-			err := configRoot.Unpack(&outCfg)
-			if err != nil {
-				return nil, fmt.Errorf("error unpacking output config while making index support: %w", err)
-			}
-		}
-
-		defaultLifecycle := lifecycle.DefaultILMConfig(info)
-		if outCfg.Output.IsSet() && outCfg.Output.Name() == "elasticsearch" {
-			esClient, err := eslegclient.NewConnectedClient(outCfg.Output.Config(), info.Beat)
-			if err != nil {
-				// this is for the benefit of tests as well as various retry mechanisms;
-				// if we can't connect to ES, print a warning and fallback to ILM, tests won't care and non-test environments can
-				// retry the connection
-				logp.L().Warnf("could not connect to ES to determine ES type during index setup. Will default to ILM.")
-			} else {
-				if esClient.IsServerless() {
-					defaultLifecycle = lifecycle.DefaultDSLConfig(info)
-				}
-			}
-
-		}
-
 		// now that we have the "correct" default, unpack the rest of the config
 		cfg := struct {
-			ILM       lifecycle.LifecycleConfig `config:",inline"`
-			Template  *config.C                 `config:"setup.template"`
-			Output    config.Namespace          `config:"output"`
-			Migration *config.C                 `config:"migration.6_to_7"`
-		}{
-			ILM: defaultLifecycle,
-		}
+			Lifecycle lifecycle.RawConfig `config:",inline"`
+			Template  *config.C           `config:"setup.template"`
+			Output    config.Namespace    `config:"output"`
+			Migration *config.C           `config:"migration.6_to_7"`
+		}{}
 		if configRoot != nil {
 			if err := configRoot.Unpack(&cfg); err != nil {
 				return nil, fmt.Errorf("error unpacking cfg settings while setting up index support: %w", err)
 			}
 		}
 
+		// consider lifecycles enabled if the user has explicitly enabled them,
+		// or if no `enabled`` setting has been set by the user, thus reverting to a default of enabled.
+		enabled := false
+		if cfg.Lifecycle.DSL.Enabled() || cfg.Lifecycle.ILM.Enabled() {
+			enabled = true
+		}
+		if (cfg.Lifecycle.DSL == nil || !cfg.Lifecycle.DSL.HasField("enabled")) && (cfg.Lifecycle.ILM == nil || !cfg.Lifecycle.ILM.HasField("enabled")) {
+			enabled = true
+		}
+
 		if err := checkTemplateESSettings(cfg.Template, cfg.Output); err != nil {
 			return nil, err
 		}
 
-		return newIndexSupport(log, info, ilmSupport, cfg.Template, cfg.ILM, cfg.Migration.Enabled())
+		return newIndexSupport(log, info, ilmSupport, cfg.Template, enabled, cfg.Migration.Enabled())
 	}
 }
 

libbeat/idxmgmt/index_support.go

@@ -97,14 +97,14 @@ func newIndexSupport(
 	info beat.Info,
 	ilmFactory lifecycle.SupportFactory,
 	tmplConfig *config.C,
-	ilmConfig lifecycle.LifecycleConfig,
+	lifecyclesEnabled bool,
 	migration bool,
 ) (*indexSupport, error) {
 	if ilmFactory == nil {
 		ilmFactory = lifecycle.DefaultSupport
 	}
 
-	ilmSupporter, err := ilmFactory(log, info, ilmConfig)
+	ilmSupporter, err := ilmFactory(log, info, lifecyclesEnabled)
 	if err != nil {
 		return nil, fmt.Errorf("error creating lifecycle supporter: %w", err)
 	}

libbeat/idxmgmt/lifecycle/file_client_handler.go

@@ -44,12 +44,13 @@ func NewFileClientHandler(c FileClient, info beat.Info, cfg RawConfig) (*FileCli
 		return nil, fmt.Errorf("only one lifecycle management type can be used, but both ILM and DSL are enabled")
 	}
 
-	lifecycleCfg := Config{}
+	// default to ILM if no configs are set
+	lifecycleCfg := DefaultILMConfig(info).ILM
 	var err error
 	if cfg.DSL.Enabled() {
 		lifecycleCfg = DefaultDSLConfig(info).DSL
 		err = cfg.DSL.Unpack(&lifecycleCfg)
-	} else {
+	} else if cfg.ILM.Enabled() {
 		lifecycleCfg = DefaultILMConfig(info).ILM
 		err = cfg.ILM.Unpack(&lifecycleCfg)
 	}

libbeat/idxmgmt/lifecycle/ilm.go

@@ -27,7 +27,7 @@ import (
 )
 
 // SupportFactory is used to define a policy type to be used.
-type SupportFactory func(*logp.Logger, beat.Info, LifecycleConfig) (Supporter, error)
+type SupportFactory func(*logp.Logger, beat.Info, bool) (Supporter, error)
 
 // Supporter implements ILM support. For loading the policies
 // a manager instance must be generated.
@@ -65,28 +65,28 @@ type Policy struct {
 }
 
 // DefaultSupport configures a new default ILM support implementation.
-func DefaultSupport(log *logp.Logger, info beat.Info, cfg LifecycleConfig) (Supporter, error) {
-	if !cfg.DSL.Enabled && !cfg.ILM.Enabled {
-		return NewNoopSupport(info, cfg)
+func DefaultSupport(log *logp.Logger, info beat.Info, lifecycleEnabled bool) (Supporter, error) {
+	if !lifecycleEnabled {
+		return NewNoopSupport(info, lifecycleEnabled)
 	}
 
-	return StdSupport(log, info, cfg)
+	return StdSupport(log, info, lifecycleEnabled)
 }
 
 // StdSupport configures a new std ILM support implementation.
-func StdSupport(log *logp.Logger, info beat.Info, cfg LifecycleConfig) (Supporter, error) {
+func StdSupport(log *logp.Logger, info beat.Info, lifecycleEnabled bool) (Supporter, error) {
 	if log == nil {
 		log = logp.NewLogger("ilm")
 	} else {
 		log = log.Named("ilm")
 	}
 
-	return NewStdSupport(log, cfg), nil
+	return NewStdSupport(log, lifecycleEnabled), nil
 }
 
 // NoopSupport configures a new noop ILM support implementation,
 // should be used when ILM is disabled
-func NoopSupport(_ *logp.Logger, info beat.Info, c LifecycleConfig) (Supporter, error) {
+func NoopSupport(_ *logp.Logger, info beat.Info, c bool) (Supporter, error) {
 	return NewNoopSupport(info, c)
 }