From 42e32d5c5acde22601cccb38ae8b1adde58917ec Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Thu, 21 Sep 2023 14:36:49 -0700 Subject: [PATCH] fix setup, config --- auditbeat/auditbeat.reference.yml | 23 ++++++++ filebeat/filebeat.reference.yml | 23 ++++++++ heartbeat/heartbeat.reference.yml | 23 ++++++++ .../_meta/config/default.reference.yml.tmpl | 1 + .../_meta/config/setup.dsl.reference.yml.tmpl | 23 ++++++++ libbeat/cmd/export/ilm_policy.go | 3 + libbeat/cmd/instance/beat.go | 2 + libbeat/idxmgmt/idxmgmt.go | 6 +- libbeat/idxmgmt/index_support.go | 57 +++++++++++-------- libbeat/idxmgmt/lifecycle/client_handler.go | 2 + .../client_handler_integration_test.go | 5 +- libbeat/idxmgmt/lifecycle/config.go | 10 ++-- libbeat/idxmgmt/lifecycle/error.go | 13 ----- .../idxmgmt/lifecycle/es_client_handler.go | 55 ++++++++++++------ libbeat/idxmgmt/lifecycle/es_client_test.go | 34 +++++++++-- .../idxmgmt/lifecycle/file_client_handler.go | 46 +++++++++++---- libbeat/idxmgmt/lifecycle/ilm.go | 8 ++- libbeat/idxmgmt/lifecycle/ilm_test.go | 46 ++++++++++----- .../idxmgmt/lifecycle/mock_client_handler.go | 9 +++ libbeat/idxmgmt/lifecycle/noop_manager.go | 22 +++++-- libbeat/idxmgmt/lifecycle/standard_manager.go | 30 +++++----- libbeat/idxmgmt/mock_manager_test.go | 14 +++-- libbeat/idxmgmt/std_test.go | 17 ++++-- libbeat/template/config.go | 5 +- libbeat/template/template.go | 3 + metricbeat/metricbeat.reference.yml | 23 ++++++++ packetbeat/packetbeat.reference.yml | 23 ++++++++ winlogbeat/winlogbeat.reference.yml | 23 ++++++++ x-pack/auditbeat/auditbeat.reference.yml | 23 ++++++++ x-pack/filebeat/filebeat.reference.yml | 23 ++++++++ .../functionbeat/functionbeat.reference.yml | 23 ++++++++ x-pack/heartbeat/heartbeat.reference.yml | 23 ++++++++ x-pack/metricbeat/metricbeat.reference.yml | 23 ++++++++ x-pack/osquerybeat/osquerybeat.reference.yml | 23 ++++++++ x-pack/packetbeat/packetbeat.reference.yml | 23 ++++++++ x-pack/winlogbeat/winlogbeat.reference.yml | 23 ++++++++ 36 files changed, 611 insertions(+), 122 deletions(-) create mode 100644 libbeat/_meta/config/setup.dsl.reference.yml.tmpl diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 892d4e1b2d09..09b66cd8cee1 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1337,6 +1337,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 2de0bc61f565..02fa95fca648 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2433,6 +2433,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index b8281727026d..edce072ed7bd 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1429,6 +1429,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/libbeat/_meta/config/default.reference.yml.tmpl b/libbeat/_meta/config/default.reference.yml.tmpl index 073340343d83..0b22112ed371 100644 --- a/libbeat/_meta/config/default.reference.yml.tmpl +++ b/libbeat/_meta/config/default.reference.yml.tmpl @@ -14,6 +14,7 @@ {{template "setup.dashboards.reference.yml.tmpl" .}} {{template "setup.template.reference.yml.tmpl" .}} {{template "setup.ilm.reference.yml.tmpl" .}} +{{template "setup.dsl.reference.yml.tmpl" .}} {{template "setup.kibana.reference.yml.tmpl" .}} {{template "logging.reference.yml.tmpl" .}} {{template "monitoring.reference.yml.tmpl" .}} diff --git a/libbeat/_meta/config/setup.dsl.reference.yml.tmpl b/libbeat/_meta/config/setup.dsl.reference.yml.tmpl new file mode 100644 index 000000000000..6ed48bc92ea5 --- /dev/null +++ b/libbeat/_meta/config/setup.dsl.reference.yml.tmpl @@ -0,0 +1,23 @@ +{{header "Index Lifecycle Management (DSL)"}} + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false \ No newline at end of file diff --git a/libbeat/cmd/export/ilm_policy.go b/libbeat/cmd/export/ilm_policy.go index 03fccc0a45b0..60c97920fd79 100644 --- a/libbeat/cmd/export/ilm_policy.go +++ b/libbeat/cmd/export/ilm_policy.go @@ -42,6 +42,9 @@ func GenGetILMPolicyCmd(settings instance.Settings) *cobra.Command { fatalfInitCmd(err) } + // the way this works, we decide to export ILM or DSL based on the user's config. + // This means that if a user has no index management config, we'll default to ILM, regardless of what the user + // is connected to. Might not be a problem since a user who doesn't have any custom lifecycle config has nothing to export? clientHandler, err := idxmgmt.NewFileClientHandler(newIdxmgmtClient(dir, version), b.Info, b.Config.LifecycleConfig) if err != nil { fatalf("error creating file handler: %s", err) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index fde397dbdab1..ecb72e3bf437 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -778,6 +778,8 @@ func (b *Beat) configure(settings Settings) error { } b.RawConfig = cfg + // set default before we unpack + b.Config.LifecycleConfig = lifecycle.DefaultILMConfig(b.Info) err = cfg.Unpack(&b.Config) if err != nil { return fmt.Errorf("error unpacking config data: %w", err) diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go index ad5126d831f9..efc9de347550 100644 --- a/libbeat/idxmgmt/idxmgmt.go +++ b/libbeat/idxmgmt/idxmgmt.go @@ -108,7 +108,8 @@ func MakeDefaultSupport(ilmSupport lifecycle.SupportFactory) SupportFactory { return func(log *logp.Logger, info beat.Info, configRoot *config.C) (Supporter, error) { const logName = "index-management" - // first fetch the ES output, check if we're running against serverless, use that to set a default config + // first fetch the ES output, check if we're running against serverless, use that to set a default config. + // the Supporter only cares about lifecycle config for checking if ILM/DSL is enabled or disabled. outCfg := struct { Output config.Namespace `config:"output"` }{} @@ -130,6 +131,7 @@ func MakeDefaultSupport(ilmSupport lifecycle.SupportFactory) SupportFactory { } } + // now that we have the "correct" default, unpack the rest of the config cfg := struct { ILM lifecycle.LifecycleConfig `config:",inline"` Template *config.C `config:"setup.template"` @@ -181,7 +183,7 @@ func checkTemplateESSettings(tmpl *config.C, out config.Namespace) error { var tmplCfg template.TemplateConfig if tmpl != nil { if err := tmpl.Unpack(&tmplCfg); err != nil { - return fmt.Errorf("unpacking template config fails: %v", err) + return fmt.Errorf("unpacking template config fails: %w", err) } } diff --git a/libbeat/idxmgmt/index_support.go b/libbeat/idxmgmt/index_support.go index 14e7fbf15336..71d0da5a19a8 100644 --- a/libbeat/idxmgmt/index_support.go +++ b/libbeat/idxmgmt/index_support.go @@ -214,7 +214,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string m.support.templateCfg.Overwrite, loadTemplate) if ilmComponent.load && !templateComponent.load { - return false, "Loading ILM policy without loading template is not recommended. Check your configuration." + return false, "Loading lifecycle policy without loading template is not recommended. Check your configuration." } if templateComponent.load && !ilmComponent.load && ilmComponent.enabled { @@ -244,7 +244,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { return err } if withILM { - log.Info("Auto ILM enable success.") + log.Info("Auto lifecycle enable success.") } // create feature objects for ILM and template setup @@ -252,7 +252,12 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), m.support.templateCfg.Overwrite, loadTemplate) - if ilmComponent.load { + if m.clientHandler.Mode() == lifecycle.DSL { + log.Info("setting up DSL") + } + + // on DSL, the template load will create the lifecycle policy + if ilmComponent.load && m.clientHandler.Mode() == lifecycle.ILM { // install ilm policy policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite) if err != nil { @@ -270,8 +275,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled if ilmComponent.enabled { - name := m.clientHandler.PolicyName() - tmplCfg, err = applyILMSettingsToTemplate(log, tmplCfg, name) + tmplCfg, err = applyLifecycleSettingsToTemplate(log, tmplCfg, m.clientHandler) if err != nil { return fmt.Errorf("error applying ILM settings: %w", err) } @@ -343,17 +347,17 @@ func unpackTemplateConfig(info beat.Info, cfg *config.C) (config template.Templa } // applies the specified ILM policy to the provided template, returns a struct of the template config -func applyILMSettingsToTemplate( +func applyLifecycleSettingsToTemplate( log *logp.Logger, tmpl template.TemplateConfig, - policyName string, + policymgr lifecycle.ClientHandler, ) (template.TemplateConfig, error) { if !tmpl.Enabled { return tmpl, nil } - if policyName == "" { - return tmpl, errors.New("no ilm policy name configured") + if policymgr.PolicyName() == "" { + return tmpl, errors.New("no policy name configured") } // init/copy index settings @@ -369,23 +373,28 @@ func applyILMSettingsToTemplate( } tmpl.Settings.Index = idxSettings - // init/copy index.lifecycle settings - var lifecycle map[string]interface{} - if ifcLifecycle := idxSettings["lifecycle"]; ifcLifecycle == nil { - lifecycle = map[string]interface{}{} - } else if tmp, ok := ifcLifecycle.(map[string]interface{}); ok { - lifecycle = make(map[string]interface{}, len(tmp)) - for k, v := range tmp { - lifecycle[k] = v + if policymgr.Mode() == lifecycle.ILM { + // init/copy index.lifecycle settings + var lifecycle map[string]interface{} + if ifcLifecycle := idxSettings["lifecycle"]; ifcLifecycle == nil { + lifecycle = map[string]interface{}{} + } else if tmp, ok := ifcLifecycle.(map[string]interface{}); ok { + lifecycle = make(map[string]interface{}, len(tmp)) + for k, v := range tmp { + lifecycle[k] = v + } + } else { + return tmpl, errors.New("settings.index.lifecycle must be an object") } - } else { - return tmpl, errors.New("settings.index.lifecycle must be an object") - } - idxSettings["lifecycle"] = lifecycle + idxSettings["lifecycle"] = lifecycle - if _, exists := lifecycle["name"]; !exists { - log.Infof("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", policyName) - lifecycle["name"] = policyName + if _, exists := lifecycle["name"]; !exists { + log.Infof("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", policymgr.PolicyName()) + lifecycle["name"] = policymgr.PolicyName() + } + } else { + // when we're in DSL mode, this is what actually creates the policy + tmpl.Settings.Lifecycle = policymgr.Policy().Body } return tmpl, nil diff --git a/libbeat/idxmgmt/lifecycle/client_handler.go b/libbeat/idxmgmt/lifecycle/client_handler.go index 6141a1cfc52a..1cf1d1b861c4 100644 --- a/libbeat/idxmgmt/lifecycle/client_handler.go +++ b/libbeat/idxmgmt/lifecycle/client_handler.go @@ -36,6 +36,8 @@ type ClientHandler interface { PolicyName() string Overwrite() bool CheckExists() bool + Policy() Policy + Mode() Mode } type VersionCheckerClient interface { diff --git a/libbeat/idxmgmt/lifecycle/client_handler_integration_test.go b/libbeat/idxmgmt/lifecycle/client_handler_integration_test.go index 4d989d414df3..f5dd04198bb0 100644 --- a/libbeat/idxmgmt/lifecycle/client_handler_integration_test.go +++ b/libbeat/idxmgmt/lifecycle/client_handler_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// //go:build integration +//go:build integration package lifecycle @@ -244,7 +244,8 @@ func TestFileClientHandler_CreateILMPolicy(t *testing.T) { assert.Equal(t, testPolicy.Name, testClient.name) assert.Equal(t, "policy", testClient.component) var out mapstr.M - json.Unmarshal([]byte(testClient.body), &out) + err = json.Unmarshal([]byte(testClient.body), &out) + require.NoError(t, err) assert.Equal(t, testPolicy.Body, out) } diff --git a/libbeat/idxmgmt/lifecycle/config.go b/libbeat/idxmgmt/lifecycle/config.go index 5543e858c3ca..4a4fcf2f22f9 100644 --- a/libbeat/idxmgmt/lifecycle/config.go +++ b/libbeat/idxmgmt/lifecycle/config.go @@ -77,34 +77,32 @@ func (cfg *Config) Validate() error { } func DefaultILMConfig(info beat.Info) LifecycleConfig { - policyFmt := fmtstr.MustCompileEvent(info.Beat) return LifecycleConfig{ ILM: Config{ Enabled: true, - PolicyName: *policyFmt, + PolicyName: *fmtstr.MustCompileEvent(info.Beat), PolicyFile: "", CheckExists: true, }, DSL: Config{ Enabled: false, - PolicyName: *policyFmt, + PolicyName: *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"), CheckExists: true, }, } } func DefaultDSLConfig(info beat.Info) LifecycleConfig { - policyFmt := fmtstr.MustCompileEvent(info.Beat) return LifecycleConfig{ ILM: Config{ Enabled: false, - PolicyName: *policyFmt, + PolicyName: *fmtstr.MustCompileEvent(info.Beat), PolicyFile: "", CheckExists: true, }, DSL: Config{ Enabled: true, - PolicyName: *policyFmt, + PolicyName: *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"), PolicyFile: "", CheckExists: true, }, diff --git a/libbeat/idxmgmt/lifecycle/error.go b/libbeat/idxmgmt/lifecycle/error.go index 444dde593a36..4f819bf8d066 100644 --- a/libbeat/idxmgmt/lifecycle/error.go +++ b/libbeat/idxmgmt/lifecycle/error.go @@ -55,19 +55,6 @@ func wrapErrf(cause, reason error, msg string, vs ...interface{}) error { } } -// ErrReason calls Reason() if the error implements this method. Otherwise return nil. -func ErrReason(err error) error { - if err == nil { - return nil - } - - ifc, ok := err.(interface{ Reason() error }) - if !ok { - return nil - } - return ifc.Reason() -} - // Cause returns the errors cause, if present. func (e *Error) Cause() error { return e.cause } diff --git a/libbeat/idxmgmt/lifecycle/es_client_handler.go b/libbeat/idxmgmt/lifecycle/es_client_handler.go index 0e379b5fd5d0..748dd8ab86f2 100644 --- a/libbeat/idxmgmt/lifecycle/es_client_handler.go +++ b/libbeat/idxmgmt/lifecycle/es_client_handler.go @@ -17,15 +17,12 @@ type ESClientHandler struct { defaultPolicy mapstr.M putPath string name string + policy Policy + mode Mode } // NewESClientHandler initializes and returns an ESClientHandler func NewESClientHandler(c ESClient, info beat.Info, cfg LifecycleConfig) (*ESClientHandler, error) { - policyName := cfg.ILM.PolicyName - if c.IsServerless() { - policyName = cfg.DSL.PolicyName - } - // trying to protect against config confusion; // it's possible that the "wrong" lifecycle got enabled somehow, // this is a last-ditch effort to fix things @@ -41,22 +38,42 @@ func NewESClientHandler(c ESClient, info beat.Info, cfg LifecycleConfig) (*ESCli } } + // by using IsServerless here, we're essentially letting the remote setting override the user config + policyName := cfg.ILM.PolicyName + if c.IsServerless() { + policyName = cfg.DSL.PolicyName + } + + // create name and policy name, err := ApplyStaticFmtstr(info, policyName) if err != nil { return nil, fmt.Errorf("error applying format string to policy name: %w", err) } - if name == "" { return nil, errors.New("could not generate usable policy name from config. Check setup.*.policy_name fields") } + // set defaults + defaultPolicy := DefaultILMPolicy + mode := ILM + path := fmt.Sprintf("%s/%s", esILMPath, name) + configType := cfg.ILM + if c.IsServerless() { - path := fmt.Sprintf("/_data_stream/%s/_lifecycle", name) - return &ESClientHandler{client: c, info: info, cfg: cfg.DSL, defaultPolicy: DefaultDSLPolicy, name: name, putPath: path}, nil + defaultPolicy = DefaultDSLPolicy + mode = DSL + path = fmt.Sprintf("/_data_stream/%s/_lifecycle", name) + configType = cfg.DSL } - path := fmt.Sprintf("%s/%s", esILMPath, name) - return &ESClientHandler{client: c, info: info, cfg: cfg.ILM, defaultPolicy: DefaultILMPolicy, name: name, putPath: path}, nil + policy, err := createPolicy(configType, info, defaultPolicy) + if err != nil { + return nil, fmt.Errorf("error creating DSL policy: %w", err) + } + + return &ESClientHandler{client: c, + info: info, cfg: configType, + defaultPolicy: defaultPolicy, name: name, putPath: path, policy: policy, mode: mode}, nil } // CheckExists returns the value of the check_exists config flag @@ -64,7 +81,7 @@ func (h *ESClientHandler) CheckExists() bool { return h.cfg.CheckExists } -// returns the value of the overwrite config flag +// Overwrite returns the value of the overwrite config flag func (h *ESClientHandler) Overwrite() bool { return h.cfg.Overwrite } @@ -116,13 +133,19 @@ func (h *ESClientHandler) PolicyName() string { return h.name } +// Policy returns the full policy +func (h *ESClientHandler) Policy() Policy { + return h.policy +} + +// Mode returns the connected instance mode +func (h *ESClientHandler) Mode() Mode { + return h.mode +} + // creates a policy from config, then performs the PUT request to ES func (h *ESClientHandler) createAndPutPolicy(cfg Config, info beat.Info, defaultPolicy mapstr.M) error { - policy, err := createPolicy(cfg, info, defaultPolicy) - if err != nil { - return fmt.Errorf("error creating lifetime policy: %w", err) - } - err = h.putPolicyToES(h.putPath, policy) + err := h.putPolicyToES(h.putPath, h.policy) if err != nil { return fmt.Errorf("error submitting policy: %w", err) } diff --git a/libbeat/idxmgmt/lifecycle/es_client_test.go b/libbeat/idxmgmt/lifecycle/es_client_test.go index 95f1fdc339b6..fbf6cf2d9fd6 100644 --- a/libbeat/idxmgmt/lifecycle/es_client_test.go +++ b/libbeat/idxmgmt/lifecycle/es_client_test.go @@ -3,11 +3,12 @@ package lifecycle import ( "testing" + "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/version" - "github.com/stretchr/testify/require" ) type mockESClient struct { @@ -42,6 +43,8 @@ func (client *mockESClient) Request(method, path string, pipeline string, params func TestESSetup(t *testing.T) { info := beat.Info{Beat: "test", Version: "9.9.9"} + bothEnabledConfig := DefaultDSLConfig(info) + bothEnabledConfig.ILM.Enabled = true cases := map[string]struct { serverless bool serverHasPolicy bool @@ -56,8 +59,8 @@ func TestESSetup(t *testing.T) { serverless: true, cfg: DefaultDSLConfig(info), err: false, - expectedPUTPath: "/_data_stream/test/_lifecycle", - expectedName: "test", + expectedPUTPath: "/_data_stream/test-9.9.9/_lifecycle", + expectedName: "test-9.9.9", expectedPolicy: DefaultDSLPolicy, }, "stateful-with-correct-default": { @@ -72,10 +75,33 @@ func TestESSetup(t *testing.T) { serverless: true, cfg: DefaultILMConfig(info), err: false, - expectedPUTPath: "/_data_stream/test/_lifecycle", + expectedPUTPath: "/_data_stream/test-9.9.9/_lifecycle", + expectedName: "test-9.9.9", + expectedPolicy: DefaultDSLPolicy, + }, + "stateful-with-wrong-defaults": { + serverless: false, + cfg: DefaultDSLConfig(info), + err: false, + expectedPUTPath: "/_ilm/policy/test", expectedName: "test", + }, + "serverless-with-both-enabled": { + serverless: true, + cfg: bothEnabledConfig, + err: false, + expectedPUTPath: "/_data_stream/test-9.9.9/_lifecycle", + expectedName: "test-9.9.9", expectedPolicy: DefaultDSLPolicy, }, + "stateful-with-both-enabled": { + serverless: false, + cfg: bothEnabledConfig, + err: false, + expectedPUTPath: "/_ilm/policy/test", + expectedName: "test", + expectedPolicy: DefaultILMPolicy, + }, "serverless-with-bare-PolicyName": { serverless: true, cfg: LifecycleConfig{DSL: Config{Enabled: true, CheckExists: true, PolicyName: *fmtstr.MustCompileEvent("")}}, diff --git a/libbeat/idxmgmt/lifecycle/file_client_handler.go b/libbeat/idxmgmt/lifecycle/file_client_handler.go index 7789aee2084c..0dff350883bd 100644 --- a/libbeat/idxmgmt/lifecycle/file_client_handler.go +++ b/libbeat/idxmgmt/lifecycle/file_client_handler.go @@ -15,11 +15,12 @@ type FileClientHandler struct { cfg Config defaultPolicy mapstr.M name string + mode Mode + policy Policy } // NewFileClientHandler initializes and returns a new FileClientHandler instance. func NewFileClientHandler(c FileClient, info beat.Info, cfg LifecycleConfig) (*FileClientHandler, error) { - // TODO: should return an error if both are enabled // default to ILM for file handler if cfg.DSL.Enabled && cfg.ILM.Enabled { return nil, errors.New("both ILM and DLM are enabled") @@ -35,23 +36,38 @@ func NewFileClientHandler(c FileClient, info beat.Info, cfg LifecycleConfig) (*F return nil, fmt.Errorf("error creating policy name: %w", err) } + // set defaults + defaultPolicy := DefaultILMPolicy + mode := ILM + configType := cfg.ILM + if cfg.DSL.Enabled { - return &FileClientHandler{client: c, info: info, cfg: cfg.DSL, defaultPolicy: DefaultDSLPolicy, name: name}, nil + defaultPolicy = DefaultDSLPolicy + mode = DSL + configType = cfg.DSL + } + + policy, err := createPolicy(configType, info, defaultPolicy) + if err != nil { + return nil, fmt.Errorf("error creating policy: %w", err) } - return &FileClientHandler{client: c, info: info, cfg: cfg.ILM, defaultPolicy: DefaultILMPolicy, name: name}, nil + + return &FileClientHandler{client: c, info: info, cfg: configType, + defaultPolicy: defaultPolicy, name: name, policy: policy, mode: mode}, nil } +// CheckExists returns the state of the check_exists config flag func (h *FileClientHandler) CheckExists() bool { return h.cfg.CheckExists } +// Overwrite returns the state of the overwrite config flag func (h *FileClientHandler) Overwrite() bool { return h.cfg.Enabled } -// CheckEnabled indicates whether or not ILM is supported for the configured mode and client version. -// If the connected ES instance is serverless, this will return false +// CheckEnabled indicates whether or not lifecycle management is supported for the configured mode and client version. func (h *FileClientHandler) CheckEnabled() (bool, error) { return checkILMEnabled(h.cfg.Enabled, h.client) } @@ -65,25 +81,31 @@ func (h *FileClientHandler) CreatePolicy(policy Policy) error { return nil } +// Policy returns the complete policy +func (h *FileClientHandler) Policy() Policy { + return h.policy +} + +// Mode returns the configured instance mode +func (h *FileClientHandler) Mode() Mode { + return h.mode +} + +// CreatePolicyFromConfig creates a lifecycle policy from its config and posts it to elasticsearch func (h *FileClientHandler) CreatePolicyFromConfig() error { // only applicable to testing if h.cfg.policyRaw != nil { return h.CreatePolicy(*h.cfg.policyRaw) } - // default to ILM - policy, err := createPolicy(h.cfg, h.info, DefaultILMPolicy) - if err != nil { - return fmt.Errorf("error creating ILM policy: %w", err) - } - - err = h.CreatePolicy(policy) + err := h.CreatePolicy(h.policy) if err != nil { return fmt.Errorf("error writing policy: %w", err) } return nil } +// PolicyName returns the generated policy name. func (h *FileClientHandler) PolicyName() string { return h.name } diff --git a/libbeat/idxmgmt/lifecycle/ilm.go b/libbeat/idxmgmt/lifecycle/ilm.go index 57cf6b73f6b5..034557416d81 100644 --- a/libbeat/idxmgmt/lifecycle/ilm.go +++ b/libbeat/idxmgmt/lifecycle/ilm.go @@ -39,6 +39,13 @@ type Supporter interface { Manager(h ClientHandler) Manager } +type Mode int + +const ( + ILM Mode = iota + DSL +) + // Manager uses a ClientHandler to install a policy. type Manager interface { CheckEnabled() (bool, error) @@ -48,7 +55,6 @@ type Manager interface { // The created flag is set to true only if a new policy is created. `created` // is false if an existing policy gets overwritten. EnsurePolicy(overwrite bool) (created bool, err error) - PolicyName() string } // Policy describes a policy to be loaded into Elasticsearch. diff --git a/libbeat/idxmgmt/lifecycle/ilm_test.go b/libbeat/idxmgmt/lifecycle/ilm_test.go index e1b286fd7b68..31c4313050dd 100644 --- a/libbeat/idxmgmt/lifecycle/ilm_test.go +++ b/libbeat/idxmgmt/lifecycle/ilm_test.go @@ -28,9 +28,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common/fmtstr" ) -// TODO: most of these tests are less useful, as we need the *manager* to tell us the policy, -// so these tests will need to be changed to provide a mock clientHandler, then we test the policy - func TestDefaultSupport_Init(t *testing.T) { info := beat.Info{Beat: "test", Version: "9.9.9"} @@ -71,7 +68,17 @@ func TestDefaultSupport_Init(t *testing.T) { } +func TestDefaultSupport_Manager_Enabled_Serverless(t *testing.T) { + cfg := DefaultDSLConfig(beat.Info{Name: "test"}) + runEnabledTests(t, cfg) +} + func TestDefaultSupport_Manager_Enabled(t *testing.T) { + cfg := DefaultILMConfig(beat.Info{Name: "test"}) + runEnabledTests(t, cfg) +} + +func runEnabledTests(t *testing.T, cfg LifecycleConfig) { cases := map[string]struct { calls []onCall cfg LifecycleConfig @@ -86,7 +93,7 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { calls: []onCall{ onCheckEnabled().Return(false, ErrESILMDisabled), }, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, err: true, }, "enabled via handler": { @@ -94,20 +101,20 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { onCheckEnabled().Return(true, nil), }, enabled: true, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, }, "handler confirms enabled flag": { calls: []onCall{ onCheckEnabled().Return(true, nil), }, - cfg: LifecycleConfig{ILM: Config{Enabled: true}}, + cfg: cfg, enabled: true, }, "io error": { calls: []onCall{ onCheckEnabled().Return(false, errors.New("ups")), }, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, err: true, }, } @@ -126,7 +133,7 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { require.Error(t, err) } if test.fail != nil { - assert.Equal(t, test.fail, ErrReason(err)) + assert.Equal(t, test.fail, err) } assert.Equal(t, test.enabled, enabled) @@ -135,12 +142,25 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { } } +func TestDefaultSupport_Manager_EnsurePolicy_Serverless(t *testing.T) { + testPolicy := Policy{ + Name: "test", + Body: DefaultDSLPolicy, + } + cfg := DefaultDSLConfig(beat.Info{Name: "test"}) + runEnsurePolicyTest(t, testPolicy, cfg) +} + func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { testPolicy := Policy{ Name: "test", Body: DefaultILMPolicy, } + cfg := DefaultILMConfig(beat.Info{Name: "test"}) + runEnsurePolicyTest(t, testPolicy, cfg) +} +func runEnsurePolicyTest(t *testing.T, testPolicy Policy, cfg LifecycleConfig) { cases := map[string]struct { calls []onCall overwrite bool @@ -155,7 +175,7 @@ func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { onHasPolicy().Return(false, nil), onCreatePolicyFromConfig().Return(nil), }, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, }, "policy already exists": { create: false, @@ -163,12 +183,12 @@ func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { onCheckExists().Return(true), onHasPolicy().Return(true, nil), }, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, }, "overwrite": { overwrite: true, create: true, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, calls: []onCall{ onCheckExists().Return(true), onCreatePolicyFromConfig().Return(nil), @@ -181,7 +201,7 @@ func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { onCreatePolicyFromConfig().Return(errOf(ErrRequestFailed)), }, fail: ErrRequestFailed, - cfg: DefaultILMConfig(beat.Info{Name: "test"}), + cfg: cfg, }, } @@ -197,7 +217,7 @@ func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { require.NoError(t, err) } else { require.Error(t, err) - assert.Equal(t, test.fail, ErrReason(err)) + assert.Equal(t, test.fail, err) } h.AssertExpectations(t) diff --git a/libbeat/idxmgmt/lifecycle/mock_client_handler.go b/libbeat/idxmgmt/lifecycle/mock_client_handler.go index 38ae7da5cdf4..ec960d478d2a 100644 --- a/libbeat/idxmgmt/lifecycle/mock_client_handler.go +++ b/libbeat/idxmgmt/lifecycle/mock_client_handler.go @@ -25,6 +25,7 @@ type mockHandler struct { mock.Mock cfg LifecycleConfig testPolicy Policy + mode Mode } type onCall struct { @@ -73,6 +74,14 @@ func (h *mockHandler) PolicyName() string { return h.testPolicy.Name } +func (h *mockHandler) Policy() Policy { + return h.testPolicy +} + +func (h *mockHandler) Mode() Mode { + return h.mode +} + func onCheckExists() onCall { return makeOnCall("CheckExists") } func (h *mockHandler) CheckExists() bool { args := h.Called() diff --git a/libbeat/idxmgmt/lifecycle/noop_manager.go b/libbeat/idxmgmt/lifecycle/noop_manager.go index 7222ca1c7037..05513a74bcfd 100644 --- a/libbeat/idxmgmt/lifecycle/noop_manager.go +++ b/libbeat/idxmgmt/lifecycle/noop_manager.go @@ -30,11 +30,23 @@ func NewNoopSupport(info beat.Info, c LifecycleConfig) (Supporter, error) { return (*noopSupport)(nil), nil } -func (*noopSupport) Enabled() bool { return false } -func (*noopSupport) Policy() Policy { return Policy{} } -func (*noopSupport) Overwrite() bool { return false } +// Enabled no-op +func (*noopSupport) Enabled() bool { return false } + +// Policy no-op +func (*noopSupport) Policy() Policy { return Policy{} } + +// Overwrite no-op +func (*noopSupport) Overwrite() bool { return false } + +// Manager no-op func (*noopSupport) Manager(_ ClientHandler) Manager { return (*noopManager)(nil) } -func (*noopManager) CheckEnabled() (bool, error) { return false, nil } +// CheckEnabled no-op +func (*noopManager) CheckEnabled() (bool, error) { return false, nil } + +// EnsurePolicy no-op func (*noopManager) EnsurePolicy(_ bool) (bool, error) { return false, errOf(ErrOpNotAvailable) } -func (*noopManager) PolicyName() string { return "" } + +// Policyname no-op +func (*noopManager) PolicyName() string { return "" } diff --git a/libbeat/idxmgmt/lifecycle/standard_manager.go b/libbeat/idxmgmt/lifecycle/standard_manager.go index 447374f75ef1..689ebf24d2b6 100644 --- a/libbeat/idxmgmt/lifecycle/standard_manager.go +++ b/libbeat/idxmgmt/lifecycle/standard_manager.go @@ -24,21 +24,20 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) +// stdSupport is a config wrapper that carries lifecycle info. type stdSupport struct { log *logp.Logger - - // cfg cfg LifecycleConfig } +// stdManager creates, checks, and updates lifecycle policies. type stdManager struct { *stdSupport client ClientHandler - - // cached info - cache infoCache + cache infoCache } +// infoCache stores config relating to caching lifecycle config type infoCache struct { LastUpdate time.Time Enabled bool @@ -47,6 +46,8 @@ type infoCache struct { var defaultCacheDuration = 5 * time.Minute // NewStdSupport creates an instance of default ILM support implementation. +// This contains only the config, and a manager must be created to write and check +// lifecycle policies. I suspect that with enough time/work, you could merge the stdSupport and stdManager objects func NewStdSupport( log *logp.Logger, cfg LifecycleConfig, @@ -57,8 +58,11 @@ func NewStdSupport( } } +// Enabled returns true if either ILM or DSL are enabled func (s *stdSupport) Enabled() bool { return s.cfg.ILM.Enabled || s.cfg.DSL.Enabled } +// Manager returns a standard support manager. unlike the stdSupport object, +// the manager is capable of writing and checking lifecycle policies. func (s *stdSupport) Manager(h ClientHandler) Manager { return &stdManager{ client: h, @@ -66,10 +70,7 @@ func (s *stdSupport) Manager(h ClientHandler) Manager { } } -func (m *stdManager) PolicyName() string { - return m.client.PolicyName() -} - +// CheckEnabled checks to see if lifecycle management is enabled. func (m *stdManager) CheckEnabled() (bool, error) { ilmEnabled, err := m.client.CheckEnabled() if err != nil { @@ -90,7 +91,7 @@ func (m *stdManager) CheckEnabled() (bool, error) { func (m *stdManager) EnsurePolicy(overwrite bool) (bool, error) { log := m.log if !m.client.CheckExists() { - log.Infof("ILM policy is not checked as setup.ilm.check_exists is disabled") + log.Infof("lifecycle policy is not checked as check_exists is disabled") return false, nil } overwrite = overwrite || m.client.Overwrite() @@ -107,25 +108,26 @@ func (m *stdManager) EnsurePolicy(overwrite bool) (bool, error) { switch { case exists && !overwrite: - log.Infof("ILM policy %v exists already.", name) + log.Infof("lifecycle policy %v exists already.", name) return false, nil case !exists || overwrite: err := m.client.CreatePolicyFromConfig() if err != nil { - log.Errorf("ILM policy %v creation failed: %v", name, err) + log.Errorf("lifecycle policy %v creation failed: %v", name, err) return false, err } - log.Infof("ILM policy %v successfully created.", name) + log.Infof("lifecycle policy %v successfully created.", name) return true, err default: - log.Infof("ILM policy not created: exists=%v, overwrite=%v.", exists, overwrite) + log.Infof("lifecycle policy not created: exists=%v, overwrite=%v.", exists, overwrite) return false, nil } } +// Valid returns true if the cache is valid func (c *infoCache) Valid() bool { return !c.LastUpdate.IsZero() && time.Since(c.LastUpdate) < defaultCacheDuration } diff --git a/libbeat/idxmgmt/mock_manager_test.go b/libbeat/idxmgmt/mock_manager_test.go index 21610a8c9688..2b251c12eaf5 100644 --- a/libbeat/idxmgmt/mock_manager_test.go +++ b/libbeat/idxmgmt/mock_manager_test.go @@ -62,7 +62,13 @@ func (m *mockILMSupport) Policy() lifecycle.Policy { return args.Get(0).(lifecycle.Policy) } -func onOverwrite() onCall { return makeOnCall("Overwrite") } +// func onMode() onCall { return makeOnCall("Mode") } +func (m *mockILMSupport) Mode() lifecycle.Mode { + args := m.Called() + return args.Get(0).(lifecycle.Mode) +} + +// func onOverwrite() onCall { return makeOnCall("Overwrite") } func (m *mockILMSupport) Overwrite() bool { return m.Called().Bool(0) } @@ -71,19 +77,19 @@ func (m *mockILMSupport) Manager(_ lifecycle.ClientHandler) lifecycle.Manager { return m } -func onCheckEnabled() onCall { return makeOnCall("CheckEnabled") } +// func onCheckEnabled() onCall { return makeOnCall("CheckEnabled") } func (m *mockILMSupport) CheckEnabled() (bool, error) { args := m.Called() return args.Bool(0), args.Error(1) } -func onEnsurePolicy() onCall { return makeOnCall("EnsurePolicy") } +// func onEnsurePolicy() onCall { return makeOnCall("EnsurePolicy") } func (m *mockILMSupport) EnsurePolicy(overwrite bool) (bool, error) { args := m.Called() return args.Bool(0), args.Error(1) } -func onPolicyName() onCall { return makeOnCall("PolicyName") } +// func onPolicyName() onCall { return makeOnCall("PolicyName") } func (m *mockILMSupport) PolicyName() string { args := m.Called() return args.String(0) diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index 57de51526b64..3299d2e59221 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -215,12 +215,12 @@ func TestIndexManager_VerifySetup(t *testing.T) { "load template with ilm without loading ilm": { ilmEnabled: true, tmplEnabled: true, loadILM: LoadModeDisabled, warn: "whithout loading ILM policy", - lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true}}, + lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true, PolicyName: *fmtstr.MustCompileEvent("test")}}, }, "load ilm without template": { ilmEnabled: true, loadILM: LoadModeUnset, warn: "without loading template is not recommended", - lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true}}, + lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true, PolicyName: *fmtstr.MustCompileEvent("test")}}, }, "template disabled but loading enabled": { loadTmpl: LoadModeEnabled, @@ -246,12 +246,12 @@ func TestIndexManager_VerifySetup(t *testing.T) { tmplEnabled: true, ilmEnabled: true, ilmOverwrite: false, loadILM: LoadModeEnabled, warn: "Overwriting ILM policy is disabled", - lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true, Overwrite: false}}, + lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true, Overwrite: false, PolicyName: *fmtstr.MustCompileEvent("test")}}, }, "everything enabled": { tmplEnabled: true, ilmEnabled: true, ilmOverwrite: true, - lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true, Overwrite: true}}, + lifecycle: lifecycle.LifecycleConfig{ILM: lifecycle.Config{Enabled: true, Overwrite: true, PolicyName: *fmtstr.MustCompileEvent("test")}}, ok: true, }, } { @@ -473,6 +473,7 @@ type mockClientHandler struct { lifecycle lifecycle.LifecycleConfig selectedCfg lifecycle.Config operations []mockCreateOp + mode lifecycle.Mode } func newMockClientHandler(cfg lifecycle.LifecycleConfig, info beat.Info) (*mockClientHandler, error) { @@ -524,6 +525,14 @@ func (h *mockClientHandler) PolicyName() string { return h.policyName } +func (h *mockClientHandler) Policy() lifecycle.Policy { + return lifecycle.Policy{} +} + +func (h *mockClientHandler) Mode() lifecycle.Mode { + return h.mode +} + func (h *mockClientHandler) createILMPolicy(policy lifecycle.Policy) error { h.recordOp(mockCreatePolicy) h.policyName = policy.Name diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 9e8a2c246321..db2c688d1c1f 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -42,8 +42,9 @@ type TemplateConfig struct { // TemplateSettings are part of the Elasticsearch template and hold index and source specific information. type TemplateSettings struct { - Index map[string]interface{} `config:"index"` - Source map[string]interface{} `config:"_source"` + Index map[string]interface{} `config:"index"` + Source map[string]interface{} `config:"_source"` + Lifecycle map[string]interface{} `config:"lifecycle"` } // DefaultConfig for index template diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 7c43efb8d7d6..2869e7932ae2 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -246,6 +246,9 @@ func (t *Template) generateComponent(properties, analyzers mapstr.M, dynamicTemp }, }, } + if len(t.config.Settings.Lifecycle) > 0 { + m.Put("template.lifecycle", t.config.Settings.Lifecycle) + } if len(analyzers) != 0 { m.Put("template.settings.analysis.analyzer", analyzers) } diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index eabdcc8e918e..499dc4914abf 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2172,6 +2172,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 7442b7f6a0f1..f9bd1d4886f7 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1803,6 +1803,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 64377c0fc6d6..092cd8213736 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1219,6 +1219,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index e9ecd33ae390..731f682a705c 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1393,6 +1393,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9ea0dabfb0b8..72dff119a7f5 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4803,6 +4803,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 7855538f621a..16018bc1553b 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1057,6 +1057,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index b8281727026d..edce072ed7bd 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1429,6 +1429,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index fa15aca7fb6f..6065914dd342 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2733,6 +2733,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 416462f3f47a..ee620feec261 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -776,6 +776,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 7442b7f6a0f1..f9bd1d4886f7 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1803,6 +1803,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 3c6799e7329c..94c1221ad2e8 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1221,6 +1221,29 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false +# ====================== Index Lifecycle Management (DSL) ====================== + +# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. +# These settings are mutually excusive with ILM settings. + +# Enable DSL support. Valid values are true, or false. +#setup.dsl.enabled: true + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.dsl.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.dsl.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy +# can be installed. +#setup.dsl.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.dsl.overwrite: false # =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.