From 46c24f96eb8fc021dbb07f943d146c66f166cc16 Mon Sep 17 00:00:00 2001 From: rkbennett <44292326+rkbennett@users.noreply.github.com> Date: Sun, 13 Sep 2020 00:08:12 -0500 Subject: [PATCH 1/2] Index Template Support Update Adds support for new index template schema Adds handling for adding ilm configurations when settings field doesn't exist Only overwrites index_patterns from template when absolutely necessary Adds legacy_template option (boolean), which will change paths between _index_template and _template, and also configure ilm fields appropriately --- .../outputs/elasticsearch/common_configs.rb | 6 ++ .../outputs/elasticsearch/http_client.rb | 16 ++++- .../elasticsearch/http_client_builder.rb | 3 +- .../outputs/elasticsearch/template_manager.rb | 63 ++++++++++++++++--- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index dc2efea56..9bb72c4d3 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -50,6 +50,12 @@ def self.included(mod) # If not set, the included template will be used. mod.config :template, :validate => :path + # The legacy_template option will use the old /_template (ELK >7.7) + # path for creating index templates. + # It will also construct the ilm configurations for the template in a manner + # which is compatible with legacy templates + mod.config :legacy_template, :validate => :boolean, :default => false + # The template_overwrite option will always overwrite the indicated template # in Elasticsearch with either the one indicated by template or the included one. # This option is set to false by default. If you always want to stay up to date diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 32a37e82a..7c9a11aa8 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -252,6 +252,10 @@ def http_compression client_settings.fetch(:http_compression, false) end + def legacy_template + client_settings.fetch(:legacy) + end + def build_adapter(options) timeout = options[:timeout] || 0 @@ -343,11 +347,19 @@ def exists?(path, use_get=false) end def template_exists?(name) - exists?("/_template/#{name}") + if legacy_template + exists?("/_template/#{name}") + else + exists?("/_index_template/#{name}") + end end def template_put(name, template) - path = "_template/#{name}" + if legacy_template + path = "_template/#{name}" + else + path = "_index_template/#{name}" + end logger.info("Installing elasticsearch template to #{path}") @pool.put(path, nil, LogStash::Json.dump(template)) end diff --git a/lib/logstash/outputs/elasticsearch/http_client_builder.rb b/lib/logstash/outputs/elasticsearch/http_client_builder.rb index fd8827f71..bbc57fa66 100644 --- a/lib/logstash/outputs/elasticsearch/http_client_builder.rb +++ b/lib/logstash/outputs/elasticsearch/http_client_builder.rb @@ -9,7 +9,8 @@ def self.build(logger, hosts, params) :pool_max_per_route => params["pool_max_per_route"], :check_connection_timeout => params["validate_after_inactivity"], :http_compression => params["http_compression"], - :headers => params["custom_headers"] || {} + :headers => params["custom_headers"] || {}, + :legacy => params["legacy_template"] } client_settings[:proxy] = params["proxy"] if params["proxy"] diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 20f2cbd2c..0c139a6f8 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -32,14 +32,63 @@ def self.install(client, template_name, template, template_overwrite) end def self.add_ilm_settings_to_template(plugin, template) - # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern - # definition - remove any existing definition of 'template' - template.delete('template') if template.include?('template') - template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" - if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias']) - plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.") + if plugin.original_params.key?('ilm_rollover_alias') + # If no index patterns in the 'index_pattern' array would include the 'ilm_rollover_alias' add the 'ilm_rollover_alias' to the 'index_pattern' array + if template.key?('index_patterns') && template['index_patterns'].kind_of?(Array) && !template['index_patterns'].any? { |idx_pattern| "#{plugin.ilm_rollover_alias}-".start_with?(idx_pattern.tr('*','')) } + plugin.logger.info("Adding index pattern name for rollover alias as ILM is enabled.") + template['index_patterns'].append("#{plugin.ilm_rollover_alias}-*") + # If 'index_pattern' is not an array and doesn't include the 'ilm_rollover_alias' set the 'index_pattern' to the 'ilm_rollover_alias' + elsif template.key?('index_patterns') && !template['index_patterns'].kind_of?(Array) && !"#{plugin.ilm_rollover_alias}-".start_with?(template['index_patterns'].tr('*','')) + plugin.logger.info("Overwriting index pattern name for rollover alias as ILM is enabled.") + template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" + # If 'index_pattern' doesn't exist, set it to the 'ilm_rollover_alias' + elsif !template.key?('index_patterns') + plugin.logger.info("Setting index pattern name for rollover alias as ILM is enabled.") + template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" + end + end + if plugin.legacy_template + # definition - remove any existing definition of 'template' + template.delete('template') if template.include?('template') + # Create settings hash if not in the template, but ilm is enabled + if !template.key?('settings') + template['settings'] = { } + end + if plugin.original_params.key?('ilm_rollover_alias') + if template['settings'] + # Overwrite the rollover_alias, sense it was defined in the output plugin + plugin.logger.info("Overwriting index rollover alias with plugin defined alias.") + template['settings'].update({ 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) + end + end + if plugin.original_params.key?('ilm_policy') + if template['settings'] + # Overwrite the ilm_policy, sense it was defined in the output plugin + plugin.logger.info("Overwriting index lifecycle name with plugin defined ilm policy.") + template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy}) + end + end + else + #plugin.logger.warn(plugin.original_params.key?('ilm_rollover_alias')) + # Create settings hash if not in the template, but ilm is enabled + if !template['template'].key?('settings') + template['template']['settings'] = { } + end + if plugin.original_params.key?('ilm_rollover_alias') + if template['template']['settings'] + # Overwrite the rollover_alias, sense it was defined in the output plugin + plugin.logger.info("Overwriting index rollover alias with plugin defined alias.") + template['template']['settings'].update({ 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) + end + end + if plugin.original_params.key?('ilm_policy') + if template['template']['settings'] + # Overwrite the ilm_policy, sense it was defined in the output plugin + plugin.logger.info("Overwriting index lifecycle name with plugin defined ilm policy.") + template['template']['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy}) + end + end end - template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) end # Template name - if template_name set, use it From 82dab65579f49e760d2a083e2b7474fdf9f7f080 Mon Sep 17 00:00:00 2001 From: rkbennett <44292326+rkbennett@users.noreply.github.com> Date: Wed, 16 Sep 2020 08:19:19 -0500 Subject: [PATCH 2/2] Set legacy_template default Sets legacy_template to default to boolean true, this is due to undefined template names when template management is enabled. The current default logstash template is only written for the legacy index template schema --- lib/logstash/outputs/elasticsearch/common_configs.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index 9bb72c4d3..f86a1c87b 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -54,7 +54,7 @@ def self.included(mod) # path for creating index templates. # It will also construct the ilm configurations for the template in a manner # which is compatible with legacy templates - mod.config :legacy_template, :validate => :boolean, :default => false + mod.config :legacy_template, :validate => :boolean, :default => true # The template_overwrite option will always overwrite the indicated template # in Elasticsearch with either the one indicated by template or the included one.