diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4e9a5f0128e2..a59b65297c28 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -173,6 +173,7 @@ is collected by it. - allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788] - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments +- elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index d4214eaf604b..a3a36dde753f 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -522,6 +522,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 45aff60ce236..f49390485aca 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1618,6 +1618,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e8b74f8c075e..fe6a72cd4746 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -614,6 +614,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index 4acd341da01e..d6ebe03ada5d 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -81,6 +81,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index c80e95ebc907..b485807776e7 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -102,6 +102,7 @@ func NewClient( CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, Transport: s.Transport, + IdleConnTimeout: s.IdleConnTimeout, }) if err != nil { return nil, err diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 7e3e64b6cc5c..6af56ac42db8 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -689,6 +689,12 @@ default is `1s`. The maximum number of seconds to wait before attempting to connect to Elasticsearch after a network error. The default is `60s`. +===== `idle_connection_timeout` + +The maximum amount of time an idle connection will remain idle before closing itself. +Zero means no limit. The format is a Go language duration (example 60s is 60 seconds). +The default is 0. + ===== `timeout` The http request timeout in seconds for the Elasticsearch request. The default is 90. diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index f7e388539243..145b5b65c2ea 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -41,7 +41,9 @@ func makeES( ) (outputs.Group, error) { log := logp.NewLogger(logSelector) if !cfg.HasField("bulk_max_size") { - _ = cfg.SetInt("bulk_max_size", -1, defaultBulkSize) + if err := cfg.SetInt("bulk_max_size", -1, defaultBulkSize); err != nil { + return outputs.Fail(err) + } } index, pipeline, err := buildSelectors(im, beat, cfg) @@ -105,6 +107,7 @@ func makeES( Observer: observer, EscapeHTML: esConfig.EscapeHTML, Transport: esConfig.Transport, + IdleConnTimeout: esConfig.Transport.IdleConnTimeout, }, Index: index, Pipeline: pipeline, diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index fc79ddb514c9..8508d7d6a700 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1357,6 +1357,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index cc05f7b52128..d31a7b901d9d 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -988,6 +988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 110370957cf7..bdc366d09024 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -404,6 +404,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 6d9a71ca99cc..09b343d8fe28 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -578,6 +578,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index bff96ef19973..90cbc52d8c22 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3988,6 +3988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index d3a2231a43ef..b7b075f26120 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -646,6 +646,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index e8b74f8c075e..fe6a72cd4746 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -614,6 +614,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 436693bdfbc7..bfb13acc660f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1918,6 +1918,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index f17d16e28b8a..6103f48c4280 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -365,6 +365,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index cc05f7b52128..d31a7b901d9d 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -988,6 +988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index eec0bca80779..f3ff654bca24 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -406,6 +406,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90