From a0a78c6db388b6174498b38973051ca474a6e1c6 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Sep 2023 15:29:28 -0400 Subject: [PATCH 1/6] Enable compression by default for Elasticsearch outputs --- libbeat/outputs/elasticsearch/config.go | 2 +- libbeat/outputs/elasticsearch/config_test.go | 22 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 7134e9333cb8..ca77a44b8331 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -65,7 +65,7 @@ var ( Password: "", APIKey: "", MaxRetries: 3, - CompressionLevel: 0, + CompressionLevel: 1, EscapeHTML: false, Kerberos: nil, LoadBalance: true, diff --git a/libbeat/outputs/elasticsearch/config_test.go b/libbeat/outputs/elasticsearch/config_test.go index 22fe4bb4c82e..32cb90c904c4 100644 --- a/libbeat/outputs/elasticsearch/config_test.go +++ b/libbeat/outputs/elasticsearch/config_test.go @@ -97,6 +97,28 @@ non_indexable_policy.dead_letter_index: } } +func TestCompressionIsOnByDefault(t *testing.T) { + config := "" + c := conf.MustNewConfigFrom(config) + elasticsearchOutputConfig, err := readConfig(c) + if err != nil { + t.Fatalf("Can't create test configuration from valid input") + } + assert.Equal(t, 1, elasticsearchOutputConfig.CompressionLevel, "Default compression level should be 1") +} + +func TestExplicitCompressionLevelOverridesDefault(t *testing.T) { + config := ` +compression_level: 0 +` + c := conf.MustNewConfigFrom(config) + elasticsearchOutputConfig, err := readConfig(c) + if err != nil { + t.Fatalf("Can't create test configuration from valid input") + } + assert.Equal(t, 0, elasticsearchOutputConfig.CompressionLevel, "Explicit compression level should override defaults") +} + func readConfig(cfg *conf.C) (*elasticsearchConfig, error) { c := defaultConfig if err := cfg.Unpack(&c); err != nil { From b957822a1b6af61b6b516ed6f572ab68603385f6 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Sep 2023 16:00:48 -0400 Subject: [PATCH 2/6] update config template --- libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index edc40e632d72..48f1ba2c007b 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -9,8 +9,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false From e0f13772e93c55ce26b6f75145b74c448c21c160 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Sep 2023 16:15:26 -0400 Subject: [PATCH 3/6] update changelog --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 228d201f74fd..34273b748366 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -9,6 +9,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] ==== Breaking changes *Affecting all Beats* +- The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] + *Auditbeat* From dcdce15720f0b0add7d90d316e1d34092c176d5d Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Sep 2023 16:45:21 -0400 Subject: [PATCH 4/6] regenerate config templates --- auditbeat/auditbeat.reference.yml | 5 +++-- filebeat/filebeat.reference.yml | 5 +++-- heartbeat/heartbeat.reference.yml | 5 +++-- metricbeat/metricbeat.reference.yml | 5 +++-- packetbeat/packetbeat.reference.yml | 5 +++-- winlogbeat/winlogbeat.reference.yml | 5 +++-- x-pack/auditbeat/auditbeat.reference.yml | 5 +++-- x-pack/filebeat/filebeat.reference.yml | 5 +++-- x-pack/heartbeat/heartbeat.reference.yml | 5 +++-- x-pack/metricbeat/metricbeat.reference.yml | 5 +++-- x-pack/packetbeat/packetbeat.reference.yml | 5 +++-- x-pack/winlogbeat/winlogbeat.reference.yml | 5 +++-- 12 files changed, 36 insertions(+), 24 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 892d4e1b2d09..c3b10d03f794 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -450,8 +450,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 2de0bc61f565..def62fbca7c6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1546,8 +1546,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index b8281727026d..fdbdcbe70421 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -542,8 +542,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index eabdcc8e918e..a03fbf080b48 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1285,8 +1285,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 7442b7f6a0f1..f1e036707b7e 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -916,8 +916,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 64377c0fc6d6..1c8ebebc91ae 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -332,8 +332,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index e9ecd33ae390..7b575e6b0e83 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -506,8 +506,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9ea0dabfb0b8..ff9b523f9471 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3916,8 +3916,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index b8281727026d..fdbdcbe70421 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -542,8 +542,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index fa15aca7fb6f..5ea3499cee58 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1846,8 +1846,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 7442b7f6a0f1..f1e036707b7e 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -916,8 +916,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 3c6799e7329c..0ec02b0e5ae5 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -334,8 +334,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false From 2d4947d4f5ce422b136f7256c35f1264bb7471f0 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Sep 2023 16:48:02 -0400 Subject: [PATCH 5/6] update docs --- libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index cbe74279dcb3..5ea65c16dc4e 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -105,7 +105,7 @@ The compression level must be in the range of `1` (best speed) to `9` (best comp Increasing the compression level will reduce the network usage but will increase the cpu usage. -The default value is `0`. +The default value is `1`. ===== `escape_html` From 9616c57e32a5661b080a2fb92aa2dc0813fb76a4 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 27 Sep 2023 10:00:08 -0400 Subject: [PATCH 6/6] make check --- x-pack/functionbeat/functionbeat.reference.yml | 5 +++-- x-pack/osquerybeat/osquerybeat.reference.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 7855538f621a..5c42e49b44fe 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -574,8 +574,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 416462f3f47a..9d2c21c5ad1d 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -293,8 +293,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false