From f6f64c82a52db0765d536de540d43d9620b4c0eb Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 28 Sep 2023 09:29:14 -0400 Subject: [PATCH 1/2] Enable compression by default for Elasticsearch outputs (#36681) * Enable compression by default for Elasticsearch outputs * update config template * update changelog * regenerate config templates * update docs * make check (cherry picked from commit 231e93a6d20e35a8fc6f34dc767110b8a27ec645) # Conflicts: # libbeat/outputs/elasticsearch/config_test.go --- CHANGELOG.next.asciidoc | 2 ++ auditbeat/auditbeat.reference.yml | 5 ++-- filebeat/filebeat.reference.yml | 5 ++-- heartbeat/heartbeat.reference.yml | 5 ++-- .../output-elasticsearch.reference.yml.tmpl | 5 ++-- libbeat/outputs/elasticsearch/config.go | 2 +- libbeat/outputs/elasticsearch/config_test.go | 26 +++++++++++++++++++ .../elasticsearch/docs/elasticsearch.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 5 ++-- packetbeat/packetbeat.reference.yml | 5 ++-- winlogbeat/winlogbeat.reference.yml | 5 ++-- x-pack/auditbeat/auditbeat.reference.yml | 5 ++-- x-pack/filebeat/filebeat.reference.yml | 5 ++-- .../functionbeat/functionbeat.reference.yml | 5 ++-- x-pack/heartbeat/heartbeat.reference.yml | 5 ++-- x-pack/metricbeat/metricbeat.reference.yml | 5 ++-- x-pack/osquerybeat/osquerybeat.reference.yml | 5 ++-- x-pack/packetbeat/packetbeat.reference.yml | 5 ++-- x-pack/winlogbeat/winlogbeat.reference.yml | 5 ++-- 19 files changed, 75 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index abe22daabcd1..b8de6e850248 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -9,6 +9,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d ==== Breaking changes *Affecting all Beats* +- The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] + *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 9ee79c8c79a5..3a8cc206910f 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -453,8 +453,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 76a6a469b609..2dfdeb602182 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1477,8 +1477,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 3fb0d0e68880..4a7b4f398f92 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -598,8 +598,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index 943e6a16b478..cb6ad43d4305 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -9,8 +9,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 816078f92251..0d8254d55cd7 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -65,7 +65,7 @@ var ( Password: "", APIKey: "", MaxRetries: 3, - CompressionLevel: 0, + CompressionLevel: 1, EscapeHTML: false, Kerberos: nil, LoadBalance: true, diff --git a/libbeat/outputs/elasticsearch/config_test.go b/libbeat/outputs/elasticsearch/config_test.go index 15d934a1e865..5c191612e661 100644 --- a/libbeat/outputs/elasticsearch/config_test.go +++ b/libbeat/outputs/elasticsearch/config_test.go @@ -97,7 +97,33 @@ non_indexable_policy.dead_letter_index: } } +<<<<<<< HEAD func readConfig(cfg *common.Config) (*elasticsearchConfig, error) { +======= +func TestCompressionIsOnByDefault(t *testing.T) { + config := "" + c := conf.MustNewConfigFrom(config) + elasticsearchOutputConfig, err := readConfig(c) + if err != nil { + t.Fatalf("Can't create test configuration from valid input") + } + assert.Equal(t, 1, elasticsearchOutputConfig.CompressionLevel, "Default compression level should be 1") +} + +func TestExplicitCompressionLevelOverridesDefault(t *testing.T) { + config := ` +compression_level: 0 +` + c := conf.MustNewConfigFrom(config) + elasticsearchOutputConfig, err := readConfig(c) + if err != nil { + t.Fatalf("Can't create test configuration from valid input") + } + assert.Equal(t, 0, elasticsearchOutputConfig.CompressionLevel, "Explicit compression level should override defaults") +} + +func readConfig(cfg *conf.C) (*elasticsearchConfig, error) { +>>>>>>> 231e93a6d2 (Enable compression by default for Elasticsearch outputs (#36681)) c := defaultConfig if err := cfg.Unpack(&c); err != nil { return nil, err diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 4085fe01785a..9ecc9972917f 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -102,7 +102,7 @@ The compression level must be in the range of `1` (best speed) to `9` (best comp Increasing the compression level will reduce the network usage but will increase the cpu usage. -The default value is `0`. +The default value is `1`. ===== `escape_html` diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 601c1712088e..74a5aaf77319 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1265,8 +1265,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index f5ef5cc71e58..26465c9d5020 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -947,8 +947,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index d85a95b7e109..50576fa5690b 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -383,8 +383,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index a16c015888ed..e65884993c21 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -509,8 +509,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6040f7e241ad..6798ba69cbe9 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3738,8 +3738,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index c4d2bcd9b4ce..d5a4a666d1b1 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -630,8 +630,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 3fb0d0e68880..4a7b4f398f92 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -598,8 +598,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 2e146e1635f2..4fc7df5ee6e7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1796,8 +1796,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 628c52e35559..1d24651900bc 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -349,8 +349,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index f5ef5cc71e58..26465c9d5020 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -947,8 +947,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 6221cea5b593..f4909990108c 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -426,8 +426,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false From 2ed4e9cf9b6acb1850e9db6bf11caeb1afcda261 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 29 Sep 2023 13:32:23 -0400 Subject: [PATCH 2/2] Resolve conflict in unit test. --- libbeat/outputs/elasticsearch/config_test.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libbeat/outputs/elasticsearch/config_test.go b/libbeat/outputs/elasticsearch/config_test.go index 5c191612e661..c0572bcdf783 100644 --- a/libbeat/outputs/elasticsearch/config_test.go +++ b/libbeat/outputs/elasticsearch/config_test.go @@ -97,12 +97,9 @@ non_indexable_policy.dead_letter_index: } } -<<<<<<< HEAD -func readConfig(cfg *common.Config) (*elasticsearchConfig, error) { -======= func TestCompressionIsOnByDefault(t *testing.T) { config := "" - c := conf.MustNewConfigFrom(config) + c := common.MustNewConfigFrom(config) elasticsearchOutputConfig, err := readConfig(c) if err != nil { t.Fatalf("Can't create test configuration from valid input") @@ -114,7 +111,7 @@ func TestExplicitCompressionLevelOverridesDefault(t *testing.T) { config := ` compression_level: 0 ` - c := conf.MustNewConfigFrom(config) + c := common.MustNewConfigFrom(config) elasticsearchOutputConfig, err := readConfig(c) if err != nil { t.Fatalf("Can't create test configuration from valid input") @@ -122,8 +119,7 @@ compression_level: 0 assert.Equal(t, 0, elasticsearchOutputConfig.CompressionLevel, "Explicit compression level should override defaults") } -func readConfig(cfg *conf.C) (*elasticsearchConfig, error) { ->>>>>>> 231e93a6d2 (Enable compression by default for Elasticsearch outputs (#36681)) +func readConfig(cfg *common.Config) (*elasticsearchConfig, error) { c := defaultConfig if err := cfg.Unpack(&c); err != nil { return nil, err