diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index abe22daabcd1..b8de6e850248 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -9,6 +9,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d ==== Breaking changes *Affecting all Beats* +- The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] + *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 9ee79c8c79a5..3a8cc206910f 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -453,8 +453,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 76a6a469b609..2dfdeb602182 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1477,8 +1477,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 3fb0d0e68880..4a7b4f398f92 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -598,8 +598,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index 943e6a16b478..cb6ad43d4305 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -9,8 +9,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 816078f92251..0d8254d55cd7 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -65,7 +65,7 @@ var ( Password: "", APIKey: "", MaxRetries: 3, - CompressionLevel: 0, + CompressionLevel: 1, EscapeHTML: false, Kerberos: nil, LoadBalance: true, diff --git a/libbeat/outputs/elasticsearch/config_test.go b/libbeat/outputs/elasticsearch/config_test.go index 15d934a1e865..c0572bcdf783 100644 --- a/libbeat/outputs/elasticsearch/config_test.go +++ b/libbeat/outputs/elasticsearch/config_test.go @@ -97,6 +97,28 @@ non_indexable_policy.dead_letter_index: } } +func TestCompressionIsOnByDefault(t *testing.T) { + config := "" + c := common.MustNewConfigFrom(config) + elasticsearchOutputConfig, err := readConfig(c) + if err != nil { + t.Fatalf("Can't create test configuration from valid input") + } + assert.Equal(t, 1, elasticsearchOutputConfig.CompressionLevel, "Default compression level should be 1") +} + +func TestExplicitCompressionLevelOverridesDefault(t *testing.T) { + config := ` +compression_level: 0 +` + c := common.MustNewConfigFrom(config) + elasticsearchOutputConfig, err := readConfig(c) + if err != nil { + t.Fatalf("Can't create test configuration from valid input") + } + assert.Equal(t, 0, elasticsearchOutputConfig.CompressionLevel, "Explicit compression level should override defaults") +} + func readConfig(cfg *common.Config) (*elasticsearchConfig, error) { c := defaultConfig if err := cfg.Unpack(&c); err != nil { diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 4085fe01785a..9ecc9972917f 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -102,7 +102,7 @@ The compression level must be in the range of `1` (best speed) to `9` (best comp Increasing the compression level will reduce the network usage but will increase the cpu usage. -The default value is `0`. +The default value is `1`. ===== `escape_html` diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 601c1712088e..74a5aaf77319 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1265,8 +1265,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index f5ef5cc71e58..26465c9d5020 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -947,8 +947,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index d85a95b7e109..50576fa5690b 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -383,8 +383,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index a16c015888ed..e65884993c21 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -509,8 +509,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6040f7e241ad..6798ba69cbe9 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3738,8 +3738,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index c4d2bcd9b4ce..d5a4a666d1b1 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -630,8 +630,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 3fb0d0e68880..4a7b4f398f92 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -598,8 +598,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 2e146e1635f2..4fc7df5ee6e7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1796,8 +1796,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 628c52e35559..1d24651900bc 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -349,8 +349,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index f5ef5cc71e58..26465c9d5020 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -947,8 +947,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 6221cea5b593..f4909990108c 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -426,8 +426,9 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] - # Set gzip compression level. - #compression_level: 0 + # Set gzip compression level. Set to 0 to disable compression. + # The default is 1. + #compression_level: 1 # Configure escaping HTML symbols in strings. #escape_html: false