From 0a5b256608c0c5d0d8e5d4e87b950a68c2a7d2a8 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 8 Apr 2024 17:27:28 +0200 Subject: [PATCH 01/26] Add `log.type: event` to log entries containing events --- go.mod | 3 +- go.sum | 10 +--- libbeat/cmd/instance/beat.go | 3 +- libbeat/docs/loggingconfig.asciidoc | 76 +++++++++++++++++++++++++ libbeat/outputs/elasticsearch/client.go | 12 ++-- libbeat/outputs/fileout/file.go | 4 +- libbeat/outputs/kafka/client.go | 3 +- libbeat/outputs/redis/client.go | 9 +-- 8 files changed, 98 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 8d278dae027d..1bbbdc63592f 100644 --- a/go.mod +++ b/go.mod @@ -289,7 +289,6 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.5.0 // indirect github.com/elastic/go-windows v1.0.1 // indirect - github.com/elastic/pkcs8 v1.0.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fearful-symmetry/gomsr v0.0.1 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect @@ -433,3 +432,5 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible + +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408152242-d3dd5568916d diff --git a/go.sum b/go.sum index e91e46ce2c67..e6e21635b920 100644 --- a/go.sum +++ b/go.sum @@ -380,6 +380,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408152242-d3dd5568916d h1:gxEYMj2FDnIB3rTWBPPzuoC2owMUuli+oi5/6UQXSrI= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408152242-d3dd5568916d/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= @@ -554,8 +556,6 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.8 h1:fwl3hp0gNmKkuERcUQTwe4cyIK6M0jJkv16EIsB6Apw= -github.com/elastic/elastic-agent-libs v0.9.8/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/elastic/elastic-agent-system-metrics v0.9.2 h1:/tvTKOt55EerU0WwGFoDhBlyWLgxyv7d8xCbny0bciw= github.com/elastic/elastic-agent-system-metrics v0.9.2/go.mod h1:VfJnKw4Jqrd9ddljXCwaGKJgN+7ADyyGk089NaXVsf0= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= @@ -595,8 +595,6 @@ github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/u github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/mito v1.11.0 h1:thk9uxsTuTFeihMf3I6WLIeZyrBLQYuisWRYRUZl6Ec= github.com/elastic/mito v1.11.0/go.mod h1:J+wCf4HccW2YoSFmZMGu+d06gN+WmnIlj5ehBqine74= -github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= -github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 h1:FzA0/n4iMt8ojGDGRoiFPSHFvvdVIvxOxyLtiFnrLBM= @@ -1821,7 +1819,6 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1960,7 +1957,6 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2113,7 +2109,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2125,7 +2120,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 8fa3678e0427..08221801c5ca 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -125,6 +125,7 @@ type beatConfig struct { BufferConfig *config.C `config:"http.buffer"` Path paths.Path `config:"path"` Logging *config.C `config:"logging"` + EventLogging *config.C `config:"logging.event_data"` MetricLogging *config.C `config:"logging.metrics"` Keystore *config.C `config:"keystore"` Instrumentation instrumentation.Config `config:"instrumentation"` @@ -808,7 +809,7 @@ func (b *Beat) configure(settings Settings) error { return fmt.Errorf("error setting timestamp precision: %w", err) } - if err := configure.Logging(b.Info.Beat, b.Config.Logging); err != nil { + if err := configure.LoggingWithTypedOutputs(b.Info.Beat, b.Config.Logging, b.Config.EventLogging, "log.type", "event"); err != nil { return fmt.Errorf("error initializing logging: %w", err) } diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 4ba73c1b60db..c9bb53440460 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -293,3 +293,79 @@ Below are some samples: `2017-12-17T18:54:16.242-0500 INFO [example] logp/core_test.go:16 some message` `2017-12-17T18:54:16.242-0500 INFO [example] logp/core_test.go:19 some message {"x": 1}` + +ifndef::serverless[] +[float] +=== Configuration options for event_data logger + +Some outputs will log raw events on errors like indexing errors in the +Elasticsearch output, to prevent logging raw events (that may contain +sensitive information) together with other log messages, a different +log file, only for log entries containing raw events, is used. It will +use the same level, selectors and all other configurations from the +default logger, but it will have it's own file configuration. + +Having a different log file for raw events also prevents event data +from drowning out the regular log files. + +IMPORTANT: No matter the default logger output configuration, raw events +will **always** be logged to a file configured by `logging.event_data.files`. + +[float] +==== `logging.event_data.files.path` + +The directory that log files are written to. The default is the logs path. See +the <> section for details. + +[float] +==== `logging.event_data.files.name` + +The name of the file that logs are written to. The default is '{beatname_lc}'-event-data. + +[float] +==== `logging.event_data.files.rotateeverybytes` + +The maximum size of a log file. If the limit is reached, a new log file is +generated. The default size limit is 5242880 (5 MB). + +[float] +==== `logging.event_data.files.keepfiles` + +The number of most recent rotated log files to keep on disk. Older files are +deleted during log rotation. The default value is 2. The `keepfiles` options has +to be in the range of 2 to 1024 files. + +[float] +==== `logging.event_data.files.permissions` + +The permissions mask to apply when rotating log files. The default value is +0600. The `permissions` option must be a valid Unix-style file permissions mask +expressed in octal notation. In Go, numbers in octal notation must start with +'0'. + +The most permissive mask allowed is 0640. If a higher permissions mask is +specified via this setting, it will be subject to an umask of 0027. + +This option is not supported on Windows. + +Examples: + +* 0640: give read and write access to the file owner, and read access to members of the group associated with the file. +* 0600: give read and write access to the file owner, and no access to all others. + +[float] +==== `logging.event_data.files.interval` + +Enable log file rotation on time intervals in addition to size-based rotation. +Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h +are boundary-aligned with minutes, hours, days, weeks, months, and years as +reported by the local system clock. All other intervals are calculated from the +unix epoch. Defaults to disabled. + +[float] +==== `logging.event_data.files.rotateonstartup` + +If the log file already exists on startup, immediately rotate it and start +writing to a new file instead of appending to the existing one. Defaults to +false. +endif::serverless[] diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 504aac710af3..a5d8ea092209 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -417,17 +417,17 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat encodedEvent := data[i].EncodedEvent.(*encodedEvent) if encodedEvent.deadLetter { stats.nonIndexable++ - client.log.Errorf("Can't deliver to dead letter index event (status=%v). Enable debug logs to view the event and cause.", status) - client.log.Debugf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg) + client.log.Errorf("Can't deliver to dead letter index event (status=%v). Look at the event log to view the event and cause.", status) + client.log.Debugw(fmt.Sprintf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg), "log.type", "event") // poison pill - this will clog the pipeline if the underlying failure is non transient. } else if client.deadLetterIndex != "" { - client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Enable debug logs to view the event and cause.", status) - client.log.Debugf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg) + client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Look at the event log to view the event and cause.", status) + client.log.Debugw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg), "log.type", "event") client.setDeadLetter(encodedEvent, status, string(msg)) } else { // drop stats.nonIndexable++ - client.log.Warnf("Cannot index event (status=%v): dropping event! Enable debug logs to view the event and cause.", status) - client.log.Debugf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg) + client.log.Warnf("Cannot index event (status=%v): dropping event! Look at the event log to view the event and cause.", status) + client.log.Debugw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg), "log.type", "event") continue } } diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index d14bd99d69ad..1294deb62c40 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -19,6 +19,7 @@ package fileout import ( "context" + "fmt" "os" "path/filepath" "time" @@ -132,7 +133,8 @@ func (out *fileOutput) Publish(_ context.Context, batch publisher.Batch) error { } else { out.log.Warnf("Failed to serialize the event: %+v", err) } - out.log.Debugf("Failed event: %v", event) + out.log.Debug("Failed event logged to event log file") + out.log.Debugw(fmt.Sprintf("Failed event: %v", event), "log.type", "event") dropped++ continue diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 24bbc61145d4..1b7b727ca141 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -228,7 +228,8 @@ func (c *client) getEventMessage(data *publisher.Event) (*message, error) { serializedEvent, err := c.codec.Encode(c.index, event) if err != nil { if c.log.IsDebug() { - c.log.Debugf("failed event: %v", event) + c.log.Debug("failed event logged to event log file") + c.log.Debugw(fmt.Sprintf("failed event: %v", event), "log.type", "event") } return nil, err } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 5a299749aac8..cce46cbaf2d1 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -20,6 +20,7 @@ package redis import ( "context" "errors" + "fmt" "regexp" "strconv" "strings" @@ -319,8 +320,8 @@ func serializeEvents( for _, d := range data { serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { - log.Errorf("Encoding event failed with error: %+v", err) - log.Debugf("Failed event: %v", d.Content) + log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) + log.Debugw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") goto failLoop } @@ -337,8 +338,8 @@ failLoop: for _, d := range rest { serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { - log.Errorf("Encoding event failed with error: %+v", err) - log.Debugf("Failed event: %v", d.Content) + log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) + log.Debugw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") i++ continue } From 35ff4e3aa572ae2ae0e8a3d348bc2f788829fb0d Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 8 Apr 2024 17:48:57 +0200 Subject: [PATCH 02/26] Add integration test --- .../tests/integration/event_log_file_test.go | 131 ++++++++++++++++++ libbeat/tests/integration/framework.go | 10 +- 2 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 filebeat/tests/integration/event_log_file_test.go diff --git a/filebeat/tests/integration/event_log_file_test.go b/filebeat/tests/integration/event_log_file_test.go new file mode 100644 index 000000000000..7dd7105f6d1d --- /dev/null +++ b/filebeat/tests/integration/event_log_file_test.go @@ -0,0 +1,131 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +var eventsLogFileCfg = ` +filebeat.inputs: + - type: filestream + id: filestream-input-id + enabled: true + parsers: + - ndjson: + target: "" + overwrite_keys: true + expand_keys: true + add_error_key: true + ignore_decoding_error: false + paths: + - %s + +output: + elasticsearch: + hosts: + - localhost:9200 + protocol: http + username: admin + password: testing + +logging: + level: debug + event_data: + files: + name: filebeat-my-event-log +` + +func TestEventsLoggerESOutput(t *testing.T) { + // First things first, ensure ES is running and we can connect to it. + // If ES is not running, the test will timeout and the only way to know + // what caused it is going through Filebeat's logs. + integration.EnsureESIsRunning(t) + + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + + logFilePath := filepath.Join(filebeat.TempDir(), "log.log") + filebeat.WriteConfigFile(fmt.Sprintf(eventsLogFileCfg, logFilePath)) + + logFile, err := os.Create(logFilePath) + if err != nil { + t.Fatalf("could not create file '%s': %s", logFilePath, err) + } + + _, _ = logFile.WriteString(` +{"message":"foo bar","int":10,"string":"str"} +{"message":"another message","int":20,"string":"str2"} +{"message":"index failure","int":"not a number","string":10} +{"message":"second index failure","int":"not a number","string":10} +`) + if err := logFile.Sync(); err != nil { + t.Fatalf("could not sync log file '%s': %s", logFilePath, err) + } + if err := logFile.Close(); err != nil { + t.Fatalf("could not close log file '%s': %s", logFilePath, err) + } + + filebeat.Start() + + // Wait for a log entry that indicates an entry in the events + // logger file. + msg := "Cannot index event (status=400)" + require.Eventually(t, func() bool { + return filebeat.LogContains(msg) + }, time.Minute, 100*time.Millisecond, + fmt.Sprintf("String '%s' not found on Filebeat logs", msg)) + + // The glob here matches the configured value for the filename + glob := filepath.Join(filebeat.TempDir(), "filebeat-my-event-log*.ndjson") + files, err := filepath.Glob(glob) + if err != nil { + t.Fatalf("could not read files matching glob '%s': %s", glob, err) + } + if len(files) != 1 { + t.Fatalf("there must be only one file matching the glob '%s', found: %s", glob, files) + } + + eventsLogFile := files[0] + data, err := os.ReadFile(eventsLogFile) + if err != nil { + t.Fatalf("could not read '%s': %s", eventsLogFile, err) + } + + strData := string(data) + eventMsg := "not a number" + if !strings.Contains(strData, eventMsg) { + t.Errorf("expecting to find '%s' on '%s'", eventMsg, eventsLogFile) + t.Errorf("Contents:\n%s", strData) + t.FailNow() + } +} diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index 229d855b9fad..444a93daa38a 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -398,7 +398,11 @@ func (b *BeatProc) WriteConfigFile(cfg string) { // when the test ends. func (b *BeatProc) openLogFile() *os.File { t := b.t - glob := fmt.Sprintf("%s-*.ndjson", filepath.Join(b.tempDir, b.beatName)) + // Beats can produce two different log files, to make sure we're + // reading the normal one we add the year to the glob. The default + // log file name looks like: filebeat-20240116.ndjson + year := time.Now().Year() + glob := fmt.Sprintf("%s-%d*.ndjson", filepath.Join(b.tempDir, b.beatName), year) files, err := filepath.Glob(glob) if err != nil { t.Fatalf("could not expand log file glob: %s", err) @@ -484,9 +488,9 @@ func EnsureESIsRunning(t *testing.T) { resp, err := http.DefaultClient.Do(req) if err != nil { // If you're reading this message, you probably forgot to start ES - // run `mage compose:Up` from Filebeat's folder to start all + // run `mage docker:composeUp` from Filebeat's folder to start all // containers required for integration tests - t.Fatalf("cannot execute HTTP request to ES: '%s', check to make sure ES is running (mage compose:Up)", err) + t.Fatalf("cannot execute HTTP request to ES: '%s', check to make sure ES is running (mage docker:composeUp)", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { From 03126eeb45d67a877a0bb2d218cb653ce662a08f Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 10 Apr 2024 09:29:23 +0200 Subject: [PATCH 03/26] Update elastic-agent-libs Update elastic-agent-libs and update code to accommodate for the breaking changes. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1bbbdc63592f..d46f50748eb4 100644 --- a/go.mod +++ b/go.mod @@ -433,4 +433,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408152242-d3dd5568916d +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240409174302-6a5335a2bd6d diff --git a/go.sum b/go.sum index e6e21635b920..600b0180735f 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408152242-d3dd5568916d h1:gxEYMj2FDnIB3rTWBPPzuoC2owMUuli+oi5/6UQXSrI= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408152242-d3dd5568916d/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240409174302-6a5335a2bd6d h1:5MEIRy49Kigthugon6w0anOWVq8i1Tp9BqjKA55OVos= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240409174302-6a5335a2bd6d/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= From fbee5888b325923e29d5d746299936dfa92ded10 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 10 Apr 2024 09:32:05 +0200 Subject: [PATCH 04/26] Fix log level on output clients Fix the log level for event data on all outputs. Now the log level from the message containing the evnt matches the one that does not contain it. --- filebeat/tests/integration/event_log_file_test.go | 2 +- libbeat/outputs/elasticsearch/client.go | 5 +++-- libbeat/outputs/redis/client.go | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/filebeat/tests/integration/event_log_file_test.go b/filebeat/tests/integration/event_log_file_test.go index 7dd7105f6d1d..134830ce666e 100644 --- a/filebeat/tests/integration/event_log_file_test.go +++ b/filebeat/tests/integration/event_log_file_test.go @@ -56,7 +56,7 @@ output: password: testing logging: - level: debug + level: info event_data: files: name: filebeat-my-event-log diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index a5d8ea092209..8a41b67c02b4 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -418,16 +418,17 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat if encodedEvent.deadLetter { stats.nonIndexable++ client.log.Errorf("Can't deliver to dead letter index event (status=%v). Look at the event log to view the event and cause.", status) - client.log.Debugw(fmt.Sprintf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg), "log.type", "event") + client.log.Errorw(fmt.Sprintf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg), "log.type", "event") // poison pill - this will clog the pipeline if the underlying failure is non transient. } else if client.deadLetterIndex != "" { client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Look at the event log to view the event and cause.", status) client.log.Debugw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg), "log.type", "event") client.setDeadLetter(encodedEvent, status, string(msg)) + } else { // drop stats.nonIndexable++ client.log.Warnf("Cannot index event (status=%v): dropping event! Look at the event log to view the event and cause.", status) - client.log.Debugw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg), "log.type", "event") + client.log.Warnw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg), "log.type", "event") continue } } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index cce46cbaf2d1..298730c6401f 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -321,7 +321,7 @@ func serializeEvents( serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) - log.Debugw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") + log.Errorw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") goto failLoop } @@ -339,7 +339,7 @@ failLoop: serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) - log.Debugw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") + log.Errorw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") i++ continue } From 4cf5b608437cd512bb84624b109c06f3d5cb778c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 11 Apr 2024 13:34:16 +0200 Subject: [PATCH 05/26] Update elastic-agent-libs --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d46f50748eb4..42205531fa4a 100644 --- a/go.mod +++ b/go.mod @@ -433,4 +433,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240409174302-6a5335a2bd6d +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 diff --git a/go.sum b/go.sum index 600b0180735f..15cbe5ee3160 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240409174302-6a5335a2bd6d h1:5MEIRy49Kigthugon6w0anOWVq8i1Tp9BqjKA55OVos= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240409174302-6a5335a2bd6d/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 h1:udY8euEzgPnTFEA4NUUSr+cur2/DCZWteMGiLwYKbFI= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= From ba98f511795eb21d44d19cb2499ede357bc65123 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 11 Apr 2024 15:42:23 +0200 Subject: [PATCH 06/26] Accommodate for breaking changes in elastic-agent-libs --- metricbeat/helper/dialer/dialer_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/helper/dialer/dialer_windows.go b/metricbeat/helper/dialer/dialer_windows.go index 94e383e4cb6c..a91b5a3868c2 100644 --- a/metricbeat/helper/dialer/dialer_windows.go +++ b/metricbeat/helper/dialer/dialer_windows.go @@ -61,7 +61,7 @@ func (t *NpipeDialerBuilder) String() string { func (t *NpipeDialerBuilder) Make(timeout time.Duration) (transport.Dialer, error) { to := timeout return transport.DialerFunc( - func(_ context.Context, _ string, _ string) (net.Conn, error) { + func(_ context.Context, _, _ string) (net.Conn, error) { return winio.DialPipe( strings.TrimSuffix(npipe.TransformString(t.Path), "/"), &to, From 0565669b9aeff08f75a55f3a03a6bac921cf6625 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 23 Apr 2024 09:10:21 -0400 Subject: [PATCH 07/26] Update elastic-agent-libs and fix log level --- go.mod | 2 +- go.sum | 4 ++-- libbeat/outputs/elasticsearch/client.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 42205531fa4a..073a8fcca7be 100644 --- a/go.mod +++ b/go.mod @@ -433,4 +433,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240423125757-a73eab058a9f diff --git a/go.sum b/go.sum index 15cbe5ee3160..d86c08d92ae8 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 h1:udY8euEzgPnTFEA4NUUSr+cur2/DCZWteMGiLwYKbFI= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240423125757-a73eab058a9f h1:RucOKuoSI53KjEA+BBmlDVnqRc98z+YeDXBIcm2VZ+w= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240423125757-a73eab058a9f/go.mod h1:h+izqYohdAEdb3OywTLvrrw7Yon5jiiP8N/F5NgDgBQ= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 8a41b67c02b4..06cca209da60 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -422,7 +422,7 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat // poison pill - this will clog the pipeline if the underlying failure is non transient. } else if client.deadLetterIndex != "" { client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Look at the event log to view the event and cause.", status) - client.log.Debugw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg), "log.type", "event") + client.log.Warnw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg), "log.type", "event") client.setDeadLetter(encodedEvent, status, string(msg)) } else { // drop From 17192fe7fb966b3ee10fcb6f6134ea59dcaf0789 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 23 Apr 2024 09:52:22 -0400 Subject: [PATCH 08/26] Update reference configuration files and NOTICE.txt --- NOTICE.txt | 36 ++------------ auditbeat/auditbeat.reference.yml | 48 +++++++++++++++++++ filebeat/filebeat.reference.yml | 48 +++++++++++++++++++ heartbeat/heartbeat.reference.yml | 48 +++++++++++++++++++ .../_meta/config/logging.reference.yml.tmpl | 48 +++++++++++++++++++ metricbeat/metricbeat.reference.yml | 48 +++++++++++++++++++ packetbeat/packetbeat.reference.yml | 48 +++++++++++++++++++ winlogbeat/winlogbeat.reference.yml | 48 +++++++++++++++++++ x-pack/auditbeat/auditbeat.reference.yml | 48 +++++++++++++++++++ x-pack/filebeat/filebeat.reference.yml | 48 +++++++++++++++++++ .../functionbeat/functionbeat.reference.yml | 48 +++++++++++++++++++ x-pack/heartbeat/heartbeat.reference.yml | 48 +++++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 48 +++++++++++++++++++ x-pack/osquerybeat/osquerybeat.reference.yml | 48 +++++++++++++++++++ x-pack/packetbeat/packetbeat.reference.yml | 48 +++++++++++++++++++ x-pack/winlogbeat/winlogbeat.reference.yml | 48 +++++++++++++++++++ 16 files changed, 723 insertions(+), 33 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index aad81b518acc..06c77e25d441 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12968,12 +12968,12 @@ SOFTWARE -------------------------------------------------------------------------------- -Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.8 +Dependency : github.com/belimawr/elastic-agent-libs +Version: v0.2.9-0.20240423125757-a73eab058a9f Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/belimawr/elastic-agent-libs@v0.2.9-0.20240423125757-a73eab058a9f/LICENSE: Apache License Version 2.0, January 2004 @@ -38752,36 +38752,6 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0 limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/elastic/pkcs8 -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/elastic/pkcs8@v1.0.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 youmark - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -------------------------------------------------------------------------------- Dependency : github.com/elazarl/goproxy Version: v0.0.0-20180725130230-947c36da3153 diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index e9a23ca6ac0d..8f684d78d038 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1549,6 +1549,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/auditbeat + + # The name of the files where the logs are written to. + #name: auditbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index bc5ebdc3d151..ff308012ed1f 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2640,6 +2640,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/filebeat + + # The name of the files where the logs are written to. + #name: filebeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 7407d213748a..3632ce12bbd3 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1636,6 +1636,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/heartbeat + + # The name of the files where the logs are written to. + #name: heartbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index 660bbb73a02a..e0921d0ff827 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -67,3 +67,51 @@ logging.files: # Rotate existing logs on startup rather than appending them to the existing # file. Defaults to true. # rotateonstartup: true + +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}}-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 2538bef77d35..985e4795910c 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2421,6 +2421,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/metricbeat + + # The name of the files where the logs are written to. + #name: metricbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 6eaee863da02..7041e79ea748 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -2015,6 +2015,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/packetbeat + + # The name of the files where the logs are written to. + #name: packetbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index a6a042f4aba4..47a61d36faf9 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1426,6 +1426,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/winlogbeat + + # The name of the files where the logs are written to. + #name: winlogbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index a0352454e093..438ae307b80d 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1605,6 +1605,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/auditbeat + + # The name of the files where the logs are written to. + #name: auditbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 0c7cab1acb12..9830e468e647 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4567,6 +4567,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/filebeat + + # The name of the files where the logs are written to. + #name: filebeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 2284fedbcce9..a3df78c4e2df 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1264,6 +1264,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/functionbeat + + # The name of the files where the logs are written to. + #name: functionbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Functionbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 7407d213748a..3632ce12bbd3 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1636,6 +1636,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/heartbeat + + # The name of the files where the logs are written to. + #name: heartbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 6877f2b45346..1d2957306db9 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2982,6 +2982,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/metricbeat + + # The name of the files where the logs are written to. + #name: metricbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 0c28af891443..7e8ccd5e8424 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -983,6 +983,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/osquerybeat + + # The name of the files where the logs are written to. + #name: osquerybeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Osquerybeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 6eaee863da02..7041e79ea748 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -2015,6 +2015,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/packetbeat + + # The name of the files where the logs are written to. + #name: packetbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 5bc8f774e03c..96e912ea41ca 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1428,6 +1428,54 @@ logging.files: # file. Defaults to true. # rotateonstartup: true +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `logging.event_data.files`. + +# logging.event_data: +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.event_data.to_files: true +#logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/winlogbeat + + # The name of the files where the logs are written to. + #name: winlogbeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The From 2d499f2842f45b0092b8116a73c7627e0845b13a Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 23 Apr 2024 10:02:01 -0400 Subject: [PATCH 09/26] Add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1c0ead5ceae7..3597d8acbacf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -197,6 +197,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Introduce log message for not supported annotations for Hints based autodiscover {pull}38213[38213] - Add persistent volume claim name to volume if available {pull}38839[38839] +- Raw events are now logged to a different file, this prevents potentially sensitive information from leaking into log files {pull}38767[38767] *Auditbeat* From d5781ace84f0c94f56f663e173a07e759781a5fc Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 24 Apr 2024 09:28:44 -0400 Subject: [PATCH 10/26] Update elastic-agent-libs --- go.mod | 3 ++- go.sum | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 073a8fcca7be..a7b74bbcdf76 100644 --- a/go.mod +++ b/go.mod @@ -289,6 +289,7 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.5.0 // indirect github.com/elastic/go-windows v1.0.1 // indirect + github.com/elastic/pkcs8 v1.0.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fearful-symmetry/gomsr v0.0.1 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect @@ -433,4 +434,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240423125757-a73eab058a9f +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 diff --git a/go.sum b/go.sum index d86c08d92ae8..448e91fdbfae 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240423125757-a73eab058a9f h1:RucOKuoSI53KjEA+BBmlDVnqRc98z+YeDXBIcm2VZ+w= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240423125757-a73eab058a9f/go.mod h1:h+izqYohdAEdb3OywTLvrrw7Yon5jiiP8N/F5NgDgBQ= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 h1:5HrVQGFDIdS4Mnta6rVSLZWDLOZS6g3xepLLN2XP/W0= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= @@ -595,6 +595,8 @@ github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/u github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/mito v1.11.0 h1:thk9uxsTuTFeihMf3I6WLIeZyrBLQYuisWRYRUZl6Ec= github.com/elastic/mito v1.11.0/go.mod h1:J+wCf4HccW2YoSFmZMGu+d06gN+WmnIlj5ehBqine74= +github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= +github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 h1:FzA0/n4iMt8ojGDGRoiFPSHFvvdVIvxOxyLtiFnrLBM= @@ -1819,6 +1821,7 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1957,6 +1960,7 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2109,6 +2113,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2120,6 +2125,7 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From d8ff8a8e73e18dee4ddb7c9c8f2708a9e40eee49 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 24 Apr 2024 14:46:57 -0400 Subject: [PATCH 11/26] Update Agentbeat spec file --- x-pack/agentbeat/agentbeat.spec.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/x-pack/agentbeat/agentbeat.spec.yml b/x-pack/agentbeat/agentbeat.spec.yml index 045188513b09..7e859cc9362a 100644 --- a/x-pack/agentbeat/agentbeat.spec.yml +++ b/x-pack/agentbeat/agentbeat.spec.yml @@ -39,6 +39,10 @@ inputs: - "gc_percent=${AUDITBEAT_GOGC:100}" - "-E" - "auditbeat.config.modules.enabled=false" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: audit/file_integrity description: "Audit File Integrity" platforms: *platforms @@ -77,6 +81,10 @@ inputs: - "gc_percent=${FILEBEAT_GOGC:100}" - "-E" - "filebeat.config.modules.enabled=false" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: aws-s3 description: "AWS S3" platforms: *platforms @@ -261,6 +269,10 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${HEARTBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: synthetics/http description: "Synthetics HTTP Monitor" platforms: *platforms @@ -304,6 +316,10 @@ inputs: - "gc_percent=${METRICBEAT_GOGC:100}" - "-E" - "metricbeat.config.modules.enabled=false" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: docker/metrics description: "Docker metrics" platforms: *platforms @@ -540,6 +556,10 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${OSQUERYBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: packet description: "Packet Capture" platforms: *platforms @@ -566,3 +586,7 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${PACKETBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" From db840653fd92c00cdc9c59625ead87370ebc0b1e Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 25 Apr 2024 12:03:37 -0400 Subject: [PATCH 12/26] Update elastic-agent-libs to v0.9.6 --- NOTICE.txt | 36 +++++++++++++++++++++++++++++++++--- go.mod | 4 +--- go.sum | 4 ++-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 06c77e25d441..7b1296145187 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12968,12 +12968,12 @@ SOFTWARE -------------------------------------------------------------------------------- -Dependency : github.com/belimawr/elastic-agent-libs -Version: v0.2.9-0.20240423125757-a73eab058a9f +Dependency : github.com/elastic/elastic-agent-libs +Version: v0.9.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/belimawr/elastic-agent-libs@v0.2.9-0.20240423125757-a73eab058a9f/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.6/LICENSE: Apache License Version 2.0, January 2004 @@ -38752,6 +38752,36 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/pkcs8 +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/pkcs8@v1.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + -------------------------------------------------------------------------------- Dependency : github.com/elazarl/goproxy Version: v0.0.0-20180725130230-947c36da3153 diff --git a/go.mod b/go.mod index a7b74bbcdf76..9d404f49e055 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.8 + github.com/elastic/elastic-agent-libs v0.9.6 github.com/elastic/elastic-agent-system-metrics v0.9.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.11.0 @@ -433,5 +433,3 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible - -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 diff --git a/go.sum b/go.sum index 448e91fdbfae..29abb980c208 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,6 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 h1:5HrVQGFDIdS4Mnta6rVSLZWDLOZS6g3xepLLN2XP/W0= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= @@ -556,6 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= +github.com/elastic/elastic-agent-libs v0.9.6 h1:3paTd2JVkxTHH8rnlVZVrTLJgacR2l8jFr+NYHHCNio= +github.com/elastic/elastic-agent-libs v0.9.6/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/elastic/elastic-agent-system-metrics v0.9.2 h1:/tvTKOt55EerU0WwGFoDhBlyWLgxyv7d8xCbny0bciw= github.com/elastic/elastic-agent-system-metrics v0.9.2/go.mod h1:VfJnKw4Jqrd9ddljXCwaGKJgN+7ADyyGk089NaXVsf0= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= From f4025d8960a3541f3ae0d4bd32ee5e07b78cdc9b Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 25 Apr 2024 15:06:31 -0400 Subject: [PATCH 13/26] Implement PR review suggestions --- x-pack/filebeat/input/cel/input.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index f0795377b0ec..9c07f4368b8e 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -1184,7 +1184,7 @@ func cloneMap(dst, src mapstr.M) { // walkMap walks to all ends of the provided path in m and applies fn to the // final element of each walk. Nested arrays are not handled. func walkMap(m mapstr.M, path string, fn func(parent mapstr.M, key string)) { - key, rest, more := strings.Cut(path, ".") + key, rest, more := strings.Cut(path, ".") //nolint:typecheck // reset is used in recursive calls. v, ok := m[key] if !ok { return From 15cf143fa84f7f1c8c2068d0b7e89b65e9437761 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 6 May 2024 15:10:08 -0400 Subject: [PATCH 14/26] Update elastic-agent-libs and use the defined constants --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- libbeat/cmd/instance/beat.go | 2 +- libbeat/outputs/elasticsearch/client.go | 6 +++--- libbeat/outputs/fileout/file.go | 2 +- libbeat/outputs/kafka/client.go | 2 +- libbeat/outputs/redis/client.go | 4 ++-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 7b1296145187..d54d6fa766e2 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.6 +Version: v0.9.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.7/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 9d404f49e055..c45ea6852438 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.6 + github.com/elastic/elastic-agent-libs v0.9.7 github.com/elastic/elastic-agent-system-metrics v0.9.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.11.0 diff --git a/go.sum b/go.sum index 29abb980c208..44e7a496ec69 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.6 h1:3paTd2JVkxTHH8rnlVZVrTLJgacR2l8jFr+NYHHCNio= -github.com/elastic/elastic-agent-libs v0.9.6/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= +github.com/elastic/elastic-agent-libs v0.9.7 h1:LZdfxbq724Y1zAdE3COp+OIPwU8SquOCLIXpI/twcdQ= +github.com/elastic/elastic-agent-libs v0.9.7/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/elastic/elastic-agent-system-metrics v0.9.2 h1:/tvTKOt55EerU0WwGFoDhBlyWLgxyv7d8xCbny0bciw= github.com/elastic/elastic-agent-system-metrics v0.9.2/go.mod h1:VfJnKw4Jqrd9ddljXCwaGKJgN+7ADyyGk089NaXVsf0= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 08221801c5ca..c3932c0b867f 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -809,7 +809,7 @@ func (b *Beat) configure(settings Settings) error { return fmt.Errorf("error setting timestamp precision: %w", err) } - if err := configure.LoggingWithTypedOutputs(b.Info.Beat, b.Config.Logging, b.Config.EventLogging, "log.type", "event"); err != nil { + if err := configure.LoggingWithTypedOutputs(b.Info.Beat, b.Config.Logging, b.Config.EventLogging, logp.TypeKey, logp.EventType); err != nil { return fmt.Errorf("error initializing logging: %w", err) } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 06cca209da60..0892ce401731 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -418,17 +418,17 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat if encodedEvent.deadLetter { stats.nonIndexable++ client.log.Errorf("Can't deliver to dead letter index event (status=%v). Look at the event log to view the event and cause.", status) - client.log.Errorw(fmt.Sprintf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg), "log.type", "event") + client.log.Errorw(fmt.Sprintf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg), logp.TypeKey, logp.EventType) // poison pill - this will clog the pipeline if the underlying failure is non transient. } else if client.deadLetterIndex != "" { client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Look at the event log to view the event and cause.", status) - client.log.Warnw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg), "log.type", "event") + client.log.Warnw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg), logp.TypeKey, logp.EventType) client.setDeadLetter(encodedEvent, status, string(msg)) } else { // drop stats.nonIndexable++ client.log.Warnf("Cannot index event (status=%v): dropping event! Look at the event log to view the event and cause.", status) - client.log.Warnw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg), "log.type", "event") + client.log.Warnw(fmt.Sprintf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg), logp.TypeKey, logp.EventType) continue } } diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 1294deb62c40..87b50f62c1af 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -134,7 +134,7 @@ func (out *fileOutput) Publish(_ context.Context, batch publisher.Batch) error { out.log.Warnf("Failed to serialize the event: %+v", err) } out.log.Debug("Failed event logged to event log file") - out.log.Debugw(fmt.Sprintf("Failed event: %v", event), "log.type", "event") + out.log.Debugw(fmt.Sprintf("Failed event: %v", event), logp.TypeKey, logp.EventType) dropped++ continue diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 1b7b727ca141..08484f017bb4 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -229,7 +229,7 @@ func (c *client) getEventMessage(data *publisher.Event) (*message, error) { if err != nil { if c.log.IsDebug() { c.log.Debug("failed event logged to event log file") - c.log.Debugw(fmt.Sprintf("failed event: %v", event), "log.type", "event") + c.log.Debugw(fmt.Sprintf("failed event: %v", event), logp.TypeKey, logp.EventType) } return nil, err } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 298730c6401f..c5d8590c759c 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -321,7 +321,7 @@ func serializeEvents( serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) - log.Errorw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") + log.Errorw(fmt.Sprintf("Failed event: %v", d.Content), logp.TypeKey, logp.EventType) goto failLoop } @@ -339,7 +339,7 @@ failLoop: serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) - log.Errorw(fmt.Sprintf("Failed event: %v", d.Content), "log.type", "event") + log.Errorw(fmt.Sprintf("Failed event: %v", d.Content), logp.TypeKey, logp.EventType) i++ continue } From 7a4d788ce66f496860c277f6f43ae32ab432ed45 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 8 May 2024 09:50:32 -0400 Subject: [PATCH 15/26] Remove changes from merge conflicts --- metricbeat/helper/dialer/dialer_windows.go | 2 +- x-pack/filebeat/input/cel/input.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/helper/dialer/dialer_windows.go b/metricbeat/helper/dialer/dialer_windows.go index a91b5a3868c2..94e383e4cb6c 100644 --- a/metricbeat/helper/dialer/dialer_windows.go +++ b/metricbeat/helper/dialer/dialer_windows.go @@ -61,7 +61,7 @@ func (t *NpipeDialerBuilder) String() string { func (t *NpipeDialerBuilder) Make(timeout time.Duration) (transport.Dialer, error) { to := timeout return transport.DialerFunc( - func(_ context.Context, _, _ string) (net.Conn, error) { + func(_ context.Context, _ string, _ string) (net.Conn, error) { return winio.DialPipe( strings.TrimSuffix(npipe.TransformString(t.Path), "/"), &to, diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 9c07f4368b8e..f0795377b0ec 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -1184,7 +1184,7 @@ func cloneMap(dst, src mapstr.M) { // walkMap walks to all ends of the provided path in m and applies fn to the // final element of each walk. Nested arrays are not handled. func walkMap(m mapstr.M, path string, fn func(parent mapstr.M, key string)) { - key, rest, more := strings.Cut(path, ".") //nolint:typecheck // reset is used in recursive calls. + key, rest, more := strings.Cut(path, ".") v, ok := m[key] if !ok { return From 8255ed9a8d322beca577c5b892c9afbf8d0df229 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 8 May 2024 10:06:44 -0400 Subject: [PATCH 16/26] Use event logger instead of "trace level" Use the event logger instead of "trace level" for debug logs containing events, both under Elastic-Agent and standalone Beat. --- libbeat/processors/actions/append.go | 6 ++---- libbeat/processors/actions/copy_fields.go | 6 ++---- .../processors/actions/decode_base64_field.go | 6 ++---- .../actions/decompress_gzip_field.go | 6 ++---- libbeat/processors/actions/rename.go | 6 ++---- libbeat/processors/actions/replace.go | 6 ++---- libbeat/processors/urldecode/urldecode.go | 6 ++---- libbeat/publisher/processing/processors.go | 19 ++++++++----------- 8 files changed, 22 insertions(+), 39 deletions(-) diff --git a/libbeat/processors/actions/append.go b/libbeat/processors/actions/append.go index 1bf2caad45fb..fd15eeb2e587 100644 --- a/libbeat/processors/actions/append.go +++ b/libbeat/processors/actions/append.go @@ -21,7 +21,6 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -82,9 +81,8 @@ func (f *appendProcessor) Run(event *beat.Event) (*beat.Event, error) { err := f.appendValues(f.config.TargetField, f.config.Fields, f.config.Values, event) if err != nil { errMsg := fmt.Errorf("failed to append fields in append processor: %w", err) - if management.TraceLevelEnabled() { - f.logger.Debug(errMsg.Error()) - } + f.logger.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType) + if f.config.FailOnError { event = backup if _, err := event.PutValue("error.message", errMsg.Error()); err != nil { diff --git a/libbeat/processors/actions/copy_fields.go b/libbeat/processors/actions/copy_fields.go index 0f4fab309a36..f0d6cbe775aa 100644 --- a/libbeat/processors/actions/copy_fields.go +++ b/libbeat/processors/actions/copy_fields.go @@ -22,7 +22,6 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -79,9 +78,8 @@ func (f *copyFields) Run(event *beat.Event) (*beat.Event, error) { err := f.copyField(field.From, field.To, event) if err != nil { errMsg := fmt.Errorf("Failed to copy fields in copy_fields processor: %w", err) - if management.TraceLevelEnabled() { - f.logger.Debug(errMsg.Error()) - } + f.logger.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType) + if f.config.FailOnError { event = backup _, _ = event.PutValue("error.message", errMsg.Error()) diff --git a/libbeat/processors/actions/decode_base64_field.go b/libbeat/processors/actions/decode_base64_field.go index c45166beb111..3ec5e0a8d7a3 100644 --- a/libbeat/processors/actions/decode_base64_field.go +++ b/libbeat/processors/actions/decode_base64_field.go @@ -24,7 +24,6 @@ import ( "strings" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -84,9 +83,8 @@ func (f *decodeBase64Field) Run(event *beat.Event) (*beat.Event, error) { err := f.decodeField(event) if err != nil { errMsg := fmt.Errorf("failed to decode base64 fields in processor: %w", err) - if management.TraceLevelEnabled() { - f.log.Debug(errMsg.Error()) - } + f.log.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType) + if f.config.FailOnError { event = backup _, _ = event.PutValue("error.message", errMsg.Error()) diff --git a/libbeat/processors/actions/decompress_gzip_field.go b/libbeat/processors/actions/decompress_gzip_field.go index 8d463600c210..993dbf3e8210 100644 --- a/libbeat/processors/actions/decompress_gzip_field.go +++ b/libbeat/processors/actions/decompress_gzip_field.go @@ -25,7 +25,6 @@ import ( "io" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" conf "github.com/elastic/elastic-agent-libs/config" @@ -76,9 +75,8 @@ func (f *decompressGzipField) Run(event *beat.Event) (*beat.Event, error) { err := f.decompressGzipField(event) if err != nil { errMsg := fmt.Errorf("Failed to decompress field in decompress_gzip_field processor: %w", err) - if management.TraceLevelEnabled() { - f.log.Debug(errMsg.Error()) - } + f.log.Debugw(errMsg.Error(), logp.EventType, logp.TypeKey) + if f.config.FailOnError { event = backup _, _ = event.PutValue("error.message", errMsg.Error()) diff --git a/libbeat/processors/actions/rename.go b/libbeat/processors/actions/rename.go index 4c49174bf54e..7503127e1032 100644 --- a/libbeat/processors/actions/rename.go +++ b/libbeat/processors/actions/rename.go @@ -22,7 +22,6 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -84,9 +83,8 @@ func (f *renameFields) Run(event *beat.Event) (*beat.Event, error) { err := f.renameField(field.From, field.To, event) if err != nil { errMsg := fmt.Errorf("Failed to rename fields in processor: %w", err) - if management.TraceLevelEnabled() { - f.logger.Debug(errMsg.Error()) - } + f.logger.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType) + if f.config.FailOnError { event = backup _, _ = event.PutValue("error.message", errMsg.Error()) diff --git a/libbeat/processors/actions/replace.go b/libbeat/processors/actions/replace.go index df4aa03fc86e..b242b9f35796 100644 --- a/libbeat/processors/actions/replace.go +++ b/libbeat/processors/actions/replace.go @@ -23,7 +23,6 @@ import ( "regexp" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -86,9 +85,8 @@ func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) { err := f.replaceField(field.Field, field.Pattern, field.Replacement, event) if err != nil { errMsg := fmt.Errorf("Failed to replace fields in processor: %w", err) - if management.TraceLevelEnabled() { - f.log.Debug(errMsg.Error()) - } + f.log.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType) + if f.config.FailOnError { event = backup _, _ = event.PutValue("error.message", errMsg.Error()) diff --git a/libbeat/processors/urldecode/urldecode.go b/libbeat/processors/urldecode/urldecode.go index 59ed552e2ae1..c9aac0cdef1d 100644 --- a/libbeat/processors/urldecode/urldecode.go +++ b/libbeat/processors/urldecode/urldecode.go @@ -23,7 +23,6 @@ import ( "net/url" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -83,9 +82,8 @@ func (p *urlDecode) Run(event *beat.Event) (*beat.Event, error) { err := p.decodeField(field.From, field.To, event) if err != nil { errMsg := fmt.Errorf("failed to decode fields in urldecode processor: %w", err) - if management.TraceLevelEnabled() { - p.log.Debug(errMsg.Error()) - } + p.log.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType) + if p.config.FailOnError { event = backup _, _ = event.PutValue("error.message", errMsg.Error()) diff --git a/libbeat/publisher/processing/processors.go b/libbeat/publisher/processing/processors.go index 69fb5090e4ce..e90202401a71 100644 --- a/libbeat/publisher/processing/processors.go +++ b/libbeat/publisher/processing/processors.go @@ -27,7 +27,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/outputs/codec/json" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/elastic-agent-libs/logp" @@ -200,18 +199,16 @@ func debugPrintProcessor(info beat.Info, log *logp.Logger) *processorFn { EscapeHTML: false, }) return newProcessor("debugPrint", func(event *beat.Event) (*beat.Event, error) { - if management.TraceLevelEnabled() { - mux.Lock() - defer mux.Unlock() + mux.Lock() + defer mux.Unlock() - b, err := encoder.Encode(info.Beat, event) - if err != nil { - //nolint:nilerr // encoder failure is not considered an error by this processor [why not?] - return event, nil - } - - log.Debugf("Publish event: %s", b) + b, err := encoder.Encode(info.Beat, event) + if err != nil { + //nolint:nilerr // encoder failure is not considered an error by this processor [why not?] + return event, nil } + + log.Debugw(fmt.Sprintf("Publish event: %s", b), logp.TypeKey, logp.EventType) return event, nil }) } From 9fdac09181645b6f6720f67442371805382725c9 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 10 May 2024 08:48:08 -0400 Subject: [PATCH 17/26] Read event log file in python tests --- libbeat/tests/system/beat/beat.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index b8c4b3c43f11..bc1126402cd4 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -517,9 +517,10 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): if logfile is None: logfile = self.beat_name + "-" + self.today + ".ndjson" - print("logfile", logfile, self.working_dir) + logfile_path = os.path.join(self.working_dir, logfile) + print("logfile ", logfile_path) try: - with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: + with open(logfile_path, "r", encoding="utf_8") as f: for line in f: if is_regexp: if msg.search(line) is not None: @@ -529,6 +530,27 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): line = line.lower() if line.find(msg) >= 0: counter = counter + 1 + + # Event log file: + logfile = self.beat_name + "-events-data-" + self.today + ".ndjson" + logfile_path = os.path.join(self.working_dir, "logs", logfile) + print("event logfile", logfile_path) + try: + with open(logfile_path, "r", encoding="utf_8") as f: + for line in f: + if is_regexp: + if msg.search(line) is not None: + counter = counter + 1 + continue + if ignore_case: + line = line.lower() + if line.find(msg) >= 0: + counter = counter + 1 + except FileNotFoundError as e: + # The events log file is not always present, so we ignore + # if it does not exit + pass + except IOError as ioe: print(ioe) counter = -1 From fd2fb3eac164dc30a45e0c12930c810d7cafddb9 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 10 May 2024 12:53:50 -0400 Subject: [PATCH 18/26] Fix flaky python test Fix the flaky python test. Moving the file instead of truncating it seems to solve the problem. Maybe the write was not properly synced to disk. The advantage of moving the file is that if the test runs we can later inspect it. --- filebeat/tests/system/test_reload_inputs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/filebeat/tests/system/test_reload_inputs.py b/filebeat/tests/system/test_reload_inputs.py index 36c5b2eeedd2..bd1e19408942 100644 --- a/filebeat/tests/system/test_reload_inputs.py +++ b/filebeat/tests/system/test_reload_inputs.py @@ -105,9 +105,9 @@ def test_start_stop(self): self.wait_until(lambda: self.output_lines() == 1) - # Remove input - with open(self.working_dir + "/configs/input.yml", 'w') as f: - f.write("") + # Remove input by moving the file + # we keep it around to help debugging + os.rename(self.working_dir + "/configs/input.yml", self.working_dir + "/configs/input.yml.disabled") # Wait until input is stopped self.wait_until( From 0b58168c232bb7b4ca8cfc46a5f561d2e4860c14 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 13 May 2024 16:44:17 -0400 Subject: [PATCH 19/26] Fix the missing python tests --- filebeat/tests/system/test_reload_inputs.py | 5 +++-- filebeat/tests/system/test_reload_modules.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/filebeat/tests/system/test_reload_inputs.py b/filebeat/tests/system/test_reload_inputs.py index bd1e19408942..53644837c2cf 100644 --- a/filebeat/tests/system/test_reload_inputs.py +++ b/filebeat/tests/system/test_reload_inputs.py @@ -152,8 +152,9 @@ def test_start_stop_replace(self): self.wait_until(lambda: self.output_lines() == 1) # Remove input - with open(self.working_dir + "/configs/input.yml", 'w') as f: - f.write("") + # Remove input by moving the file + # we keep it around to help debugging + os.rename(self.working_dir + "/configs/input.yml", self.working_dir + "/configs/input.yml.disabled") # Wait until input is stopped self.wait_until( diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index 5b8e08f49f40..8e38775b1eff 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -144,9 +144,9 @@ def test_start_stop(self): self.wait_until(lambda: self.output_lines() == 1, max_timeout=10) print(self.output_lines()) - # Remove input - with open(self.working_dir + "/configs/system.yml", 'w') as f: - f.write("") + # Remove input by moving the file + # we keep it around to help debugging + os.rename(self.working_dir + "/configs/input.yml", self.working_dir + "/configs/input.yml.disabled") # Wait until input is stopped self.wait_until( From 999cc4638f6cc1c6906f3c9a435a07c1d84430d5 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 13 May 2024 16:50:46 -0400 Subject: [PATCH 20/26] Fix lint warnings --- libbeat/outputs/kafka/client.go | 16 ++++++++-------- libbeat/outputs/redis/client.go | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 08484f017bb4..afeb02a5534d 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -214,14 +214,14 @@ func (c *client) getEventMessage(data *publisher.Event) (*message, error) { if msg.topic == "" { topic, err := c.topic.Select(event) if err != nil { - return nil, fmt.Errorf("setting kafka topic failed with %v", err) + return nil, fmt.Errorf("setting kafka topic failed with %w", err) } if topic == "" { return nil, errNoTopicsSelected } msg.topic = topic if _, err := data.Cache.Put("topic", topic); err != nil { - return nil, fmt.Errorf("setting kafka topic in publisher event failed: %v", err) + return nil, fmt.Errorf("setting kafka topic in publisher event failed: %w", err) } } @@ -271,7 +271,7 @@ func (c *client) errorWorker(ch <-chan *sarama.ProducerError) { msg := errMsg.Msg.Metadata.(*message) msg.ref.fail(msg, errMsg.Err) - if errMsg.Err == breaker.ErrBreakerOpen { + if errors.Is(errMsg.Err, breaker.ErrBreakerOpen) { // ErrBreakerOpen is a very special case in Sarama. It happens only when // there have been repeated critical (broker / topic-level) errors, and it // puts Sarama into a state where it immediately rejects all input @@ -357,18 +357,18 @@ func (r *msgRef) done() { } func (r *msgRef) fail(msg *message, err error) { - switch err { - case sarama.ErrInvalidMessage: + switch { + case errors.Is(err, sarama.ErrInvalidMessage): r.client.log.Errorf("Kafka (topic=%v): dropping invalid message", msg.topic) r.client.observer.Dropped(1) - case sarama.ErrMessageSizeTooLarge, sarama.ErrInvalidMessageSize: + case errors.Is(err, sarama.ErrMessageSizeTooLarge) || errors.Is(err, sarama.ErrInvalidMessageSize): r.client.log.Errorf("Kafka (topic=%v): dropping too large message of size %v.", msg.topic, len(msg.key)+len(msg.value)) r.client.observer.Dropped(1) - case breaker.ErrBreakerOpen: + case errors.Is(err, breaker.ErrBreakerOpen): // Add this message to the failed list, but don't overwrite r.err since // all the breaker error means is "there were a lot of other errors". r.failed = append(r.failed, msg.data) @@ -412,7 +412,7 @@ func (r *msgRef) dec() { } func (c *client) Test(d testing.Driver) { - if c.config.Net.TLS.Enable == true { + if c.config.Net.TLS.Enable { d.Warn("TLS", "Kafka output doesn't support TLS testing") } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index c5d8590c759c..1fcd46e6f647 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -318,6 +318,7 @@ func serializeEvents( succeeded := data for _, d := range data { + d := d serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) @@ -336,6 +337,7 @@ failLoop: succeeded = data[:i] rest := data[i+1:] for _, d := range rest { + d := d serializedEvent, err := codec.Encode(index, &d.Content) if err != nil { log.Errorf("Encoding event failed with error: %+v. Look at the event log file to view the event", err) From b5c11a1a5b8b5c98b0f7dfe00793d8e2dbbbfb1f Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 14 May 2024 08:23:46 -0400 Subject: [PATCH 21/26] Fix module filename on Python test --- filebeat/tests/system/test_reload_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index 8e38775b1eff..4d0b530acd38 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -146,7 +146,7 @@ def test_start_stop(self): # Remove input by moving the file # we keep it around to help debugging - os.rename(self.working_dir + "/configs/input.yml", self.working_dir + "/configs/input.yml.disabled") + os.rename(self.working_dir + "/configs/system.yml", self.working_dir + "/configs/system.yml.disabled") # Wait until input is stopped self.wait_until( From a1c3277c3d88b9c734e5696dc18e4c3a6066b57e Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 15 May 2024 08:22:01 -0400 Subject: [PATCH 22/26] Update elastic-agent-libs to v0.9.8 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index d54d6fa766e2..aad81b518acc 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.7 +Version: v0.9.8 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.7/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.8/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index c45ea6852438..8d278dae027d 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.7 + github.com/elastic/elastic-agent-libs v0.9.8 github.com/elastic/elastic-agent-system-metrics v0.9.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.11.0 diff --git a/go.sum b/go.sum index 44e7a496ec69..e91e46ce2c67 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.7 h1:LZdfxbq724Y1zAdE3COp+OIPwU8SquOCLIXpI/twcdQ= -github.com/elastic/elastic-agent-libs v0.9.7/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= +github.com/elastic/elastic-agent-libs v0.9.8 h1:fwl3hp0gNmKkuERcUQTwe4cyIK6M0jJkv16EIsB6Apw= +github.com/elastic/elastic-agent-libs v0.9.8/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/elastic/elastic-agent-system-metrics v0.9.2 h1:/tvTKOt55EerU0WwGFoDhBlyWLgxyv7d8xCbny0bciw= github.com/elastic/elastic-agent-system-metrics v0.9.2/go.mod h1:VfJnKw4Jqrd9ddljXCwaGKJgN+7ADyyGk089NaXVsf0= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= From 360724d2db9d55ecc6005ad1d497944958bdf6de Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 15 May 2024 08:31:56 -0400 Subject: [PATCH 23/26] Update event logger ES output test Ensure the event data is not present in the normal log file --- filebeat/tests/integration/event_log_file_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/filebeat/tests/integration/event_log_file_test.go b/filebeat/tests/integration/event_log_file_test.go index 134830ce666e..5b2758b40186 100644 --- a/filebeat/tests/integration/event_log_file_test.go +++ b/filebeat/tests/integration/event_log_file_test.go @@ -128,4 +128,9 @@ func TestEventsLoggerESOutput(t *testing.T) { t.Errorf("Contents:\n%s", strData) t.FailNow() } + + // Ensure the normal log file does not contain the event data + if filebeat.LogContains(eventMsg) { + t.Fatalf("normal log file must NOT contain event data, '%s' found in the logs", eventMsg) + } } From f888e2619decbe03b5f89e4d68c759e2e6810b4c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 15 May 2024 12:24:19 -0400 Subject: [PATCH 24/26] Support reading both log files for integration tests This commit extents the integration tests framework to read the events log file. --- libbeat/tests/integration/framework.go | 139 ++++++++++++++++++------- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index 444a93daa38a..41a1fbcc37c0 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -53,6 +53,7 @@ type BeatProc struct { configFile string fullPath string logFileOffset int64 + eventLogFileOffset int64 t *testing.T tempDir string stdin io.WriteCloser @@ -283,13 +284,27 @@ func (b *BeatProc) Stop() { } // LogMatch tests each line of the logfile to see if contains any -// match of the provided regular expression. It will open the log -// file on every call, read until EOF, then close it. LogContains +// match of the provided regular expression. It will open the log +// file on every call, read until EOF, then close it. LogContains // will be faster so use that if possible. func (b *BeatProc) LogMatch(match string) bool { re := regexp.MustCompile(match) logFile := b.openLogFile() - _, err := logFile.Seek(b.logFileOffset, io.SeekStart) + + found := false + found, b.logFileOffset = b.logRegExpMatch(re, logFile, b.logFileOffset) + if found { + return found + } + + eventLogFile := b.openEventLogFile() + found, b.eventLogFileOffset = b.logRegExpMatch(re, eventLogFile, b.eventLogFileOffset) + + return found +} + +func (b *BeatProc) logRegExpMatch(re *regexp.Regexp, logFile *os.File, offset int64) (bool, int64) { + _, err := logFile.Seek(offset, io.SeekStart) if err != nil { b.t.Fatalf("could not set offset for '%s': %s", logFile.Name(), err) } @@ -306,7 +321,7 @@ func (b *BeatProc) LogMatch(match string) bool { for { data, err := r.ReadBytes('\n') line := string(data) - b.logFileOffset += int64(len(data)) + offset += int64(len(data)) if err != nil { if err != io.EOF { @@ -316,20 +331,49 @@ func (b *BeatProc) LogMatch(match string) bool { } if re.MatchString(line) { - return true + return true, offset } } - return false + return false, offset } // LogContains looks for `s` as a substring of every log line, // it will open the log file on every call, read it until EOF, -// then close it. +// then close it. It keeps track of the offset so subsequent calls +// will only read log entries that were not read by the previous +// call. +// +// The events log file is read after the normal log file and its +// offset is tracked separately. func (b *BeatProc) LogContains(s string) bool { - t := b.t logFile := b.openLogFile() - _, err := logFile.Seek(b.logFileOffset, io.SeekStart) + defer logFile.Close() + + found := false + found, b.logFileOffset = b.searchStrInLogs(logFile, s, b.logFileOffset) + if found { + return found + } + + eventLogFile := b.openEventLogFile() + if eventLogFile == nil { + return false + } + defer eventLogFile.Close() + found, b.eventLogFileOffset = b.searchStrInLogs(eventLogFile, s, b.eventLogFileOffset) + + return found +} + +// searchStrInLogs search for s as a substring of any line in logFile starting +// from offset. +// +// It will close logFile and return the current offset. +func (b *BeatProc) searchStrInLogs(logFile *os.File, s string, offset int64) (bool, int64) { + t := b.t + + _, err := logFile.Seek(offset, io.SeekStart) if err != nil { t.Fatalf("could not set offset for '%s': %s", logFile.Name(), err) } @@ -346,7 +390,7 @@ func (b *BeatProc) LogContains(s string) bool { for { data, err := r.ReadBytes('\n') line := string(data) - b.logFileOffset += int64(len(data)) + offset += int64(len(data)) if err != nil { if err != io.EOF { @@ -356,11 +400,11 @@ func (b *BeatProc) LogContains(s string) bool { } if strings.Contains(line, s) { - return true + return true, offset } } - return false + return false, offset } // WaitForLogs waits for the specified string s to be present in the logs within @@ -393,36 +437,36 @@ func (b *BeatProc) WriteConfigFile(cfg string) { b.baseArgs = append(b.baseArgs, "-c", b.configFile) } -// openLogFile opens the log file for reading and returns it. -// It also registers a cleanup function to close the file -// when the test ends. -func (b *BeatProc) openLogFile() *os.File { +// openGlobFile opens a file defined by glob. The glob must resolve to a single +// file otherwise the test fails. It returns a *os.File and a boolean indicating +// whether a file was found. +// +// If `waitForFile` is true, it will wait up to 5 seconds for the file to +// be created. The test will fail if the file is not found. If it is false +// and no file is found, nil and false are returned. +func (b *BeatProc) openGlobFile(glob string, waitForFile bool) *os.File { t := b.t - // Beats can produce two different log files, to make sure we're - // reading the normal one we add the year to the glob. The default - // log file name looks like: filebeat-20240116.ndjson - year := time.Now().Year() - glob := fmt.Sprintf("%s-%d*.ndjson", filepath.Join(b.tempDir, b.beatName), year) + files, err := filepath.Glob(glob) if err != nil { t.Fatalf("could not expand log file glob: %s", err) } - require.Eventually(t, func() bool { - files, err = filepath.Glob(glob) - if err != nil { - t.Fatalf("could not expand log file glob: %s", err) - } - return len(files) == 1 - }, 5*time.Second, 100*time.Millisecond, - "waiting for log file matching glob '%s' to be created", glob) + if waitForFile && len(files) == 0 { + require.Eventually(t, func() bool { + files, err = filepath.Glob(glob) + if err != nil { + t.Fatalf("could not expand log file glob: %s", err) + } + return len(files) == 1 + }, 5*time.Second, 100*time.Millisecond, + "waiting for log file matching glob '%s' to be created", glob) + } - // On a normal operation there must be a single log, if there are more - // than one, then there is an issue and the Beat is logging too much, - // which is enough to stop the test - if len(files) != 1 { - t.Fatalf("there must be only one log file for %s, found: %d", - glob, len(files)) + // We only reach this line if `waitForFile` is false, so we need + // to check whether we found a file + if len(files) == 0 { + return nil } f, err := os.Open(files[0]) @@ -433,6 +477,31 @@ func (b *BeatProc) openLogFile() *os.File { return f } +// openLogFile opens the log file for reading and returns it. +// It's the caller's responsibility to close the file. +func (b *BeatProc) openLogFile() *os.File { + // Beats can produce two different log files, to make sure we're + // reading the normal one we add the year to the glob. The default + // log file name looks like: filebeat-20240116.ndjson + year := time.Now().Year() + glob := fmt.Sprintf("%s-%d*.ndjson", filepath.Join(b.tempDir, b.beatName), year) + + return b.openGlobFile(glob, true) +} + +// openEventLogFile opens the log file for reading and returns it. +// If the events log file does not exist, nil is returned +// It's the caller's responsibility to close the file. +func (b *BeatProc) openEventLogFile() *os.File { + // Beats can produce two different log files, to make sure we're + // reading the normal one we add the year to the glob. The default + // log file name looks like: filebeat-20240116.ndjson + year := time.Now().Year() + glob := fmt.Sprintf("%s-events-data-%d*.ndjson", filepath.Join(b.tempDir, b.beatName), year) + + return b.openGlobFile(glob, false) +} + // createTempDir creates a temporary directory that will be // removed after the tests passes. // From 8711a5707c4842f0b4cd050247eab230d48cd2d9 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 21 May 2024 11:52:55 -0400 Subject: [PATCH 25/26] Fix integration tests framework --- libbeat/tests/integration/framework.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index 41a1fbcc37c0..cc305255709f 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -290,6 +290,7 @@ func (b *BeatProc) Stop() { func (b *BeatProc) LogMatch(match string) bool { re := regexp.MustCompile(match) logFile := b.openLogFile() + defer logFile.Close() found := false found, b.logFileOffset = b.logRegExpMatch(re, logFile, b.logFileOffset) @@ -298,6 +299,10 @@ func (b *BeatProc) LogMatch(match string) bool { } eventLogFile := b.openEventLogFile() + if eventLogFile == nil { + return false + } + defer eventLogFile.Close() found, b.eventLogFileOffset = b.logRegExpMatch(re, eventLogFile, b.eventLogFileOffset) return found @@ -479,6 +484,8 @@ func (b *BeatProc) openGlobFile(glob string, waitForFile bool) *os.File { // openLogFile opens the log file for reading and returns it. // It's the caller's responsibility to close the file. +// If the log file is not found, the test fails. The returned +// value is never nil. func (b *BeatProc) openLogFile() *os.File { // Beats can produce two different log files, to make sure we're // reading the normal one we add the year to the glob. The default From 4673969702173ebf2e5c5f97225d92dde8143b66 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 22 May 2024 18:03:01 -0400 Subject: [PATCH 26/26] Remove extra line in changelog next --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8594a86d3cdc..42b1d53c1de5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -198,7 +198,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669] - Introduce log message for not supported annotations for Hints based autodiscover {pull}38213[38213] - Add persistent volume claim name to volume if available {pull}38839[38839] - - Raw events are now logged to a different file, this prevents potentially sensitive information from leaking into log files {pull}38767[38767] *Auditbeat*