From a572a424c3d35d1a61f1ee3257b9dd6378225512 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 25 Oct 2023 08:21:40 +0200 Subject: [PATCH 01/22] Document ES output API usage (#36940) This commit documents the `_bulk` API usage by the Elasticsearch output and how different status codes are handled. Co-authored-by: Denis --- .../elasticsearch/docs/elasticsearch.asciidoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 5ea65c16dc4..7e3e64b6cc5 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -750,3 +750,17 @@ output.elasticsearch: non_indexable_policy.dead_letter_index: index: "my-dead-letter-index" ------------------------------------------------------------------------------ + +[[es-apis]] +==== Elasticsearch APIs +{beatname_uc} will use the `_bulk` API from {es}, the events are sent +in the order they arrive to the publishing pipeline, a single `_bulk` +request may contain events from different inputs/modules. Temporary +failures are re-tried. + +The status code for each event is checked and handled as: + +* `< 300`: The event is counted as `events.acked` +* `409` (Conflict): The event is counted as `events.duplicates` +* `429` (Too Many Requests): The event is counted as `events.toomany` +* `> 399 and < 500`: The `non_indexable_policy` is applied. \ No newline at end of file From bbf0111c1d50b13e9e2742758ab19e89562ad1a4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Wed, 25 Oct 2023 20:46:50 +1030 Subject: [PATCH 02/22] x-pack/filebeat/input/internal/httplog: fix incorrect append source (#36956) --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/internal/httplog/roundtripper.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a8b019f0863..43427a7365b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -113,6 +113,7 @@ is collected by it. - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395] - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772] - Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857] +- Fix handling of response errors in HTTPJSON and CEL request trace logging. {pull}36956[36956] *Heartbeat* diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index bbb76cb5045..78e872efa66 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -158,9 +158,9 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err switch len(errorsMessages) { case 0: case 1: - respParts = append(reqParts, zap.String("error.message", errorsMessages[0])) + respParts = append(respParts, zap.String("error.message", errorsMessages[0])) default: - respParts = append(reqParts, zap.Strings("error.message", errorsMessages)) + respParts = append(respParts, zap.Strings("error.message", errorsMessages)) } log.Debug("HTTP response", respParts...) From 5d6c308da027fd7702e60a8b64cdbe0b40755355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:53:16 +0200 Subject: [PATCH 03/22] [Heartbeat] Fix monitor duration wrapper (#36900) Fixes #36892. Monitor duration is not being calculated correctly, where start time is initialized after monitor execution and wrapping order is overriding retries event order. --- CHANGELOG.next.asciidoc | 1 + .../monitors/wrappers/summarizer/plugdrop.go | 4 + .../monitors/wrappers/summarizer/plugerr.go | 6 + .../wrappers/summarizer/plugmondur.go | 15 +- .../wrappers/summarizer/plugstatestat.go | 4 + .../monitors/wrappers/summarizer/plugurl.go | 2 + .../wrappers/summarizer/summarizer.go | 19 +- .../wrappers/summarizer/summarizer_test.go | 183 ++++++++++++++++++ 8 files changed, 227 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 43427a7365b..4e9a5f0128e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -119,6 +119,7 @@ is collected by it. - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Fix retries to trigger on a down monitor with no previous state. {pull}36842[36842] +- Fix monitor duration calculation with retries. {pull}36900[36900] *Metricbeat* diff --git a/heartbeat/monitors/wrappers/summarizer/plugdrop.go b/heartbeat/monitors/wrappers/summarizer/plugdrop.go index fff6c143bf0..a4ddc61abe7 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugdrop.go +++ b/heartbeat/monitors/wrappers/summarizer/plugdrop.go @@ -43,3 +43,7 @@ func (d DropBrowserExtraEvents) BeforeSummary(event *beat.Event) BeforeSummaryAc func (d DropBrowserExtraEvents) BeforeRetry() { // noop } + +func (d DropBrowserExtraEvents) BeforeEachEvent(event *beat.Event) { + // noop +} diff --git a/heartbeat/monitors/wrappers/summarizer/plugerr.go b/heartbeat/monitors/wrappers/summarizer/plugerr.go index 1010370f520..83ab6de4f5a 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugerr.go +++ b/heartbeat/monitors/wrappers/summarizer/plugerr.go @@ -46,6 +46,8 @@ func NewBrowserErrPlugin() *BrowserErrPlugin { } } +func (esp *BrowserErrPlugin) BeforeEachEvent(event *beat.Event) {} // noop + func (esp *BrowserErrPlugin) EachEvent(event *beat.Event, eventErr error) EachEventActions { // track these to determine if the journey // needs an error injected due to incompleteness @@ -127,6 +129,10 @@ func (esp *LightweightErrPlugin) BeforeRetry() { // noop } +func (esp *LightweightErrPlugin) BeforeEachEvent(event *beat.Event) { + // noop +} + // errToFieldVal reflects on the error and returns either an *ecserr.ECSErr if possible, and a look.Reason otherwise func errToFieldVal(eventErr error) (errVal interface{}) { var asECS *ecserr.ECSErr diff --git a/heartbeat/monitors/wrappers/summarizer/plugmondur.go b/heartbeat/monitors/wrappers/summarizer/plugmondur.go index f677e57693f..d71cc96ff2c 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugmondur.go +++ b/heartbeat/monitors/wrappers/summarizer/plugmondur.go @@ -31,12 +31,15 @@ type LightweightDurationPlugin struct { } func (lwdsp *LightweightDurationPlugin) EachEvent(event *beat.Event, _ error) EachEventActions { - // Effectively only runs once, on the first event + return 0 // noop +} + +func (lwdsp *LightweightDurationPlugin) BeforeEachEvent(event *beat.Event) { + // Effectively capture on the first event if lwdsp.startedAt == nil { now := time.Now() lwdsp.startedAt = &now } - return 0 } func (lwdsp *LightweightDurationPlugin) BeforeSummary(event *beat.Event) BeforeSummaryActions { @@ -44,7 +47,10 @@ func (lwdsp *LightweightDurationPlugin) BeforeSummary(event *beat.Event) BeforeS return 0 } -func (lwdsp *LightweightDurationPlugin) BeforeRetry() {} +func (lwdsp *LightweightDurationPlugin) BeforeRetry() { + // Reset event start time + lwdsp.startedAt = nil +} // BrowserDurationPlugin handles the logic for writing the `monitor.duration.us` field // for browser monitors. @@ -82,4 +88,5 @@ func (bwdsp *BrowserDurationPlugin) BeforeSummary(event *beat.Event) BeforeSumma return 0 } -func (bwdsp *BrowserDurationPlugin) BeforeRetry() {} +func (bwdsp *BrowserDurationPlugin) BeforeRetry() {} +func (bwdsp *BrowserDurationPlugin) BeforeEachEvent(event *beat.Event) {} // noop diff --git a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go index 4acfee4dc36..cf7e90af5f3 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go +++ b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go @@ -74,6 +74,8 @@ func (ssp *BrowserStateStatusPlugin) BeforeRetry() { ssp.cssp.BeforeRetry() } +func (ssp *BrowserStateStatusPlugin) BeforeEachEvent(event *beat.Event) {} //noop + // LightweightStateStatusPlugin encapsulates the writing of the primary fields used by the summary, // those being `state.*`, `status.*` , `event.type`, and `monitor.check_group` type LightweightStateStatusPlugin struct { @@ -113,6 +115,8 @@ func (ssp *LightweightStateStatusPlugin) BeforeRetry() { ssp.cssp.BeforeRetry() } +func (ssp *LightweightStateStatusPlugin) BeforeEachEvent(event *beat.Event) {} // noop + type commonSSP struct { js *jobsummary.JobSummary stateTracker *monitorstate.Tracker diff --git a/heartbeat/monitors/wrappers/summarizer/plugurl.go b/heartbeat/monitors/wrappers/summarizer/plugurl.go index dc4394aa42a..e47463575a3 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugurl.go +++ b/heartbeat/monitors/wrappers/summarizer/plugurl.go @@ -52,3 +52,5 @@ func (busp *BrowserURLPlugin) BeforeSummary(event *beat.Event) BeforeSummaryActi func (busp *BrowserURLPlugin) BeforeRetry() { busp.urlFields = nil } + +func (busp *BrowserURLPlugin) BeforeEachEvent(event *beat.Event) {} //noop diff --git a/heartbeat/monitors/wrappers/summarizer/summarizer.go b/heartbeat/monitors/wrappers/summarizer/summarizer.go index 9c3f1bd8abd..ad0902d45af 100644 --- a/heartbeat/monitors/wrappers/summarizer/summarizer.go +++ b/heartbeat/monitors/wrappers/summarizer/summarizer.go @@ -42,6 +42,12 @@ type Summarizer struct { startedAt time.Time } +func (s Summarizer) beforeEachEvent(event *beat.Event) { + for _, plugin := range s.plugins { + plugin.BeforeEachEvent(event) + } +} + // EachEventActions is a set of options using bitmasks to inform execution after the EachEvent callback type EachEventActions uint8 @@ -58,6 +64,9 @@ const RetryBeforeSummary = 1 // in one location. Prior to this code was strewn about a bit more and following it was // a bit trickier. type SummarizerPlugin interface { + // BeforeEachEvent is called on each event, and allows for the mutation of events + // before monitor execution + BeforeEachEvent(event *beat.Event) // EachEvent is called on each event, and allows for the mutation of events EachEvent(event *beat.Event, err error) EachEventActions // BeforeSummary is run on the final (summary) event for each monitor. @@ -106,6 +115,10 @@ func (s *Summarizer) setupPlugins() { // This adds the state and summary top level fields. func (s *Summarizer) Wrap(j jobs.Job) jobs.Job { return func(event *beat.Event) ([]jobs.Job, error) { + + // call BeforeEachEvent for each plugin before running job + s.beforeEachEvent(event) + conts, eventErr := j(event) s.mtx.Lock() @@ -145,14 +158,14 @@ func (s *Summarizer) Wrap(j jobs.Job) jobs.Job { // kibana queries // 2. If the site error is very short 1s gives it a tiny bit of time to recover delayedRootJob := func(event *beat.Event) ([]jobs.Job, error) { + time.Sleep(s.retryDelay) for _, p := range s.plugins { p.BeforeRetry() } - time.Sleep(s.retryDelay) - return s.rootJob(event) + return s.Wrap(s.rootJob)(event) } - conts = []jobs.Job{delayedRootJob} + return []jobs.Job{delayedRootJob}, eventErr } } diff --git a/heartbeat/monitors/wrappers/summarizer/summarizer_test.go b/heartbeat/monitors/wrappers/summarizer/summarizer_test.go index 2a94b3e6f59..e579a649c8e 100644 --- a/heartbeat/monitors/wrappers/summarizer/summarizer_test.go +++ b/heartbeat/monitors/wrappers/summarizer/summarizer_test.go @@ -19,11 +19,13 @@ package summarizer import ( "fmt" + "sync" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/heartbeat/look" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitorstate" @@ -219,3 +221,184 @@ func TestSummarizer(t *testing.T) { }) } } + +// Test wrapper plugin hook order. Guaranteed order for plugins to be called upon determines +// what data can be appended to the event at each stage through retries. With this guarantee, +// plugins just need to ascertain that their invariants apply through hook execution order +func TestSummarizerPluginOrder(t *testing.T) { + t.Parallel() + + // these tests use strings to describe sequences of events + tests := []struct { + name string + maxAttempts int + expectedOrder []string + }{ + { + "one attempt", + 1, + []string{"bee", "job", "ee", "bs"}, + }, + { + "two attempts", + 2, + []string{"bee", "job", "ee", "bs", "br", "bee", "job", "ee", "bs"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Monitor setup + tracker := monitorstate.NewTracker(monitorstate.NilStateLoader, false) + sf := stdfields.StdMonitorFields{ID: "testmon", Name: "testmon", Type: "http", MaxAttempts: uint16(tt.maxAttempts)} + + // Test locals + calls := make(chan string, 100) + mtx := sync.Mutex{} + appendCall := func(event string) { + mtx.Lock() + defer mtx.Unlock() + + // Append call to global chan + calls <- event + } + + // We simplify these to always down since hook order should not be + // determined by status + job := func(event *beat.Event) (j []jobs.Job, retErr error) { + + calls <- "job" + + event.Fields = mapstr.M{ + "monitor": mapstr.M{ + "id": "test", + "status": string(monitorstate.StatusDown), + }, + } + + return nil, fmt.Errorf("dummyerr") + } + + s := NewSummarizer(job, sf, tracker) + // Shorten retry delay to make tests run faster + s.retryDelay = 2 * time.Millisecond + // Add mock plugin + s.plugins = append(s.plugins, &MockPlugin{ + eachEvent: func(_ *beat.Event, _ error) { + appendCall("ee") + }, + beforeSummary: func(_ *beat.Event) { + appendCall("bs") + }, + beforeRetry: func() { + appendCall("br") + }, + beforeEachEvent: func(_ *beat.Event) { + appendCall("bee") + }, + }) + wrapped := s.Wrap(job) + + _, _ = jobs.ExecJobAndConts(t, wrapped) + + close(calls) + + // gather order + rcvdOrder := []string{} + for c := range calls { + rcvdOrder = append(rcvdOrder, c) + } + + require.Equal(t, tt.expectedOrder, rcvdOrder) + require.Len(t, rcvdOrder, len(tt.expectedOrder)) + }) + } +} + +func TestRetryLightweightMonitorDuration(t *testing.T) { + t.Parallel() + + // Monitor setup + tracker := monitorstate.NewTracker(monitorstate.NilStateLoader, false) + sf := stdfields.StdMonitorFields{ID: "testmon", Name: "testmon", Type: "http", MaxAttempts: uint16(2)} + + // We simplify these to always down + job := func(event *beat.Event) (j []jobs.Job, retErr error) { + + // some platforms don't have enough precision to track immediate monitors time + time.Sleep(100 * time.Millisecond) + + event.Fields = mapstr.M{ + "monitor": mapstr.M{ + "id": "test", + "status": string(monitorstate.StatusDown), + }, + } + + return nil, fmt.Errorf("dummyerr") + } + + var retryStart time.Time + + s := NewSummarizer(job, sf, tracker) + // Shorten retry delay to make tests run faster + s.retryDelay = 2 * time.Millisecond + // Add mock plugin + s.plugins = append(s.plugins, &MockPlugin{ + beforeRetry: func() { + retryStart = time.Now() + }, + eachEvent: func(_ *beat.Event, _ error) {}, + beforeSummary: func(_ *beat.Event) {}, + beforeEachEvent: func(_ *beat.Event) {}, + }) + wrapped := s.Wrap(job) + + events, _ := jobs.ExecJobAndConts(t, wrapped) + + retryElapsed := time.Since(retryStart) + require.False(t, retryStart.IsZero()) + var rcvdDuration interface{} + for _, event := range events { + summaryIface, _ := event.GetValue("summary") + summary := summaryIface.(*jobsummary.JobSummary) + + if summary.FinalAttempt { + rcvdDuration, _ = event.GetValue("monitor.duration.us") + } + } + require.Greater(t, rcvdDuration, int64(0)) + // Ensures monitor duration only takes into account the last attempt execution time + // by comparing it to the time spent after last retry started (retryElapsed) + require.GreaterOrEqual(t, look.RTTMS(retryElapsed), rcvdDuration) +} + +type MockPlugin struct { + eachEvent func(e *beat.Event, err error) + beforeSummary func(e *beat.Event) + beforeRetry func() + beforeEachEvent func(e *beat.Event) +} + +func (mp *MockPlugin) EachEvent(e *beat.Event, err error) EachEventActions { + mp.eachEvent(e, err) + + return 0 +} + +func (mp *MockPlugin) BeforeSummary(e *beat.Event) BeforeSummaryActions { + mp.beforeSummary(e) + + return 0 +} + +func (mp *MockPlugin) BeforeRetry() { + mp.beforeRetry() +} + +func (mp *MockPlugin) BeforeEachEvent(e *beat.Event) { + mp.beforeEachEvent(e) +} From 55df09fadeae98979aa012801b669c7fbef321dd Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:26:36 -0500 Subject: [PATCH 04/22] Add support for idle_connection_timeout to elasticsearch output (#36843) * Add support for idle_connection_timeout to elasticsearch output --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 5 +++++ filebeat/filebeat.reference.yml | 5 +++++ heartbeat/heartbeat.reference.yml | 5 +++++ .../_meta/config/output-elasticsearch.reference.yml.tmpl | 5 +++++ libbeat/outputs/elasticsearch/client.go | 1 + libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc | 6 ++++++ libbeat/outputs/elasticsearch/elasticsearch.go | 5 ++++- metricbeat/metricbeat.reference.yml | 5 +++++ packetbeat/packetbeat.reference.yml | 5 +++++ winlogbeat/winlogbeat.reference.yml | 5 +++++ x-pack/auditbeat/auditbeat.reference.yml | 5 +++++ x-pack/filebeat/filebeat.reference.yml | 5 +++++ x-pack/functionbeat/functionbeat.reference.yml | 5 +++++ x-pack/heartbeat/heartbeat.reference.yml | 5 +++++ x-pack/metricbeat/metricbeat.reference.yml | 5 +++++ x-pack/osquerybeat/osquerybeat.reference.yml | 5 +++++ x-pack/packetbeat/packetbeat.reference.yml | 5 +++++ x-pack/winlogbeat/winlogbeat.reference.yml | 5 +++++ 19 files changed, 87 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4e9a5f0128e..a59b65297c2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -173,6 +173,7 @@ is collected by it. - allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788] - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments +- elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index d4214eaf604..a3a36dde753 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -522,6 +522,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 45aff60ce23..f49390485ac 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1618,6 +1618,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e8b74f8c075..fe6a72cd474 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -614,6 +614,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index 4acd341da01..d6ebe03ada5 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -81,6 +81,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index c80e95ebc90..b485807776e 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -102,6 +102,7 @@ func NewClient( CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, Transport: s.Transport, + IdleConnTimeout: s.IdleConnTimeout, }) if err != nil { return nil, err diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 7e3e64b6cc5..6af56ac42db 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -689,6 +689,12 @@ default is `1s`. The maximum number of seconds to wait before attempting to connect to Elasticsearch after a network error. The default is `60s`. +===== `idle_connection_timeout` + +The maximum amount of time an idle connection will remain idle before closing itself. +Zero means no limit. The format is a Go language duration (example 60s is 60 seconds). +The default is 0. + ===== `timeout` The http request timeout in seconds for the Elasticsearch request. The default is 90. diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index f7e38853924..145b5b65c2e 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -41,7 +41,9 @@ func makeES( ) (outputs.Group, error) { log := logp.NewLogger(logSelector) if !cfg.HasField("bulk_max_size") { - _ = cfg.SetInt("bulk_max_size", -1, defaultBulkSize) + if err := cfg.SetInt("bulk_max_size", -1, defaultBulkSize); err != nil { + return outputs.Fail(err) + } } index, pipeline, err := buildSelectors(im, beat, cfg) @@ -105,6 +107,7 @@ func makeES( Observer: observer, EscapeHTML: esConfig.EscapeHTML, Transport: esConfig.Transport, + IdleConnTimeout: esConfig.Transport.IdleConnTimeout, }, Index: index, Pipeline: pipeline, diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index fc79ddb514c..8508d7d6a70 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1357,6 +1357,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index cc05f7b5212..d31a7b901d9 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -988,6 +988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 110370957cf..bdc366d0902 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -404,6 +404,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 6d9a71ca99c..09b343d8fe2 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -578,6 +578,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index bff96ef1997..90cbc52d8c2 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3988,6 +3988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index d3a2231a43e..b7b075f2612 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -646,6 +646,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index e8b74f8c075..fe6a72cd474 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -614,6 +614,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 436693bdfbc..bfb13acc660 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1918,6 +1918,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index f17d16e28b8..6103f48c428 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -365,6 +365,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index cc05f7b5212..d31a7b901d9 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -988,6 +988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index eec0bca8077..f3ff654bca2 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -406,6 +406,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 From d8a1377371141e7896827d6e4ae952cd6502a361 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:12:48 +1030 Subject: [PATCH 05/22] mod: update version of github.com/elastic/go-libaudit to v2.4.0 (#36964) --- CHANGELOG.next.asciidoc | 2 ++ NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 5 +++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a59b65297c2..aecead61b01 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -251,6 +251,8 @@ is collected by it. *Auditbeat* +- Upgrade go-libaudit to v2.4.0. {issue}36776[36776] {pull}36964[36964] + *Libbeat* *Heartbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 7061208f1ef..0d59be5ef12 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13659,11 +13659,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-libaudit/v2 -Version: v2.3.3 +Version: v2.4.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.3.3/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.4.0/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 227b7659e63..037291b36cd 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.3.5 github.com/elastic/elastic-agent-client/v7 v7.4.0 github.com/elastic/go-concert v0.2.0 - github.com/elastic/go-libaudit/v2 v2.3.3 + github.com/elastic/go-libaudit/v2 v2.4.0 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-lookslike v0.3.0 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f diff --git a/go.sum b/go.sum index 6ce895072b5..8d1a2265e37 100644 --- a/go.sum +++ b/go.sum @@ -670,8 +670,8 @@ github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa2 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38= github.com/elastic/go-elasticsearch/v8 v8.10.0 h1:ALg3DMxSrx07YmeMNcfPf7cFh1Ep2+Qa19EOXTbwr2k= github.com/elastic/go-elasticsearch/v8 v8.10.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE= -github.com/elastic/go-libaudit/v2 v2.3.3 h1:PO+9/HDSn65UAyydkkoTf81QET14fWmocHFiGEX/E6M= -github.com/elastic/go-libaudit/v2 v2.3.3/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo= +github.com/elastic/go-libaudit/v2 v2.4.0 h1:PqaGnB+dncrdUXqzQMyJu/dGysAtk6m5V3GIBMY473I= +github.com/elastic/go-libaudit/v2 v2.4.0/go.mod h1:AjlnhinP+kKQuUJoXLVrqxBM8uyhQmkzoV6jjsCFP4Q= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-lookslike v0.3.0 h1:HDI/DQ65V85ZqM7D/sbxcK2wFFnh3+7iFvBk2v2FTHs= @@ -2341,6 +2341,7 @@ golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From 8982110e828f512db0985ec5737881824b9309ba Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 26 Oct 2023 12:45:19 +0200 Subject: [PATCH 06/22] Add benchmark for processors that create event backups (#36960) This will come handy once we have another PR with optimizations. Also, extended a test case in the append processor due to the lack of coverage. --- libbeat/processors/actions/append_test.go | 3 +- libbeat/processors/processor_test.go | 221 ++++++++++++++++++++++ 2 files changed, 223 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/actions/append_test.go b/libbeat/processors/actions/append_test.go index 8cb8549b389..ddc03d6f322 100644 --- a/libbeat/processors/actions/append_test.go +++ b/libbeat/processors/actions/append_test.go @@ -273,6 +273,7 @@ func Test_appendProcessor_Run(t *testing.T) { logger: log, config: appendProcessorConfig{ Fields: []string{"field"}, + Values: []interface{}{"value3", "value4"}, TargetField: "target", }, }, @@ -281,7 +282,7 @@ func Test_appendProcessor_Run(t *testing.T) { Meta: mapstr.M{}, Fields: mapstr.M{ "field": "I'm being appended", - "target": []interface{}{"value1", "value2", "I'm being appended"}, + "target": []interface{}{"value1", "value2", "I'm being appended", "value3", "value4"}, }, }, }, diff --git a/libbeat/processors/processor_test.go b/libbeat/processors/processor_test.go index 41ed628fbfb..91d122365e7 100644 --- a/libbeat/processors/processor_test.go +++ b/libbeat/processors/processor_test.go @@ -18,15 +18,23 @@ package processors_test import ( + "fmt" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" _ "github.com/elastic/beats/v7/libbeat/processors/actions" _ "github.com/elastic/beats/v7/libbeat/processors/add_cloud_metadata" + _ "github.com/elastic/beats/v7/libbeat/processors/add_process_metadata" + _ "github.com/elastic/beats/v7/libbeat/processors/convert" + _ "github.com/elastic/beats/v7/libbeat/processors/decode_csv_fields" + _ "github.com/elastic/beats/v7/libbeat/processors/dissect" + _ "github.com/elastic/beats/v7/libbeat/processors/extract_array" + _ "github.com/elastic/beats/v7/libbeat/processors/urldecode" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -566,3 +574,216 @@ func TestDropMissingFields(t *testing.T) { assert.Equal(t, expectedEvent, processedEvent.Fields) } + +const ( + fieldCount = 20 + depth = 3 +) + +func BenchmarkEventBackups(b *testing.B) { + // listing all the processors that revert changes in case of an error + yml := []map[string]interface{}{ + { + "append": map[string]interface{}{ + "target_field": "append_target", + "values": []interface{}{"third", "fourth"}, + "fail_on_error": true, + }, + }, + { + "copy_fields": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "copy_from", + "to": "copy.to", + }, + }, + "fail_on_error": true, + }, + }, + { + "decode_base64_field": map[string]interface{}{ + "field": map[string]interface{}{ + "from": "base64_from", + "to": "base64_to", + }, + "fail_on_error": true, + }, + }, + { + "decompress_gzip_field": map[string]interface{}{ + "field": map[string]interface{}{ + "from": "gzip_from", + "to": "gzip_to", + }, + "fail_on_error": true, + }, + }, + { + "rename": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "rename_from", + "to": "rename.to", + }, + }, + "fail_on_error": true, + }, + }, + { + "replace": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "field": "replace_test", + "pattern": "to replace", + "replacement": "replaced", + }, + }, + "fail_on_error": true, + }, + }, + { + "truncate_fields": map[string]interface{}{ + "fields": []interface{}{"to_truncate"}, + "max_characters": 4, + "fail_on_error": true, + }, + }, + { + "convert": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "convert_from", + "to": "convert.to", + "type": "integer", + }, + }, + "fail_on_error": true, + }, + }, + { + "decode_csv_fields": map[string]interface{}{ + "fields": map[string]interface{}{ + "csv_from": "csv.to", + }, + "fail_on_error": true, + }, + }, + // it creates a backup unless `ignore_failure` is true + { + "dissect": map[string]interface{}{ + "tokenizer": "%{key1} %{key2}", + "field": "to_dissect", + }, + }, + { + "extract_array": map[string]interface{}{ + "field": "array_test", + "mappings": map[string]interface{}{ + "array_first": 0, + "array_second": 1, + }, + "fail_on_error": true, + }, + }, + { + "urldecode": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "url_from", + "to": "url.to", + }, + }, + + "fail_on_error": true, + }, + }, + } + + processors := GetProcessors(b, yml) + event := &beat.Event{ + Timestamp: time.Now(), + Meta: mapstr.M{}, + Fields: mapstr.M{ + "append_target": []interface{}{"first", "second"}, + "copy_from": "to_copy", + "base64_from": "dmFsdWU=", + // "decompressed data" + "gzip_from": string([]byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 74, 73, 77, 206, 207, 45, 40, 74, 45, 46, 78, 77, 81, 72, 73, 44, 73, 4, 4, 0, 0, 255, 255, 108, 158, 105, 19, 17, 0, 0, 0}), + "rename_from": "renamed_value", + "replace_test": "something to replace", + "to_truncate": "something very long", + "convert_from": "42", + "csv_from": "1,2,3,4", + "to_dissect": "some words", + "array_test": []string{"first", "second"}, + "url_from": "https%3A%2F%2Fwww.elastic.co%3Fsome", + }, + } + + expFields := mapstr.M{ + "append_target": []interface{}{"first", "second", "third", "fourth"}, + "copy_from": "to_copy", + "copy": mapstr.M{ + "to": "to_copy", + }, + "base64_from": "dmFsdWU=", + "base64_to": "value", + "gzip_from": string([]byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 74, 73, 77, 206, 207, 45, 40, 74, 45, 46, 78, 77, 81, 72, 73, 44, 73, 4, 4, 0, 0, 255, 255, 108, 158, 105, 19, 17, 0, 0, 0}), + "gzip_to": "decompressed data", + "rename": mapstr.M{"to": "renamed_value"}, + "replace_test": "something replaced", + "to_truncate": "some", + "convert_from": "42", + "convert": mapstr.M{"to": int32(42)}, + "csv_from": "1,2,3,4", + "csv": mapstr.M{"to": []string{"1", "2", "3", "4"}}, + "to_dissect": "some words", + "dissect": mapstr.M{ + "key1": "some", + "key2": "words", + }, + "array_test": []string{"first", "second"}, + "array_first": "first", + "array_second": "second", + "url_from": "https%3A%2F%2Fwww.elastic.co%3Fsome", + "url": mapstr.M{"to": "https://www.elastic.co?some"}, + } + + generateFields(b, event.Meta, fieldCount, depth) + generateFields(b, event.Fields, fieldCount, depth) + + var ( + result *beat.Event + clone *beat.Event + err error + ) + + b.Run("run processors that use backups", func(b *testing.B) { + for i := 0; i < b.N; i++ { + clone = event.Clone() // necessary for making and comparing changes + result, err = processors.Run(clone) + } + require.NoError(b, err) + require.NotNil(b, result) + }) + + require.Equal(b, fmt.Sprintf("%p", clone), fmt.Sprintf("%p", result), "should be the same event") + for key := range expFields { + require.Equal(b, expFields[key], clone.Fields[key], fmt.Sprintf("%s does not match", key)) + } +} + +func generateFields(t require.TestingT, m mapstr.M, count, nesting int) { + for i := 0; i < count; i++ { + var err error + if nesting == 0 { + _, err = m.Put(fmt.Sprintf("field-%d", i), fmt.Sprintf("value-%d", i)) + } else { + nested := mapstr.M{} + generateFields(t, nested, count, nesting-1) + _, err = m.Put(fmt.Sprintf("field-%d", i), nested) + } + require.NoError(t, err) + } +} From ee864b52d115b531e8e8983ba35cc786b05069b0 Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:24:23 -0500 Subject: [PATCH 07/22] Deprecate RSA2ELK Filebeat modules (#36887) - Add deprecation notices to RSA2ELK Filebeat modules. - Add doc page for migrating off of deprecated modules. --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/howto/howto.asciidoc | 2 ++ .../migrate-from-deprecated-module.asciidoc | 30 +++++++++++++++++++ filebeat/docs/modules/barracuda.asciidoc | 2 ++ filebeat/docs/modules/bluecoat.asciidoc | 2 ++ filebeat/docs/modules/cisco.asciidoc | 4 +++ filebeat/docs/modules/cylance.asciidoc | 2 ++ filebeat/docs/modules/f5.asciidoc | 2 ++ filebeat/docs/modules/fortinet.asciidoc | 6 ++++ filebeat/docs/modules/imperva.asciidoc | 2 ++ filebeat/docs/modules/infoblox.asciidoc | 2 ++ filebeat/docs/modules/juniper.asciidoc | 4 +++ filebeat/docs/modules/microsoft.asciidoc | 2 ++ filebeat/docs/modules/netscout.asciidoc | 2 ++ filebeat/docs/modules/proofpoint.asciidoc | 2 ++ filebeat/docs/modules/radware.asciidoc | 2 ++ filebeat/docs/modules/snort.asciidoc | 2 ++ filebeat/docs/modules/sonicwall.asciidoc | 2 ++ filebeat/docs/modules/sophos.asciidoc | 2 ++ filebeat/docs/modules/squid.asciidoc | 2 ++ filebeat/docs/modules/tomcat.asciidoc | 2 ++ filebeat/docs/modules/zscaler.asciidoc | 2 ++ .../module/barracuda/_meta/docs.asciidoc | 2 ++ .../module/bluecoat/_meta/docs.asciidoc | 2 ++ .../filebeat/module/cisco/_meta/docs.asciidoc | 4 +++ .../module/cylance/_meta/docs.asciidoc | 2 ++ x-pack/filebeat/module/f5/_meta/docs.asciidoc | 2 ++ .../module/fortinet/_meta/docs.asciidoc | 6 ++++ .../module/imperva/_meta/docs.asciidoc | 2 ++ .../module/infoblox/_meta/docs.asciidoc | 2 ++ .../module/juniper/_meta/docs.asciidoc | 4 +++ .../module/microsoft/_meta/docs.asciidoc | 2 ++ .../module/netscout/_meta/docs.asciidoc | 2 ++ .../module/proofpoint/_meta/docs.asciidoc | 2 ++ .../module/radware/_meta/docs.asciidoc | 2 ++ .../filebeat/module/snort/_meta/docs.asciidoc | 2 ++ .../module/sonicwall/_meta/docs.asciidoc | 2 ++ .../module/sophos/_meta/docs.asciidoc | 2 ++ .../filebeat/module/squid/_meta/docs.asciidoc | 2 ++ .../module/tomcat/_meta/docs.asciidoc | 2 ++ .../module/zscaler/_meta/docs.asciidoc | 2 ++ 41 files changed, 125 insertions(+) create mode 100644 filebeat/docs/howto/migrate-from-deprecated-module.asciidoc diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index aecead61b01..74a66ef3c23 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -302,6 +302,7 @@ is collected by it. *Filebeat* +- Deprecate rsa2elk Filebeat modules. {issue}36125[36125] {pull}36887[36887] *Heartbeat* diff --git a/filebeat/docs/howto/howto.asciidoc b/filebeat/docs/howto/howto.asciidoc index 7d18cd48225..7b9a3cffa03 100644 --- a/filebeat/docs/howto/howto.asciidoc +++ b/filebeat/docs/howto/howto.asciidoc @@ -16,6 +16,7 @@ Learn how to perform common {beatname_uc} configuration tasks. * <> * <> * <> +* <> -- @@ -46,4 +47,5 @@ include::{libbeat-dir}/yaml.asciidoc[] include::migrate-to-filestream.asciidoc[] +include::migrate-from-deprecated-module.asciidoc[] diff --git a/filebeat/docs/howto/migrate-from-deprecated-module.asciidoc b/filebeat/docs/howto/migrate-from-deprecated-module.asciidoc new file mode 100644 index 00000000000..fd163353f92 --- /dev/null +++ b/filebeat/docs/howto/migrate-from-deprecated-module.asciidoc @@ -0,0 +1,30 @@ +[[migrate-from-deprecated-module]] +== Migrating from a Deprecated Filebeat Module + +If a Filebeat module has been deprecated, there are a few options available for +a path forward: + +1. Migrate to an Elastic integration, if available. The deprecation notice will +link to an appropriate integration, if one exists. + +2. https://www.elastic.co/guide/en/fleet/current/migrate-beats-to-agent.html[Migrate to Elastic Agent] +for ingesting logs. If a specific integration for the vendor/product does not +exist, then one of the custom integrations can be used for ingesting events. A +https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html[custom pipeline] +may also be attached to the integration for further processing. + - https://docs.elastic.co/integrations/cel[CEL Custom API] - Collect events from an API using CEL (Common Expression Language) + - https://docs.elastic.co/integrations/httpjson[Custom API] - Collect events from an API using the HTTPJSON input + - https://docs.elastic.co/integrations/gcp_pubsub[Custom Google Pub/Sub] - Collect events from Google Pub/Sub topics + - https://docs.elastic.co/integrations/http_endpoint[Custom HTTP Endpoint] - Collect events from a listening HTTP port + - https://docs.elastic.co/integrations/journald[Custom Journald] - Collect events from journald + - https://docs.elastic.co/integrations/kafka_log[Custom Kafka] - Collect events from a Kafka topic + - https://docs.elastic.co/integrations/log[Custom Logs] - Collect events from files + - https://docs.elastic.co/integrations/tcp[Custom TCP] - Collect events from a listening TCP port + - https://docs.elastic.co/integrations/udp[Custom UDP] - Collect events from a listening UDP port + - https://docs.elastic.co/integrations/winlog[Custom Windows Event] - Collect events from a Windows Event Log channel + +3. Migrate to a different Filebeat module. In some cases, a Filebeat module may +be superseded by a new module. The deprecation notice will link to an appropriate +module, if one exists. + +4. Use a custom Filebeat input, processors, and ingest pipeline (if necessary). diff --git a/filebeat/docs/modules/barracuda.asciidoc b/filebeat/docs/modules/barracuda.asciidoc index 4b9f65fdfaf..958c9a4444b 100644 --- a/filebeat/docs/modules/barracuda.asciidoc +++ b/filebeat/docs/modules/barracuda.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Barracuda module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/barracuda[Barracuda Web Application Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/bluecoat.asciidoc b/filebeat/docs/modules/bluecoat.asciidoc index 843ef578aa3..c9dc391c6b4 100644 --- a/filebeat/docs/modules/bluecoat.asciidoc +++ b/filebeat/docs/modules/bluecoat.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Bluecoat module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index da25fb38c05..ca094907b98 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -281,6 +281,8 @@ include::../include/timezone-support.asciidoc[] [float] ==== `nexus` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_nexus[Cisco Nexus] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "cisconxos" device revision 134. @@ -326,6 +328,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `meraki` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_meraki[Cisco Meraki] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "ciscomeraki" device revision 118. diff --git a/filebeat/docs/modules/cylance.asciidoc b/filebeat/docs/modules/cylance.asciidoc index ecb87052881..8eb9b356ce7 100644 --- a/filebeat/docs/modules/cylance.asciidoc +++ b/filebeat/docs/modules/cylance.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Cylance module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/f5.asciidoc b/filebeat/docs/modules/f5.asciidoc index 366587eb675..d8a5f50d784 100644 --- a/filebeat/docs/modules/f5.asciidoc +++ b/filebeat/docs/modules/f5.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == F5 module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/f5_bigip[F5 BIG-IP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/fortinet.asciidoc b/filebeat/docs/modules/fortinet.asciidoc index 3149c062ae2..f6575c7db65 100644 --- a/filebeat/docs/modules/fortinet.asciidoc +++ b/filebeat/docs/modules/fortinet.asciidoc @@ -81,6 +81,8 @@ events. Defaults to `[fortinet-firewall, forwarded]`. [float] ==== `clientendpoint` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_forticlient[Fortinet FortiClient Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "forticlientendpoint" device revision 0. @@ -126,6 +128,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimail` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimail[Fortinet FortiMail] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetfortimail" device revision 131. @@ -171,6 +175,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimanager` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimanager[Fortinet FortiManager Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetmgr" device revision 134. diff --git a/filebeat/docs/modules/imperva.asciidoc b/filebeat/docs/modules/imperva.asciidoc index 085c44e126b..efe2a0bcdab 100644 --- a/filebeat/docs/modules/imperva.asciidoc +++ b/filebeat/docs/modules/imperva.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Imperva module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/infoblox.asciidoc b/filebeat/docs/modules/infoblox.asciidoc index 50539d2f9bd..2cc018bd20b 100644 --- a/filebeat/docs/modules/infoblox.asciidoc +++ b/filebeat/docs/modules/infoblox.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Infoblox module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/infoblox_nios[Infoblox NIOS] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/juniper.asciidoc b/filebeat/docs/modules/juniper.asciidoc index 274c559fae2..dd445e809a7 100644 --- a/filebeat/docs/modules/juniper.asciidoc +++ b/filebeat/docs/modules/juniper.asciidoc @@ -142,6 +142,8 @@ This is a list of JunOS fields that are mapped to ECS. [float] ==== `junos` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/juniper_srx[Juniper SRX] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "junosrouter" device revision 134. @@ -187,6 +189,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `netscreen` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. See <> for migration options."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "netscreen" device revision 134. diff --git a/filebeat/docs/modules/microsoft.asciidoc b/filebeat/docs/modules/microsoft.asciidoc index 2772f12d3f1..4e5ff72b30b 100644 --- a/filebeat/docs/modules/microsoft.asciidoc +++ b/filebeat/docs/modules/microsoft.asciidoc @@ -224,6 +224,8 @@ And for all other Defender ATP event types, go to Host -> Events. [float] ==== `dhcp` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/microsoft_dhcp[Microsoft DHCP] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "msdhcp" device revision 99. diff --git a/filebeat/docs/modules/netscout.asciidoc b/filebeat/docs/modules/netscout.asciidoc index 552153d1c67..2a9bfe34f67 100644 --- a/filebeat/docs/modules/netscout.asciidoc +++ b/filebeat/docs/modules/netscout.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Netscout module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] This is a module for receiving Arbor Peakflow SP logs over Syslog or a file. diff --git a/filebeat/docs/modules/proofpoint.asciidoc b/filebeat/docs/modules/proofpoint.asciidoc index 5f58e6764f3..cb420ae1e10 100644 --- a/filebeat/docs/modules/proofpoint.asciidoc +++ b/filebeat/docs/modules/proofpoint.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Proofpoint module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/proofpoint_tap[Proofpoint TAP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/radware.asciidoc b/filebeat/docs/modules/radware.asciidoc index 23ad0b7a179..1ebab4928e4 100644 --- a/filebeat/docs/modules/radware.asciidoc +++ b/filebeat/docs/modules/radware.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Radware module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/snort.asciidoc b/filebeat/docs/modules/snort.asciidoc index 8599c6d3e65..0a5c7c0b1c0 100644 --- a/filebeat/docs/modules/snort.asciidoc +++ b/filebeat/docs/modules/snort.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Snort module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/snort[Snort] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/sonicwall.asciidoc b/filebeat/docs/modules/sonicwall.asciidoc index bd8803659fe..a760da67ba0 100644 --- a/filebeat/docs/modules/sonicwall.asciidoc +++ b/filebeat/docs/modules/sonicwall.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Sonicwall module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/sonicwall[SonicWall Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/sophos.asciidoc b/filebeat/docs/modules/sophos.asciidoc index 1832e6603c3..a5d12dca366 100644 --- a/filebeat/docs/modules/sophos.asciidoc +++ b/filebeat/docs/modules/sophos.asciidoc @@ -152,6 +152,8 @@ This is a list of SophosXG fields that are mapped to ECS. [float] ==== `utm` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "astarosg" device revision 123. diff --git a/filebeat/docs/modules/squid.asciidoc b/filebeat/docs/modules/squid.asciidoc index 785fdac83cc..3f10e40ddd8 100644 --- a/filebeat/docs/modules/squid.asciidoc +++ b/filebeat/docs/modules/squid.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Squid module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/tomcat.asciidoc b/filebeat/docs/modules/tomcat.asciidoc index bf64a816b05..14aa52b2921 100644 --- a/filebeat/docs/modules/tomcat.asciidoc +++ b/filebeat/docs/modules/tomcat.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Tomcat module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/apache_tomcat[Apache Tomcat] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/zscaler.asciidoc b/filebeat/docs/modules/zscaler.asciidoc index 25f914cb088..b6adfd84400 100644 --- a/filebeat/docs/modules/zscaler.asciidoc +++ b/filebeat/docs/modules/zscaler.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Zscaler module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/zscaler_zia[Zscaler Internet Access] Elastic integration instead."] + experimental[] //temporarily override modulename to create working link diff --git a/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc b/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc index 79d27b36166..abca823d933 100644 --- a/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Barracuda module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/barracuda[Barracuda Web Application Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc b/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc index 3497b6873ea..33fe07fd426 100644 --- a/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Bluecoat module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index 6375c1999e8..eb4777ea0b4 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -274,6 +274,8 @@ include::../include/timezone-support.asciidoc[] [float] ==== `nexus` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_nexus[Cisco Nexus] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "cisconxos" device revision 134. @@ -319,6 +321,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `meraki` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_meraki[Cisco Meraki] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "ciscomeraki" device revision 118. diff --git a/x-pack/filebeat/module/cylance/_meta/docs.asciidoc b/x-pack/filebeat/module/cylance/_meta/docs.asciidoc index 4cd22f8b797..f8cff0616f4 100644 --- a/x-pack/filebeat/module/cylance/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cylance/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Cylance module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/f5/_meta/docs.asciidoc b/x-pack/filebeat/module/f5/_meta/docs.asciidoc index e83d14081ff..28d57c59be5 100644 --- a/x-pack/filebeat/module/f5/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/f5/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == F5 module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/f5_bigip[F5 BIG-IP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc index 2cd30c1437a..4b224e49019 100644 --- a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc @@ -74,6 +74,8 @@ events. Defaults to `[fortinet-firewall, forwarded]`. [float] ==== `clientendpoint` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_forticlient[Fortinet FortiClient Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "forticlientendpoint" device revision 0. @@ -119,6 +121,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimail` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimail[Fortinet FortiMail] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetfortimail" device revision 131. @@ -164,6 +168,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimanager` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimanager[Fortinet FortiManager Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetmgr" device revision 134. diff --git a/x-pack/filebeat/module/imperva/_meta/docs.asciidoc b/x-pack/filebeat/module/imperva/_meta/docs.asciidoc index c5b1ab37691..6639cabb9b7 100644 --- a/x-pack/filebeat/module/imperva/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/imperva/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Imperva module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc b/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc index 1be04b5b739..dc894b8f3c3 100644 --- a/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Infoblox module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/infoblox_nios[Infoblox NIOS] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/juniper/_meta/docs.asciidoc b/x-pack/filebeat/module/juniper/_meta/docs.asciidoc index 7de15937c4e..a33b9c03a83 100644 --- a/x-pack/filebeat/module/juniper/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/juniper/_meta/docs.asciidoc @@ -135,6 +135,8 @@ This is a list of JunOS fields that are mapped to ECS. [float] ==== `junos` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/juniper_srx[Juniper SRX] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "junosrouter" device revision 134. @@ -180,6 +182,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `netscreen` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. See <> for migration options."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "netscreen" device revision 134. diff --git a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc index b075caa035b..048354d95a8 100644 --- a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc @@ -217,6 +217,8 @@ And for all other Defender ATP event types, go to Host -> Events. [float] ==== `dhcp` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/microsoft_dhcp[Microsoft DHCP] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "msdhcp" device revision 99. diff --git a/x-pack/filebeat/module/netscout/_meta/docs.asciidoc b/x-pack/filebeat/module/netscout/_meta/docs.asciidoc index 463c93b5c0f..6ae6035957e 100644 --- a/x-pack/filebeat/module/netscout/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/netscout/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Netscout module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] This is a module for receiving Arbor Peakflow SP logs over Syslog or a file. diff --git a/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc b/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc index 21a768ab432..0e678fa7896 100644 --- a/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Proofpoint module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/proofpoint_tap[Proofpoint TAP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/radware/_meta/docs.asciidoc b/x-pack/filebeat/module/radware/_meta/docs.asciidoc index ba9bc87444c..834586f8535 100644 --- a/x-pack/filebeat/module/radware/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/radware/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Radware module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/snort/_meta/docs.asciidoc b/x-pack/filebeat/module/snort/_meta/docs.asciidoc index 3170a20db9d..025afa52f35 100644 --- a/x-pack/filebeat/module/snort/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/snort/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Snort module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/snort[Snort] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc b/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc index d1f82f17dd2..adccfd17799 100644 --- a/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Sonicwall module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/sonicwall[SonicWall Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/sophos/_meta/docs.asciidoc b/x-pack/filebeat/module/sophos/_meta/docs.asciidoc index 6d6147984a1..f998d7ce62c 100644 --- a/x-pack/filebeat/module/sophos/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/sophos/_meta/docs.asciidoc @@ -145,6 +145,8 @@ This is a list of SophosXG fields that are mapped to ECS. [float] ==== `utm` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "astarosg" device revision 123. diff --git a/x-pack/filebeat/module/squid/_meta/docs.asciidoc b/x-pack/filebeat/module/squid/_meta/docs.asciidoc index 10411c4f73d..25a8e1edc1e 100644 --- a/x-pack/filebeat/module/squid/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/squid/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Squid module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc b/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc index d56df0ba01b..cb7339f3388 100644 --- a/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Tomcat module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/apache_tomcat[Apache Tomcat] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc b/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc index 1c62e822558..8a1b07a1372 100644 --- a/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Zscaler module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/zscaler_zia[Zscaler Internet Access] Elastic integration instead."] + experimental[] //temporarily override modulename to create working link From 2a26f9af28835415d28ba8b439666ce1b9b12bc1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 26 Oct 2023 14:17:45 -0400 Subject: [PATCH 08/22] go.mod - Use github.com/apache/arrow/go/v12 v12.0.1 (#36953) Move to release tag instead of release candidate. https://github.com/apache/arrow/releases/tag/go%2Fv12.0.1 Closes #36949 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 0d59be5ef12..9615dd77f88 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2970,11 +2970,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/apache/arrow/go/v12 -Version: v12.0.1-0.20230605094802-c153c6d36ccf +Version: v12.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/apache/arrow/go/v12@v12.0.1-0.20230605094802-c153c6d36ccf/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/apache/arrow/go/v12@v12.0.1/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 037291b36cd..9bb1e6803aa 100644 --- a/go.mod +++ b/go.mod @@ -193,7 +193,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/Azure/go-autorest/autorest/adal v0.9.14 - github.com/apache/arrow/go/v12 v12.0.1-0.20230605094802-c153c6d36ccf + github.com/apache/arrow/go/v12 v12.0.1 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4 diff --git a/go.sum b/go.sum index 8d1a2265e37..038eeca63f6 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= -github.com/apache/arrow/go/v12 v12.0.1-0.20230605094802-c153c6d36ccf h1:s5MDQXJmEalr0Urt0rPlX5UAE2BcHTiex/2Lt2O9p84= -github.com/apache/arrow/go/v12 v12.0.1-0.20230605094802-c153c6d36ccf/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= +github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= +github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= From b0455f428addcb8bc1a3eadec94a6e7f229542cc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 26 Oct 2023 17:55:17 -0400 Subject: [PATCH 09/22] auditbeat/module - docs and comment nits (#36952) Really minor fixes to docs and comments in auditbeat/modules. As I was reading the code I was applying some changes, so I figured I should contribute them. Removing an unnecessary cast revealed an incorrect operation between `& ^` vs `&^`. It looks like this was supposed to be a bitwise clear `&^` operation to remove ConfigChange and AttributesModified before the lookup from actionOrderMap. --------- Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- .../docs/modules/file_integrity.asciidoc | 2 +- auditbeat/module/auditd/audit_linux.go | 3 +-- .../module/file_integrity/_meta/docs.asciidoc | 2 +- auditbeat/module/file_integrity/action.go | 2 +- auditbeat/module/file_integrity/config.go | 4 ++-- auditbeat/module/file_integrity/event.go | 8 +++---- .../module/file_integrity/file_parsers.go | 2 +- .../file_integrity/fileorigin_darwin.go | 24 ++++++++++--------- auditbeat/module/file_integrity/metricset.go | 2 +- 9 files changed, 25 insertions(+), 24 deletions(-) diff --git a/auditbeat/docs/modules/file_integrity.asciidoc b/auditbeat/docs/modules/file_integrity.asciidoc index ccc4289c521..a12c4df47ca 100644 --- a/auditbeat/docs/modules/file_integrity.asciidoc +++ b/auditbeat/docs/modules/file_integrity.asciidoc @@ -121,7 +121,7 @@ units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, *`max_file_size`*:: The maximum size of a file in bytes for which {beatname_uc} will compute hashes and run file parsers. Files larger than this size will not be hashed or analysed by configured file parsers. The default -value is 100 MiB. For convenience units can be specified as a suffix to the +value is 100 MiB. For convenience, units can be specified as a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. diff --git a/auditbeat/module/auditd/audit_linux.go b/auditbeat/module/auditd/audit_linux.go index baff3363bed..9a00b03c482 100644 --- a/auditbeat/module/auditd/audit_linux.go +++ b/auditbeat/module/auditd/audit_linux.go @@ -979,8 +979,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) { if c.SocketType == "" { return "", fmt.Errorf("failed to create audit client: %w", err) } - // Ignore errors if a socket type has been specified. It will fail during - // further setup and its necessary for unit tests to pass + // Ignore errors if a socket type has been specified. return c.SocketType, nil } defer client.Close() diff --git a/auditbeat/module/file_integrity/_meta/docs.asciidoc b/auditbeat/module/file_integrity/_meta/docs.asciidoc index 3c537a28091..0f32ef64f93 100644 --- a/auditbeat/module/file_integrity/_meta/docs.asciidoc +++ b/auditbeat/module/file_integrity/_meta/docs.asciidoc @@ -114,7 +114,7 @@ units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, *`max_file_size`*:: The maximum size of a file in bytes for which {beatname_uc} will compute hashes and run file parsers. Files larger than this size will not be hashed or analysed by configured file parsers. The default -value is 100 MiB. For convenience units can be specified as a suffix to the +value is 100 MiB. For convenience, units can be specified as a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. diff --git a/auditbeat/module/file_integrity/action.go b/auditbeat/module/file_integrity/action.go index 9cfb22747af..779db7941bd 100644 --- a/auditbeat/module/file_integrity/action.go +++ b/auditbeat/module/file_integrity/action.go @@ -155,7 +155,7 @@ func (action Action) InOrder(existedBefore, existsNow bool) ActionArray { hasConfigChange := action&ConfigChange != 0 hasUpdate := action&Updated != 0 hasAttrMod := action&AttributesModified != 0 - action = Action(int(action) & int(^(ConfigChange | AttributesModified))) + action = Action(int(action) &^ (ConfigChange | AttributesModified)) if hasAttrMod { action |= Updated } diff --git a/auditbeat/module/file_integrity/config.go b/auditbeat/module/file_integrity/config.go index 1edd19d9ba0..e431e640766 100644 --- a/auditbeat/module/file_integrity/config.go +++ b/auditbeat/module/file_integrity/config.go @@ -91,8 +91,8 @@ type Config struct { // Validate validates the config data and return an error explaining all the // problems with the config. This method modifies the given config. func (c *Config) Validate() error { - // Resolve symlinks and make filepaths absolute if possible - // anything that does not resolve will be logged during + // Resolve symlinks and make filepaths absolute if possible. + // Anything that does not resolve will be logged during // scanning and metric set collection. for i, p := range c.Paths { p, err := filepath.EvalSymlinks(p) diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index a86130d3ec8..fd4d68828a4 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -99,7 +99,7 @@ var typeNames = map[Type]string{ SymlinkType: "symlink", } -// Digest is a output of a hash function. +// Digest is an output of a hash function. type Digest []byte // String returns the digest value in lower-case hexadecimal form. @@ -110,7 +110,7 @@ func (d Digest) String() string { // MarshalText encodes the digest to a hexadecimal representation of itself. func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } -// Event describe the filesystem change and includes metadata about the file. +// Event describes the filesystem change and includes metadata about the file. type Event struct { Timestamp time.Time `json:"timestamp"` // Time of event. Path string `json:"path"` // The path associated with the event. @@ -119,7 +119,7 @@ type Event struct { Source Source `json:"source"` // Source of the event. Action Action `json:"action"` // Action (like created, updated). Hashes map[HashType]Digest `json:"hash,omitempty"` // File hashes. - ParserResults mapstr.M `json:"file,omitempty"` // Results from runnimg file parsers. + ParserResults mapstr.M `json:"file,omitempty"` // Results from running file parsers. // Metadata rtt time.Duration // Time taken to collect the info. @@ -142,7 +142,7 @@ type Metadata struct { Mode os.FileMode `json:"mode"` // Permissions SetUID bool `json:"setuid"` // setuid bit (POSIX only) SetGID bool `json:"setgid"` // setgid bit (POSIX only) - Origin []string `json:"origin"` // External origin info for the file (MacOS only) + Origin []string `json:"origin"` // External origin info for the file (macOS only) SELinux string `json:"selinux"` // security.selinux xattr value (Linux only) POSIXACLAccess []byte `json:"posix_acl_access"` // system.posix_acl_access xattr value (Linux only) } diff --git a/auditbeat/module/file_integrity/file_parsers.go b/auditbeat/module/file_integrity/file_parsers.go index 77f6e30451e..cb1d619f1ed 100644 --- a/auditbeat/module/file_integrity/file_parsers.go +++ b/auditbeat/module/file_integrity/file_parsers.go @@ -23,7 +23,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -// FileParser is a file analyser the provides enrichment for file.* fields. +// FileParser is a file analyser providing enrichment for file.* fields. type FileParser interface { Parse(dst mapstr.M, path string) error } diff --git a/auditbeat/module/file_integrity/fileorigin_darwin.go b/auditbeat/module/file_integrity/fileorigin_darwin.go index 1cf6b350d32..459951c8cda 100644 --- a/auditbeat/module/file_integrity/fileorigin_darwin.go +++ b/auditbeat/module/file_integrity/fileorigin_darwin.go @@ -47,27 +47,29 @@ var ( ) // GetFileOrigin fetches the kMDItemWhereFroms metadata for the given path. This -// is special metadata in the filesystem that encodes information of an external -// origin of this file. It is always encoded as a list of strings, with +// is special metadata in the filesystem that encodes information about the +// external origin of this file. It is always encoded as a list of strings, with // different meanings depending on the origin: // // For files downloaded from a web browser, the first string is the URL for // the source document. The second URL (optional), is the web address where the // download link was followed: -// [ "https://cdn.kernel.org/pub/linux/kernel/v4.x/ChangeLog-4.13.16", "https://www.kernel.org/" ] +// +// ["https://cdn.kernel.org/pub/linux/kernel/v4.x/ChangeLog-4.13.16", "https://www.kernel.org/"] // // For files or directories transferred via Airdrop, the origin is one string // with the name of the computer that sent the file: -// [ "Adrian's MacBook Pro" ] +// +// ["Adrian's MacBook Pro"] // // For files attached to e-mails (using Mail app), three strings are // returned: Sender address, subject and e-mail identifier: -// [ "Adrian Serrano \u003cadrian@elastic.co\u003e", -// -// "Sagrada Familia tickets", -// "message:%3CCAMZw10FD4fktC9qdJgLjwW=a8LM4gbJ44jFcaK8.BOWg1t4OwQ@elastic.co%3E" // -// ], +// [ +// "Adrian Serrano \u003cadrian@elastic.co\u003e", +// "Sagrada Familia tickets", +// "message:%3CCAMZw10FD4fktC9qdJgLjwW=a8LM4gbJ44jFcaK8.BOWg1t4OwQ@elastic.co%3E" +// ], // // For all other files the result is an empty (nil) list. func GetFileOrigin(path string) ([]string, error) { @@ -108,8 +110,8 @@ func GetFileOrigin(path string) ([]string, error) { return nil, fmt.Errorf("plist unmarshal failed: %w", err) } - // The returned list seems to be padded with empty strings when some of - // the fields are missing (i.e. no context URL). Get rid of trailing empty + // The returned list seems to be padded with empty strings when some + // fields are missing (i.e. no context URL). Get rid of trailing empty // strings: n := len(urls) for n > 0 && len(urls[n-1]) == 0 { diff --git a/auditbeat/module/file_integrity/metricset.go b/auditbeat/module/file_integrity/metricset.go index bcada27db9f..4000231fd33 100644 --- a/auditbeat/module/file_integrity/metricset.go +++ b/auditbeat/module/file_integrity/metricset.go @@ -380,7 +380,7 @@ func store(b datastore.Bucket, e *Event) error { return nil } -// load loads an Event from the datastore. It return a nil Event if the key was +// load loads an Event from the datastore. It returns a nil Event if the key was // not found. It returns an error if there was a failure reading from the // datastore or decoding the data. func load(b datastore.Bucket, path string) (*Event, error) { From 27792cfd516607db718af4975dd790353d9ae559 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Thu, 26 Oct 2023 23:56:52 +0200 Subject: [PATCH 10/22] [Docs] Update packetbeat-options.asciidoc (#36968) Fix packetbeat documentation to avoid '{beatname}' being shown instead of the correct beat name. --- packetbeat/docs/packetbeat-options.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index 4a74dd5593e..c5cb4d95d6b 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -57,7 +57,7 @@ packetbeat.interfaces.buffer_size_mb: 100 [float] === Windows Npcap installation options -On Windows {beatname} requires an Npcap DLL installation. This is provided by {beatname} +On Windows {beatname_uc} requires an Npcap DLL installation. This is provided by {beatname_uc} for users of the Elastic Licenced version. In some cases users may wish to use their own installed version. In order to do this the `packetbeat.npcap.never_install` option can be used. Setting this option to `true` will not attempt to install the From ae11b47ef38d5f1b307db27634f5d45aab6f0828 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 27 Oct 2023 13:13:13 -0400 Subject: [PATCH 11/22] Update to elastic-agent-libs v0.6.2. (#36969) * Update to elastic-agent-libs v0.6.2. Increases the version of golang.org/x/net to one that has the HTTP2 rapid reset fix. * Add changelog entry. * Updates for latest version of golang.org/x/net/publicsuffix * Update all remaining subdomains --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 24 ++-- go.mod | 12 +- go.sum | 24 ++-- .../squid/log/test/access1.log-expected.json | 110 +++++++++--------- 5 files changed, 86 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 74a66ef3c23..37763f34d56 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -174,6 +174,7 @@ is collected by it. - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] +- Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 9615dd77f88..ea5514da6eb 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12712,11 +12712,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.6.0 +Version: v0.6.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.6.2/LICENSE: Apache License Version 2.0, January 2004 @@ -24738,11 +24738,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.12.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24849,11 +24849,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.12.0 +Version: v0.17.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.17.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24960,11 +24960,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.12.0 +Version: v0.13.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.13.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24997,11 +24997,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.12.0 +Version: v0.13.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -51107,11 +51107,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.11.0 +Version: v0.13.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/go.mod b/go.mod index 9bb1e6803aa..55ef8b4acc5 100644 --- a/go.mod +++ b/go.mod @@ -152,14 +152,14 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.25.0 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.14.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.10.0 - golang.org/x/net v0.12.0 + golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.10.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.12.0 - golang.org/x/text v0.12.0 + golang.org/x/sys v0.13.0 + golang.org/x/text v0.13.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.9.1 google.golang.org/api v0.126.0 @@ -202,7 +202,7 @@ require ( github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.6.4 - github.com/elastic/elastic-agent-libs v0.6.0 + github.com/elastic/elastic-agent-libs v0.6.2 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.7.0 github.com/elastic/go-elasticsearch/v8 v8.10.0 @@ -365,7 +365,7 @@ require ( go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect - golang.org/x/term v0.11.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect diff --git a/go.sum b/go.sum index 038eeca63f6..df5741e581b 100644 --- a/go.sum +++ b/go.sum @@ -654,8 +654,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+e github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= -github.com/elastic/elastic-agent-libs v0.6.0 h1:HnL/OpAzIHlK8y1J69XQuAx4tlCzd6e2kldMHvXARvY= -github.com/elastic/elastic-agent-libs v0.6.0/go.mod h1:K6U+n84siZ66ZyG36h1/x+fw1oIZbFXEypAC6KSiFOg= +github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3+oKIQDzdPDo= +github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ= @@ -2028,8 +2028,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2171,8 +2171,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2342,8 +2342,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2352,8 +2352,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2368,8 +2368,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index d7620e20648..f993048245a 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -4825,9 +4825,9 @@ "rsa.time.event_time_str": "1157689377", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 233, "source.ip": "10.105.33.214", @@ -4839,10 +4839,10 @@ "url.extension": "js", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", "url.path": "/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -4887,9 +4887,9 @@ "rsa.time.event_time_str": "1157689377", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 236, "source.ip": "10.105.33.214", @@ -4901,10 +4901,10 @@ "url.extension": "css", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", "url.path": "/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -4949,9 +4949,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -4963,10 +4963,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5068,9 +5068,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 136, "source.ip": "10.105.33.214", @@ -5082,10 +5082,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", "url.path": "/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5130,9 +5130,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 237, "source.ip": "10.105.33.214", @@ -5144,10 +5144,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5190,9 +5190,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 218, "source.ip": "10.105.33.214", @@ -5204,10 +5204,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", "url.path": "/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5252,9 +5252,9 @@ "rsa.time.event_time_str": "1157689379", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -5266,10 +5266,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5314,9 +5314,9 @@ "rsa.time.event_time_str": "1157689379", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -5328,10 +5328,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", "url.path": "/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5376,9 +5376,9 @@ "rsa.time.event_time_str": "1157689380", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -5390,10 +5390,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -6059,4 +6059,4 @@ "url.top_level_domain": "com", "user.name": "badeyek" } -] \ No newline at end of file +] From a0669d2d9c96ee28709a88fe29a502e325675c90 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Fri, 27 Oct 2023 11:26:42 -0700 Subject: [PATCH 12/22] Fix pipeline setup under serverless (#36973) * fix pipeline setup under serverless * add docs, fix errors * fix fix linter --- filebeat/fileset/pipelines.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/filebeat/fileset/pipelines.go b/filebeat/fileset/pipelines.go index 07d6c4c21ac..c58a7c13edc 100644 --- a/filebeat/fileset/pipelines.go +++ b/filebeat/fileset/pipelines.go @@ -37,6 +37,7 @@ type PipelineLoader interface { LoadJSON(path string, json map[string]interface{}) ([]byte, error) Request(method, path string, pipeline string, params map[string]string, body interface{}) (int, []byte, error) GetVersion() version.V + IsServerless() bool } // MultiplePipelineUnsupportedError is an error returned when a fileset uses multiple pipelines but is @@ -65,16 +66,17 @@ func (reg *ModuleRegistry) LoadPipelines(esClient PipelineLoader, overwrite bool // check that all the required Ingest Node plugins are available requiredProcessors := fileset.GetRequiredProcessors() reg.log.Debugf("Required processors: %s", requiredProcessors) - if len(requiredProcessors) > 0 { + // APIs do not exist on serverless + if len(requiredProcessors) > 0 && !esClient.IsServerless() { err := checkAvailableProcessors(esClient, requiredProcessors) if err != nil { - return fmt.Errorf("error loading pipeline for fileset %s/%s: %v", module.config.Module, fileset.name, err) + return fmt.Errorf("error loading pipeline for fileset %s/%s: %w", module.config.Module, fileset.name, err) } } pipelines, err := fileset.GetPipelines(esClient.GetVersion()) if err != nil { - return fmt.Errorf("error getting pipeline for fileset %s/%s: %v", module.config.Module, fileset.name, err) + return fmt.Errorf("error getting pipeline for fileset %s/%s: %w", module.config.Module, fileset.name, err) } // Filesets with multiple pipelines can only be supported by Elasticsearch >= 6.5.0 @@ -88,7 +90,7 @@ func (reg *ModuleRegistry) LoadPipelines(esClient PipelineLoader, overwrite bool for _, pipeline := range pipelines { err = LoadPipeline(esClient, pipeline.id, pipeline.contents, overwrite, reg.log.With("pipeline", pipeline.id)) if err != nil { - err = fmt.Errorf("error loading pipeline for fileset %s/%s: %v", module.config.Module, fileset.name, err) + err = fmt.Errorf("error loading pipeline for fileset %s/%s: %w", module.config.Module, fileset.name, err) break } pipelineIDsLoaded = append(pipelineIDsLoaded, pipeline.id) @@ -169,7 +171,7 @@ func interpretError(initialErr error, body []byte) error { "This is the response I got from Elasticsearch: %s", body) } - return fmt.Errorf("couldn't load pipeline: %v. Additionally, error decoding response body: %s", + return fmt.Errorf("couldn't load pipeline: %w. Additionally, error decoding response body: %s", initialErr, body) } @@ -194,5 +196,5 @@ func interpretError(initialErr error, body []byte) error { "This is the response I got from Elasticsearch: %s", body) } - return fmt.Errorf("couldn't load pipeline: %v. Response body: %s", initialErr, body) + return fmt.Errorf("couldn't load pipeline: %w. Response body: %s", initialErr, body) } From fe5088338011c48230e7d6aa6ce07fed5a895e5c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 27 Oct 2023 14:49:53 -0400 Subject: [PATCH 13/22] Auditbeat,Metricbeat - add /inputs/ to HTTP monitoring endpoint (#36971) Make metrics published by "inputs" available through the /inputs/ route on the HTTP monitoring endpoint of Auditbeat and Metricbeat. For Agent, include a snapshot of those metrics within the Agent diagnostics bundle as "input_metrics.json". When running under Agent, each module instance is configured with only a single metricset. That module is given a unique `id`. That ID is what will be used as the `id` within the /inputs/ data. And that `id` will also be added as context to the logger that is passed into every metricset so that any log messages from a metricset can be associated back to the agent stream ID). Relates #36945 Remove module and metricset keys from metricset metrics. For the `/inputs/` API, `input` is they key used to identify the type of "input" running. The `module` and `metricset` keys become redundant with the addition of `input`. I don't know of anything that relies on those fields. --- CHANGELOG.next.asciidoc | 2 ++ metricbeat/beater/metricbeat.go | 19 +++++++++++++++++++ metricbeat/mb/builders.go | 21 +++++++++++++++++---- metricbeat/mb/mb.go | 5 +++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 37763f34d56..baf3dd3e7ec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -253,6 +253,7 @@ is collected by it. *Auditbeat* - Upgrade go-libaudit to v2.4.0. {issue}36776[36776] {pull}36964[36964] +- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each dataset instance. {pull}36971[36971] *Libbeat* @@ -268,6 +269,7 @@ is collected by it. - Add GCP Carbon Footprint metricbeat data {pull}34820[34820] - Add event loop utilization metric to Kibana module {pull}35020[35020] - Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms {pull}36647[36647] +- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971] *Osquerybeat* diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index cbc44f88bf5..acd4aa02b1e 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/module" conf "github.com/elastic/elastic-agent-libs/config" @@ -155,6 +156,24 @@ func newMetricbeat(b *beat.Beat, c *conf.C, options ...Option) (*Metricbeat, err return metricbeat, nil } + if b.API != nil { + if err := inputmon.AttachHandler(b.API.Router()); err != nil { + return nil, fmt.Errorf("failed attach inputs api to monitoring endpoint server: %w", err) + } + } + + if b.Manager != nil { + b.Manager.RegisterDiagnosticHook("input_metrics", "Metrics from active inputs.", + "input_metrics.json", "application/json", func() []byte { + data, err := inputmon.MetricSnapshotJSON() + if err != nil { + logp.L().Warnw("Failed to collect input metric snapshot for Agent diagnostics.", "error", err) + return []byte(err.Error()) + } + return data + }) + } + moduleOptions := append( []module.Option{module.WithMaxStartDelay(config.MaxStartDelay)}, metricbeat.moduleOptions...) diff --git a/metricbeat/mb/builders.go b/metricbeat/mb/builders.go index 269c194063c..c9b1ace587d 100644 --- a/metricbeat/mb/builders.go +++ b/metricbeat/mb/builders.go @@ -182,20 +182,33 @@ func newBaseMetricSets(r *Register, m Module) ([]BaseMetricSet, error) { } msID := id.String() metrics := monitoring.NewRegistry() - monitoring.NewString(metrics, "module").Set(m.Name()) - monitoring.NewString(metrics, "metricset").Set(name) + monitoring.NewString(metrics, "input").Set(m.Name() + "/" + name) if host != "" { monitoring.NewString(metrics, "host").Set(host) } - monitoring.NewString(metrics, "id").Set(msID) + monitoring.NewString(metrics, "ephemeral_id").Set(msID) + if configuredID := m.Config().ID; configuredID != "" { + // If a module ID was configured, then use that as the ID within metrics. + // Note that the "ephemeral_id" is what is used as the monitoring registry + // key. This module ID is not unique to the MetricSet instance when multiple + // hosts are monitored or if multiple different MetricSet types were enabled + // under the same module instance. + monitoring.NewString(metrics, "id").Set(configuredID) + } else { + monitoring.NewString(metrics, "id").Set(msID) + } + logger := logp.NewLogger(m.Name() + "." + name) + if m.Config().ID != "" { + logger = logger.With("id", m.Config().ID) + } metricsets = append(metricsets, BaseMetricSet{ id: msID, name: name, module: m, host: host, metrics: metrics, - logger: logp.NewLogger(m.Name() + "." + name), + logger: logger, }) } } diff --git a/metricbeat/mb/mb.go b/metricbeat/mb/mb.go index 06b85662838..7e18dc9029d 100644 --- a/metricbeat/mb/mb.go +++ b/metricbeat/mb/mb.go @@ -362,6 +362,7 @@ func (b *BaseMetricSet) Registration() MetricSetRegistration { // the metricset fetches not only the predefined fields but add alls raw data under // the raw namespace to the event. type ModuleConfig struct { + ID string `config:"id"` // Optional ID (not guaranteed to be unique). Hosts []string `config:"hosts"` Period time.Duration `config:"period" validate:"positive"` Timeout time.Duration `config:"timeout" validate:"positive"` @@ -375,8 +376,8 @@ type ModuleConfig struct { func (c ModuleConfig) String() string { return fmt.Sprintf(`{Module:"%v", MetricSets:%v, Enabled:%v, `+ - `Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`, - c.Module, c.MetricSets, c.Enabled, len(c.Hosts), c.Period, c.Timeout, + `ID:"%s", Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`, + c.Module, c.MetricSets, c.Enabled, c.ID, len(c.Hosts), c.Period, c.Timeout, c.Raw, c.Query) } From 4453d992cc659ef0655cf6cc19488da66135b193 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Sat, 28 Oct 2023 06:50:26 +1030 Subject: [PATCH 14/22] auditbeat/module/auditd: add ignore_errors config option (#36851) Setting ignore_errors to true allows incompletely valid rule sets to be used in a configuration. This is equivalent to the -i flag of auditctl. --- CHANGELOG.next.asciidoc | 1 + auditbeat/docs/modules/auditd.asciidoc | 3 + auditbeat/module/auditd/_meta/docs.asciidoc | 3 + .../auditd/{config_linux.go => config.go} | 37 ++- auditbeat/module/auditd/config_linux_test.go | 254 ---------------- auditbeat/module/auditd/config_test.go | 279 ++++++++++++++++++ 6 files changed, 316 insertions(+), 261 deletions(-) rename auditbeat/module/auditd/{config_linux.go => config.go} (83%) delete mode 100644 auditbeat/module/auditd/config_linux_test.go create mode 100644 auditbeat/module/auditd/config_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index baf3dd3e7ec..e833f99d0b1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -178,6 +178,7 @@ is collected by it. *Auditbeat* +- Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851] *Filebeat* diff --git a/auditbeat/docs/modules/auditd.asciidoc b/auditbeat/docs/modules/auditd.asciidoc index a0d2693487e..9204e243f64 100644 --- a/auditbeat/docs/modules/auditd.asciidoc +++ b/auditbeat/docs/modules/auditd.asciidoc @@ -212,6 +212,9 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are supported and will expand in lexicographical order. The format is the same as that of the `audit_rules` field. +*`ignore_errors`*:: This setting allows errors during rule loading and parsing +to be ignored, but logged as warnings. + *`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to prevent backpressure from propagating to the kernel and impacting audited processes. diff --git a/auditbeat/module/auditd/_meta/docs.asciidoc b/auditbeat/module/auditd/_meta/docs.asciidoc index 587a40dd982..b1dd7d87c63 100644 --- a/auditbeat/module/auditd/_meta/docs.asciidoc +++ b/auditbeat/module/auditd/_meta/docs.asciidoc @@ -205,6 +205,9 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are supported and will expand in lexicographical order. The format is the same as that of the `audit_rules` field. +*`ignore_errors`*:: This setting allows errors during rule loading and parsing +to be ignored, but logged as warnings. + *`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to prevent backpressure from propagating to the kernel and impacting audited processes. diff --git a/auditbeat/module/auditd/config_linux.go b/auditbeat/module/auditd/config.go similarity index 83% rename from auditbeat/module/auditd/config_linux.go rename to auditbeat/module/auditd/config.go index 4a03584c3b7..6762a3924a9 100644 --- a/auditbeat/module/auditd/config_linux.go +++ b/auditbeat/module/auditd/config.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build unix + package auditd import ( @@ -30,6 +32,7 @@ import ( "github.com/joeshaw/multierror" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-libaudit/v2/rule" "github.com/elastic/go-libaudit/v2/rule/flags" ) @@ -46,6 +49,7 @@ type Config struct { RuleFiles []string `config:"audit_rule_files"` // List of rule files. SocketType string `config:"socket_type"` // Socket type to use with the kernel (unicast or multicast). Immutable bool `config:"immutable"` // Sets kernel audit config immutable. + IgnoreErrors bool `config:"ignore_errors"` // Ignore errors when reading and parsing rules, equivalent to auditctl -i. // Tuning options (advanced, use with care) ReassemblerMaxInFlight uint32 `config:"reassembler.max_in_flight"` @@ -120,11 +124,19 @@ func (c Config) rules() []auditRule { } func (c *Config) loadRules() error { + var log *logp.Logger + if c.IgnoreErrors { + log = logp.NewLogger(moduleName) + } + var paths []string for _, pattern := range c.RuleFiles { absPattern, err := filepath.Abs(pattern) if err != nil { - return fmt.Errorf("unable to get the absolute path for %s: %w", pattern, err) + if log == nil { + return fmt.Errorf("unable to get the absolute path for %s: %w", pattern, err) + } + log.Warnf("unable to get the absolute path for %s: %v", pattern, err) } files, err := filepath.Glob(absPattern) if err != nil { @@ -136,7 +148,7 @@ func (c *Config) loadRules() error { knownRules := ruleSet{} - rules, err := readRules(bytes.NewBufferString(c.RulesBlob), "(audit_rules at auditbeat.yml)", knownRules) + rules, err := readRules(bytes.NewBufferString(c.RulesBlob), "(audit_rules at auditbeat.yml)", knownRules, log) if err != nil { return err } @@ -145,9 +157,13 @@ func (c *Config) loadRules() error { for _, filename := range paths { fHandle, err := os.Open(filename) if err != nil { - return fmt.Errorf("unable to open rule file '%s': %w", filename, err) + if log == nil { + return fmt.Errorf("unable to open rule file '%s': %w", filename, err) + } + log.Warnf("unable to open rule file '%s': %v", filename, err) + continue } - rules, err = readRules(fHandle, filename, knownRules) + rules, err = readRules(fHandle, filename, knownRules, log) if err != nil { return err } @@ -170,7 +186,11 @@ func (c Config) failureMode() (uint32, error) { } } -func readRules(reader io.Reader, source string, knownRules ruleSet) (rules []auditRule, err error) { +// readRules reads the audit rules from reader, adding them to knownRules. If +// log is nil, errors will result in an empty rules set being returned. Otherwise +// errors will be logged as warnings and any successfully parsed rules will be +// returned. +func readRules(reader io.Reader, source string, knownRules ruleSet, log *logp.Logger) (rules []auditRule, err error) { var errs multierror.Errors s := bufio.NewScanner(reader) @@ -207,8 +227,11 @@ func readRules(reader io.Reader, source string, knownRules ruleSet) (rules []aud rules = append(rules, rule) } - if len(errs) > 0 { - return nil, fmt.Errorf("failed loading rules: %w", errs.Err()) + if len(errs) != 0 { + if log == nil { + return nil, fmt.Errorf("failed loading rules: %w", errs.Err()) + } + log.Warnf("errors loading rules: %v", errs.Err()) } return rules, nil } diff --git a/auditbeat/module/auditd/config_linux_test.go b/auditbeat/module/auditd/config_linux_test.go deleted file mode 100644 index d48eea8c6c8..00000000000 --- a/auditbeat/module/auditd/config_linux_test.go +++ /dev/null @@ -1,254 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package auditd - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - conf "github.com/elastic/elastic-agent-libs/config" -) - -func TestConfigValidate(t *testing.T) { - data := ` -audit_rules: | - # Comments and empty lines are ignored. - -w /etc/passwd -p wa -k auth - - -a always,exit -S execve -k exec` - - config, err := parseConfig(t, data) - if err != nil { - t.Fatal(err) - } - rules := config.rules() - - assert.EqualValues(t, []string{ - "-w /etc/passwd -p wa -k auth", - "-a always,exit -S execve -k exec", - }, commands(rules)) -} - -func TestConfigValidateWithError(t *testing.T) { - data := ` -audit_rules: | - -x bad -F flag - -a always,exit -w /etc/passwd - -a always,exit -S fake -k exec` - - _, err := parseConfig(t, data) - if err == nil { - t.Fatal("expected error") - } - t.Log(err) -} - -func TestConfigValidateWithDuplicates(t *testing.T) { - data := ` -audit_rules: | - -w /etc/passwd -p rwxa -k auth - -w /etc/passwd -k auth` - - _, err := parseConfig(t, data) - if err == nil { - t.Fatal("expected error") - } - t.Log(err) -} - -func TestConfigValidateFailureMode(t *testing.T) { - config := defaultConfig - config.FailureMode = "boom" - err := config.Validate() - assert.Error(t, err) - t.Log(err) -} - -func TestConfigValidateConnectionType(t *testing.T) { - config := defaultConfig - config.SocketType = "Satellite" - err := config.Validate() - assert.Error(t, err) - t.Log(err) -} - -func TestConfigValidateImmutable(t *testing.T) { - tcs := []struct { - name string - socketType string - mustFail bool - }{ - { - name: "Must pass for default", - socketType: "", - mustFail: false, - }, - { - name: "Must pass for unicast", - socketType: "unicast", - mustFail: false, - }, - { - name: "Must fail for multicast", - socketType: "multicast", - mustFail: true, - }, - } - - for _, tc := range tcs { - tc := tc - t.Run(tc.name, func(t *testing.T) { - config := defaultConfig - config.SocketType = tc.socketType - config.Immutable = true - err := config.Validate() - if tc.mustFail { - assert.Error(t, err) - t.Log(err) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestConfigRuleOrdering(t *testing.T) { - const fileMode = 0o644 - config := defaultConfig - config.RulesBlob = strings.Join([]string{ - makeRuleFlags(0, 0), - makeRuleFlags(0, 1), - makeRuleFlags(0, 2), - }, "\n") - - dir1, err := ioutil.TempDir("", "rules1") - if err != nil { - t.Fatal(err) - } - - for _, file := range []struct { - order int - name string - }{ - {0, "00_first.conf"}, - {5, "99_last.conf"}, - {2, "03_auth.conf"}, - {4, "20_exec.conf"}, - {3, "10_network_access.conf"}, - {1, "01_32bit_abi.conf"}, - } { - path := filepath.Join(dir1, file.name) - content := []byte(strings.Join([]string{ - makeRuleFlags(1+file.order, 0), - makeRuleFlags(1+file.order, 1), - makeRuleFlags(1+file.order, 2), - makeRuleFlags(1+file.order, 3), - }, "\n")) - if err = ioutil.WriteFile(path, content, fileMode); err != nil { - t.Fatal(err) - } - } - - dir2, err := ioutil.TempDir("", "rules0") - if err != nil { - t.Fatal(err) - } - - for _, file := range []struct { - order int - name string - }{ - {3, "99_tail.conf"}, - {0, "00_head.conf"}, - {2, "50_mid.conf"}, - {1, "13.conf"}, - } { - path := filepath.Join(dir2, file.name) - content := []byte(strings.Join([]string{ - makeRuleFlags(10+file.order, 0), - makeRuleFlags(10+file.order, 1), - makeRuleFlags(10+file.order, 2), - makeRuleFlags(10+file.order, 3), - }, "\n")) - if err = ioutil.WriteFile(path, content, fileMode); err != nil { - t.Fatal(err) - } - } - - config.RuleFiles = []string{ - fmt.Sprintf("%s/*.conf", dir1), - fmt.Sprintf("%s/*.conf", dir2), - } - - if err = config.Validate(); err != nil { - t.Fatal(err) - } - - rules := config.rules() - fileNo, ruleNo := 0, 0 - for _, rule := range rules { - parts := strings.Split(rule.flags, " ") - assert.Len(t, parts, 6, rule.flags) - fields := strings.Split(parts[5], ":") - assert.Len(t, fields, 3, rule.flags) - fileID, err := strconv.Atoi(fields[1]) - if err != nil { - t.Fatal(err, rule.flags) - } - ruleID, err := strconv.Atoi(fields[2]) - if err != nil { - t.Fatal(err, rule.flags) - } - if fileID > fileNo { - fileNo = fileID - ruleNo = 0 - } - assert.Equal(t, fileNo, fileID, rule.flags) - assert.Equal(t, ruleNo, ruleID, rule.flags) - ruleNo++ - } -} - -func makeRuleFlags(fileID, ruleID int) string { - return fmt.Sprintf("-w /path/%d/%d -p rwxa -k rule:%d:%d", fileID, ruleID, fileID, ruleID) -} - -func parseConfig(t testing.TB, yaml string) (Config, error) { - c, err := conf.NewConfigWithYAML([]byte(yaml), "") - if err != nil { - t.Fatal(err) - } - - config := defaultConfig - err = c.Unpack(&config) - return config, err -} - -func commands(rules []auditRule) []string { - var cmds []string - for _, r := range rules { - cmds = append(cmds, r.flags) - } - return cmds -} diff --git a/auditbeat/module/auditd/config_test.go b/auditbeat/module/auditd/config_test.go new file mode 100644 index 00000000000..81da2d9b85b --- /dev/null +++ b/auditbeat/module/auditd/config_test.go @@ -0,0 +1,279 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build unix + +package auditd + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestConfig(t *testing.T) { + logp.TestingSetup() + + t.Run("Validate", func(t *testing.T) { + data := ` +audit_rules: | + # Comments and empty lines are ignored. + -w /etc/passwd -p wa -k auth + + -a always,exit -S execve -k exec` + + config, err := parseConfig(t, data) + if err != nil { + t.Fatal(err) + } + rules := config.rules() + + assert.EqualValues(t, []string{ + "-w /etc/passwd -p wa -k auth", + "-a always,exit -S execve -k exec", + }, commands(rules)) + }) + + t.Run("ValidateWithError", func(t *testing.T) { + data := ` +audit_rules: | + -x bad -F flag + -a always,exit -w /etc/passwd + -a always,exit -S fake -k exec` + + _, err := parseConfig(t, data) + if err == nil { + t.Fatal("expected error") + } + t.Log(err) + }) + + t.Run("ValidateWithErrorIgnored", func(t *testing.T) { + data := ` +ignore_errors: true +audit_rules: | + -x bad -F flag + -a always,exit -w /etc/passwd + -a always,exit -S fake -k exec + -w /etc/passwd -k auth` + + cfg, err := parseConfig(t, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cfg.auditRules) != 1 { + t.Fatalf("unexpected number of rules from parseConfig: got %d, want %d", len(cfg.auditRules), 1) + } + }) + + t.Run("ValidateWithDuplicates", func(t *testing.T) { + data := ` +audit_rules: | + -w /etc/passwd -p rwxa -k auth + -w /etc/passwd -k auth` + + _, err := parseConfig(t, data) + if err == nil { + t.Fatal("expected error") + } + t.Log(err) + }) + + t.Run("ValidateFailureMode", func(t *testing.T) { + config := defaultConfig + config.FailureMode = "boom" + err := config.Validate() + assert.Error(t, err) + t.Log(err) + }) + + t.Run("ValidateConnectionType", func(t *testing.T) { + config := defaultConfig + config.SocketType = "Satellite" + err := config.Validate() + assert.Error(t, err) + t.Log(err) + }) + + t.Run("ValidateImmutable", func(t *testing.T) { + tcs := []struct { + name string + socketType string + mustFail bool + }{ + { + name: "Must pass for default", + socketType: "", + mustFail: false, + }, + { + name: "Must pass for unicast", + socketType: "unicast", + mustFail: false, + }, + { + name: "Must fail for multicast", + socketType: "multicast", + mustFail: true, + }, + } + + for _, tc := range tcs { + tc := tc + t.Run(tc.name, func(t *testing.T) { + config := defaultConfig + config.SocketType = tc.socketType + config.Immutable = true + err := config.Validate() + if tc.mustFail { + assert.Error(t, err) + t.Log(err) + } else { + assert.NoError(t, err) + } + }) + } + }) + + t.Run("RuleOrdering", func(t *testing.T) { + const fileMode = 0o644 + config := defaultConfig + config.RulesBlob = strings.Join([]string{ + makeRuleFlags(0, 0), + makeRuleFlags(0, 1), + makeRuleFlags(0, 2), + }, "\n") + + dir1, err := ioutil.TempDir("", "rules1") + if err != nil { + t.Fatal(err) + } + + for _, file := range []struct { + order int + name string + }{ + {0, "00_first.conf"}, + {5, "99_last.conf"}, + {2, "03_auth.conf"}, + {4, "20_exec.conf"}, + {3, "10_network_access.conf"}, + {1, "01_32bit_abi.conf"}, + } { + path := filepath.Join(dir1, file.name) + content := []byte(strings.Join([]string{ + makeRuleFlags(1+file.order, 0), + makeRuleFlags(1+file.order, 1), + makeRuleFlags(1+file.order, 2), + makeRuleFlags(1+file.order, 3), + }, "\n")) + if err = ioutil.WriteFile(path, content, fileMode); err != nil { + t.Fatal(err) + } + } + + dir2, err := ioutil.TempDir("", "rules0") + if err != nil { + t.Fatal(err) + } + + for _, file := range []struct { + order int + name string + }{ + {3, "99_tail.conf"}, + {0, "00_head.conf"}, + {2, "50_mid.conf"}, + {1, "13.conf"}, + } { + path := filepath.Join(dir2, file.name) + content := []byte(strings.Join([]string{ + makeRuleFlags(10+file.order, 0), + makeRuleFlags(10+file.order, 1), + makeRuleFlags(10+file.order, 2), + makeRuleFlags(10+file.order, 3), + }, "\n")) + if err = ioutil.WriteFile(path, content, fileMode); err != nil { + t.Fatal(err) + } + } + + config.RuleFiles = []string{ + fmt.Sprintf("%s/*.conf", dir1), + fmt.Sprintf("%s/*.conf", dir2), + } + + if err = config.Validate(); err != nil { + t.Fatal(err) + } + + rules := config.rules() + fileNo, ruleNo := 0, 0 + for _, rule := range rules { + parts := strings.Split(rule.flags, " ") + assert.Len(t, parts, 6, rule.flags) + fields := strings.Split(parts[5], ":") + assert.Len(t, fields, 3, rule.flags) + fileID, err := strconv.Atoi(fields[1]) + if err != nil { + t.Fatal(err, rule.flags) + } + ruleID, err := strconv.Atoi(fields[2]) + if err != nil { + t.Fatal(err, rule.flags) + } + if fileID > fileNo { + fileNo = fileID + ruleNo = 0 + } + assert.Equal(t, fileNo, fileID, rule.flags) + assert.Equal(t, ruleNo, ruleID, rule.flags) + ruleNo++ + } + }) +} + +func makeRuleFlags(fileID, ruleID int) string { + return fmt.Sprintf("-w /path/%d/%d -p rwxa -k rule:%d:%d", fileID, ruleID, fileID, ruleID) +} + +func parseConfig(t testing.TB, yaml string) (Config, error) { + c, err := conf.NewConfigWithYAML([]byte(yaml), "") + if err != nil { + t.Fatal(err) + } + + config := defaultConfig + err = c.Unpack(&config) + return config, err +} + +func commands(rules []auditRule) []string { + var cmds []string + for _, r := range rules { + cmds = append(cmds, r.flags) + } + return cmds +} From 9dd92030bd796db3724828f318cff0ff15249279 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:15:48 -0400 Subject: [PATCH 15/22] [updatecli] update elastic stack version for testing 8.12.0-fa958a90 (#36864) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3d7bbd0bacb..28bb4bb0c7f 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-cb808527-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-fa958a90-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-cb808527-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-fa958a90-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-cb808527-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-fa958a90-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From adcd4b0c7cc2ef43b4b95ff5ed73d2c9f5a58548 Mon Sep 17 00:00:00 2001 From: Richa Talwar <102972658+ritalwar@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:03:20 +0530 Subject: [PATCH 16/22] Fix event mapping implementation for statsd module (#36925) * Fix eventmapping implementation for statsd module. --- CHANGELOG.next.asciidoc | 1 + .../module/airflow/statsd/_meta/data.json | 8 ++-- .../module/airflow/statsd/data_test.go | 16 +++---- .../metricbeat/module/statsd/server/data.go | 20 +++++---- .../module/statsd/server/data_test.go | 22 +++++++--- .../metricbeat/module/statsd/server/server.go | 42 +++++++++++-------- 6 files changed, 64 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e833f99d0b1..eef14a45f56 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -143,6 +143,7 @@ is collected by it. - Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413] - Add log error when statsd server fails to start {pull}36477[36477] - Fix CassandraConnectionClosures metric configuration {pull}34742[34742] +- Fix event mapping implementation for statsd module {pull}36925[36925] *Osquerybeat* diff --git a/x-pack/metricbeat/module/airflow/statsd/_meta/data.json b/x-pack/metricbeat/module/airflow/statsd/_meta/data.json index 4e5c41437cb..ff89b0ddde2 100644 --- a/x-pack/metricbeat/module/airflow/statsd/_meta/data.json +++ b/x-pack/metricbeat/module/airflow/statsd/_meta/data.json @@ -2,13 +2,13 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "airflow": { "dag_duration": { - "15m_rate": 0.2, - "1m_rate": 0.2, - "5m_rate": 0.2, + "15m_rate": 0, + "1m_rate": 0, + "5m_rate": 0, "count": 1, "max": 200, "mean": 200, - "mean_rate": 0.2222490946071946, + "mean_rate": 38960.532980091164, "median": 200, "min": 200, "p75": 200, diff --git a/x-pack/metricbeat/module/airflow/statsd/data_test.go b/x-pack/metricbeat/module/airflow/statsd/data_test.go index c2c07d32f34..fcfd943bc68 100644 --- a/x-pack/metricbeat/module/airflow/statsd/data_test.go +++ b/x-pack/metricbeat/module/airflow/statsd/data_test.go @@ -11,15 +11,14 @@ import ( "sync" "testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" - + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/auditbeat/core" _ "github.com/elastic/beats/v7/libbeat/processors/actions" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" ) func init() { @@ -42,14 +41,14 @@ func getConfig() map[string]interface{} { } } -func createEvent(t *testing.T) { +func createEvent(data string, t *testing.T) { udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", STATSD_HOST, STATSD_PORT)) require.NoError(t, err) conn, err := net.DialUDP("udp", nil, udpAddr) require.NoError(t, err) - _, err = fmt.Fprint(conn, "dagrun.duration.failed.a_dagid:200|ms|#k1:v1,k2:v2") + _, err = fmt.Fprint(conn, data) require.NoError(t, err) } @@ -70,15 +69,16 @@ func TestData(t *testing.T) { wg.Done() go ms.Run(reporter) - events = reporter.(*mbtest.CapturingPushReporterV2).BlockingCapture(1) + events = reporter.(*mbtest.CapturingPushReporterV2).BlockingCapture(2) close(done) }(wg) wg.Wait() - createEvent(t) + createEvent("dagrun.duration.failed.a_dagid:200|ms|#k1:v1,k2:v2", t) + createEvent("dagrun.duration.failed.b_dagid:500|ms|#k1:v1,k2:v2", t) <-done - + assert.Len(t, events, 2) if len(events) == 0 { t.Fatal("received no events") } diff --git a/x-pack/metricbeat/module/statsd/server/data.go b/x-pack/metricbeat/module/statsd/server/data.go index bac70457fd2..27024e26284 100644 --- a/x-pack/metricbeat/module/statsd/server/data.go +++ b/x-pack/metricbeat/module/statsd/server/data.go @@ -102,18 +102,18 @@ func parse(b []byte) ([]statsdMetric, error) { return metrics, nil } -func eventMapping(metricName string, metricValue interface{}, metricSetFields mapstr.M, mappings map[string]StatsdMapping) { +func eventMapping(metricName string, metricValue interface{}, mappings map[string]StatsdMapping) mapstr.M { + m := mapstr.M{} if len(mappings) == 0 { - metricSetFields[common.DeDot(metricName)] = metricValue - return + m[common.DeDot(metricName)] = metricValue + return m } for _, mapping := range mappings { // The metricname match the one with no labels in mappings - // Let's insert it dedotted and continue if metricName == mapping.Metric { - metricSetFields[mapping.Value.Field] = metricValue - return + m[mapping.Value.Field] = metricValue + return m } res := mapping.regex.FindStringSubmatch(metricName) @@ -121,7 +121,7 @@ func eventMapping(metricName string, metricValue interface{}, metricSetFields ma // Not all labels match // Skip and continue to next mapping if len(res) != (len(mapping.Labels) + 1) { - logger.Debug("not all labels match in statsd.mapping, skipped") + logger.Debug("not all labels match in statsd.mappings, skipped") continue } @@ -133,13 +133,15 @@ func eventMapping(metricName string, metricValue interface{}, metricSetFields ma continue } - metricSetFields[label.Field] = res[i] + m[label.Field] = res[i] } } // Let's add the metric with the value field - metricSetFields[mapping.Value.Field] = metricValue + m[mapping.Value.Field] = metricValue + break } + return m } func newMetricProcessor(ttl time.Duration) *metricProcessor { diff --git a/x-pack/metricbeat/module/statsd/server/data_test.go b/x-pack/metricbeat/module/statsd/server/data_test.go index 2377a2fae5b..2bdc97ab5c2 100644 --- a/x-pack/metricbeat/module/statsd/server/data_test.go +++ b/x-pack/metricbeat/module/statsd/server/data_test.go @@ -737,11 +737,9 @@ func TestEventMapping(t *testing.T) { }, } { t.Run(test.metricName, func(t *testing.T) { - metricSetFields := mapstr.M{} builtMappings, _ := buildMappings(mappings) - eventMapping(test.metricName, test.metricValue, metricSetFields, builtMappings) - - assert.Equal(t, test.expected, metricSetFields) + ms := eventMapping(test.metricName, test.metricValue, builtMappings) + assert.Equal(t, test.expected, ms) }) } } @@ -1132,7 +1130,7 @@ func TestTagsGrouping(t *testing.T) { require.NoError(t, err) events := ms.getEvents() - assert.Len(t, events, 2) + assert.Len(t, events, 4) actualTags := []mapstr.M{} for _, e := range events { @@ -1146,6 +1144,18 @@ func TestTagsGrouping(t *testing.T) { "k2": "v2", }, }, + { + "labels": mapstr.M{ + "k1": "v1", + "k2": "v2", + }, + }, + { + "labels": mapstr.M{ + "k1": "v2", + "k2": "v3", + }, + }, { "labels": mapstr.M{ "k1": "v2", @@ -1224,7 +1234,7 @@ func TestData(t *testing.T) { require.NoError(t, err) events := ms.getEvents() - assert.Len(t, events, 1) + assert.Len(t, events, 10) mbevent := mbtest.StandardizeEvent(ms, *events[0]) mbtest.WriteEventToDataJSON(t, mbevent, "") diff --git a/x-pack/metricbeat/module/statsd/server/server.go b/x-pack/metricbeat/module/statsd/server/server.go index 48aee89e460..c2366a71b6b 100644 --- a/x-pack/metricbeat/module/statsd/server/server.go +++ b/x-pack/metricbeat/module/statsd/server/server.go @@ -95,7 +95,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { mappings, err := buildMappings(config.Mappings) if err != nil { - return nil, fmt.Errorf("invalid mapping configuration for `statsd.mapping`: %w", err) + return nil, fmt.Errorf("invalid mapping configuration for `statsd.mappings`: %w", err) } return &MetricSet{ BaseMetricSet: base, @@ -107,8 +107,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Host returns the hostname or other module specific value that identifies a // specific host or service instance from which to collect metrics. -func (b *MetricSet) Host() string { - return b.server.(*udp.UdpServer).GetHost() +func (m *MetricSet) Host() string { + return m.server.(*udp.UdpServer).GetHost() } func buildMappings(config []StatsdMapping) (map[string]StatsdMapping, error) { @@ -163,30 +163,36 @@ func buildMappings(config []StatsdMapping) (map[string]StatsdMapping, error) { return mappings, nil } +// It processes metric groups, applies event mappings, and creates Metricbeat events. +// The generated events include metric fields, labels, and the namespace associated with the MetricSet. +// Returns a slice of Metricbeat events. func (m *MetricSet) getEvents() []*mb.Event { groups := m.processor.GetAll() - events := make([]*mb.Event, len(groups)) - for idx, tagGroup := range groups { - - mapstrTags := mapstr.M{} + // If there are no metric groups, return nil to indicate no events. + if len(groups) == 0 { + return nil + } + events := make([]*mb.Event, 0, len(groups)) + for _, tagGroup := range groups { + mapstrTags := make(mapstr.M, len(tagGroup.tags)) for k, v := range tagGroup.tags { mapstrTags[k] = v } - sanitizedMetrics := mapstr.M{} for k, v := range tagGroup.metrics { - eventMapping(k, v, sanitizedMetrics, m.mappings) - } + // Apply event mapping to the metric and get MetricSetFields. + ms := eventMapping(k, v, m.mappings) - if len(sanitizedMetrics) == 0 { - continue - } - - events[idx] = &mb.Event{ - MetricSetFields: sanitizedMetrics, - RootFields: mapstr.M{"labels": mapstrTags}, - Namespace: m.Module().Name(), + // If no MetricSetFields were generated, continue to the next metric. + if len(ms) == 0 { + continue + } + events = append(events, &mb.Event{ + MetricSetFields: ms, + RootFields: mapstr.M{"labels": mapstrTags}, + Namespace: m.Module().Name(), + }) } } return events From e1aa97097a6e0acfbff77650859b7923db87b700 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 30 Oct 2023 12:32:23 -0400 Subject: [PATCH 17/22] chore: Update snapshot.yml (#36988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 28bb4bb0c7f..7e210e41426 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-fa958a90-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-bb5042c3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-fa958a90-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-bb5042c3-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-fa958a90-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-bb5042c3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 069bf040d9689b77539b721dd8b1599b41cefd7c Mon Sep 17 00:00:00 2001 From: Christiano Haesbaert Date: Tue, 31 Oct 2023 08:52:42 +0100 Subject: [PATCH 18/22] Fix copy arguments for strict aligned architectures in ktrace decoding (#36976) Small typo. In practice only affects arm32 and arm64 as I doubt there are users of other architectures around. --- CHANGELOG-developer.next.asciidoc | 3 ++- CHANGELOG.next.asciidoc | 1 + x-pack/auditbeat/tracing/int_aligned.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 7fefb8d14c2..6a900dae8cf 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -89,7 +89,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fixed some race conditions in tests {pull}36185[36185] - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525] - Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650] -- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714] +- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714] +- Fix copy arguments for strict aligned architectures. {pull}36976[36976] ==== Added diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eef14a45f56..fd5543cb927 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -180,6 +180,7 @@ is collected by it. *Auditbeat* - Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851] +- Fix copy arguments for strict aligned architectures. {pull}36976[36976] *Filebeat* diff --git a/x-pack/auditbeat/tracing/int_aligned.go b/x-pack/auditbeat/tracing/int_aligned.go index 25c6094ef9a..6c8c4c53972 100644 --- a/x-pack/auditbeat/tracing/int_aligned.go +++ b/x-pack/auditbeat/tracing/int_aligned.go @@ -16,7 +16,7 @@ import ( var errBadSize = errors.New("bad size for integer") func copyInt(dst unsafe.Pointer, src unsafe.Pointer, len uint8) error { - copy((*(*[maxIntSizeBytes]byte)(src))[:len], (*(*[maxIntSizeBytes]byte)(src))[:len]) + copy((*(*[maxIntSizeBytes]byte)(dst))[:len], (*(*[maxIntSizeBytes]byte)(src))[:len]) return nil } From a3ad99606854d4d31590fc49a77f02d21eedb09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Tue, 31 Oct 2023 08:58:24 +0100 Subject: [PATCH 19/22] Add state namespace metricset. (#36978) --- deploy/kubernetes/metricbeat-kubernetes.yaml | 1 + deploy/kubernetes/metricbeat/metricbeat-configmap.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index acece92b8d6..fa19c02043b 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -161,6 +161,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset diff --git a/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml index 388d4d7b550..0cee9833798 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml @@ -29,6 +29,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset From 009b3f0262d2ace24df0faf2ba1ae429708c107c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 31 Oct 2023 11:52:07 -0400 Subject: [PATCH 20/22] docs - Clarify how to use 'certificate' for servers (#36991) The docs didn't show where to put issuer certificate in the context of TLS servers. This explains how to build up a PEM certificate bundle --- libbeat/docs/shared-ssl-config.asciidoc | 98 +++++++++++++++++++------ 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/libbeat/docs/shared-ssl-config.asciidoc b/libbeat/docs/shared-ssl-config.asciidoc index 1b27c3b217f..3c456de4206 100644 --- a/libbeat/docs/shared-ssl-config.asciidoc +++ b/libbeat/docs/shared-ssl-config.asciidoc @@ -443,34 +443,84 @@ certificate_authorities: [[server-certificate]] ==== `certificate: "/etc/server/cert.pem"` -For server authentication, the path to the SSL authentication certificate must -be specified for TLS. If the certificate is not specified, startup will fail. +The end-entity (leaf) certificate that the server uses to identify itself. +If the certificate is signed by a certificate authority (CA), then it should +include intermediate CA certificates, sorted from leaf to root. +For servers, a `certificate` and <> must be specified. -When this option is configured, the <> option is also required. -The certificate option support embedding of the certificate: +The certificate option supports embedding of the PEM certificate content. This +example contains the leaf certificate followed by issuer's certificate. [source,yaml] ---- certificate: | - -----BEGIN CERTIFICATE----- - MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF - ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 - MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB - BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n - fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl - 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t - /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP - PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 - CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O - BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux - 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D - 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw - 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA - H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu - 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 - yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk - sxSmbIUfc2SGJGCJD4I= - -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIF2jCCA8KgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJVUzEW + MBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8g + UmVhbDEOMAwGA1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMwHhcNMjMxMDMw + MTkyMzU4WhcNMjMxMDMxMTkyMzU4WjB2MQswCQYDVQQGEwJVUzEWMBQGA1UEBxMN + U2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8gUmVhbDEOMAwG + A1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMxDzANBgNVBAMTBnNlcnZlcjCC + AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALW37cart7l0KE3LCStFbiGm + Rr/QSkuPv+Y+SXFT4zXrMFP3mOfUCVsR4lugv+jmql9qjbwR9jKsgKXA1kSvNXSZ + lLYWRcNnQ+QzwKxJf/jy246nSfqb2FKvVMs580lDwKHHxn/FSpHV93O4Goy5cLfF + ACE7BSdJdxl5DVAMmmkzd6gBGgN8dQIbcyJYuIZYQt44PqSYh/BomTyOXKrmvX4y + t7/pF+ldJjWZq/6SfCq6WE0jSrpI1P/42Qd9h5Tsnl6qsUGA2Tz5ZqKz2cyxaIlK + wL9tYDionfFIl+jZcxkGPF2a14O1TycCI0B/z+0VL+HR/8fKAB0NdP+QRLaPWOrn + DvraAO+bVKC6VrQyUYNUOwtd2gMUqm6Hzrf4s3wjP754eSJkvnSoSAB6l7ZmJKe5 + Pz5oDDOVPwKHv/MrhsCSMNFeXSEO+rq9TtYEAFQI5rFGHlURga8kA1T1pirHyEtS + 2o8GUSPSHVulaPdFnHg4xfTexfRYLCqya75ISJuY2/+2GblCie/re1GFitZCZ46/ + xiQQDOjgL96soDVZ+cTtMpXanslgDapTts9LPIJTd9FUJCY1omISGiSjABRuTlCV + 8054ja4BKVahSd5BqqtVkWyV64SCut6kce2ndwBkyFvlZ6cteLCW7KtzYvba4XBb + YIAs+H+9e/bZUVhws5mFAgMBAAGjgYMwgYAwDgYDVR0PAQH/BAQDAgeAMB0GA1Ud + JQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAOBgNVHQ4EBwQFAQIDBAUwPwYDVR0R + BDgwNoIJbG9jYWxob3N0ghFiZWF0cy5leGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAA + AAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAgEAldSZOUi+OUR46ERQuINl1oED + mjNsQ9FNP/RDu8mPJaNb5v2sAbcpuZb9YdnScT+d+n0+LMd5uz2g67Qr73QCpXwL + 9YJIs56i7qMTKXlVvRQrvF9P/zP3sm5Zfd2I/x+8oXgEeYsxAWipJ8RsbnN1dtu8 + C4l+P0E58jjrjom11W90RiHYaT0SI2PPBTTRhYLz0HayThPZDMdFnIQqVxUYbQD5 + ybWu77hnsvC/g2C8/N2LAdQGJJ67owMa5T3YRneiaSvvOf3I45oeLE+olGAPdrSq + 5Sp0G7fcAKMRPxcwYeD7V5lfYMtb+RzECpYAHT8zHKLZl6/34q2k8P8EWEpAsD80 + +zSbCkdvNiU5lU90rV8E2baTKCg871k4O8sT48eUyDps6ZUCfT1dgefXeyOTV5bY + 864Zo6bWJhAJ7Qa2d4HJkqPzSbqsosHVobojgkOcMqkStLHd8sgtCoFmJMflbp7E + ghawl/RVFEkL9+TWy9fR8sJWRx13P8CUP6AL9kVmcU2c3gMNpvQfIii9QOnQrRsi + yZj9FKl+ZM49I6RQ6dY5JVgWtpVm/+GBVuy1Aj91JEjw7r1jAeir5K9LAXG8kEN9 + irndx1SK2MMTY79lGHFGQRv3vnQGI0Wzjtn31YJ7qIFNJ1WWbAZLR9FBtzmMeXM6 + puoJ9UYvfIcHUGPdZGU= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIFpjCCA46gAwIBAgIBATANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJVUzEW + MBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8g + UmVhbDEOMAwGA1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMwHhcNMjMxMDMw + MTkyMzU2WhcNMjMxMDMxMTkyMzU2WjBlMQswCQYDVQQGEwJVUzEWMBQGA1UEBxMN + U2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8gUmVhbDEOMAwG + A1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMwggIiMA0GCSqGSIb3DQEBAQUA + A4ICDwAwggIKAoICAQDQP3hJt4jTIo+tBXB/R4RuBTvv6OOago9joxlNDm0abseJ + ehE0V8FDi0SSpa7ZiqwCGq/deu5OIWVNpFCLHeH5YBriNmB7oPkNRCleu50JsUrG + RjSTtBIJcu/CVpD7Q5XMbhbhYcPArrxrSreo3ox8a+2X7b8nA1xPgIcWqSCgs9iV + lwKHaQWNTUXYwwZG7b9WG4EJaki6t1+1QbDDJU0oWrZNg23wQEBvEVRDQs7kadvm + 9YtZLPULlSyV4Rk3yNW8dPXHjcz2wp3PBPIWIQe9mzYU608307TkUMVN2EEOImxl + Wm1RtXYvvVb1LiY0C2lYbN3jLZQzffK5RsS87ocqTQM+HvDBv/PupHDvW08wietu + RtRbdx/2cN0GLmOHnkWKx+GlYDZfAtIj958fTKl2hHyNqJ1pE7vksSYBwBxMFQem + eSGzw5pO53kmPcZO203YQ2qoJd7z1aLf7eAOqDn5zwlYNc00bZ6DwTZsyptGv9sZ + zcZuovppPgCN4f1I9ja/NPKep+sVKfQqR5HuOFOPFcr6oOioESJSgIvXXF9RhCVh + UMeZKWWSCNm1ea4h6q8OJdQfM7XXkXm+dEyF0TogC00CidZWuYMZcgXND5p/1Di5 + PkCKPUMllCoK0oaTfFioNW7qtNbDGQrW+spwDa4kjJNKYtDD0jjPgFMgSzQ2MwID + AQABo2EwXzAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsG + AQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFImOXc9Tv+mgn9jOsPig + 9vlAUTa+MA0GCSqGSIb3DQEBCwUAA4ICAQBZ9tqU88Nmgf+vDgkKMKLmLMaRCRlV + HcYrm7WoWLX+q6VSbmvf5eD5OrzzbAnnp16iXap8ivsAEFTo8XWh/bjl7G/2jetR + xZD2WHtzmAg3s4SVsEHIyFUF1ERwnjO2ndHjoIsx8ktUk1aNrmgPI6s07fkULDm+ + 2aXyBSZ9/oimZM/s3IqYJecxwE+yyS+FiS6mSDCCVIyQXdtVAbFHegyiBYv8EbwF + Xz70QiqQtxotGlfts/3uN1s+xnEoWz5E6S5DQn4xQh0xiKSXPizMXou9xKzypeSW + qtNdwtg62jKWDaVriBfrvoCnyjjCIjmcTcvA2VLmeZShyTuIucd0lkg2NKIGeM7I + o33hmdiKaop1fVtj8zqXvCRa3ecmlvcxPKX0otVFORFNOfaPjH/CjW0CnP0LByGK + YW19w0ncJZa9cc1SlNL28lnBhW+i1+ViR02wtjabH9XO+mtxuaEPDZ1hLhhjktqI + Y2oFUso4C5xiTU/hrH8+cFv0dn/+zyQoLfJEQbUX9biFeytt7T4Yynwhdy7jryqH + fdy/QM26YnsE8D7l4mv99z+zII0IRGnQOuLTuNAIyGJUf69hCDubZFDeHV/IB9hU + 6GA6lBpsJlTDgfJLbtKuAHxdn1DO+uGg0GxgwggH6Vh9x9yQK2E6BaepJisL/zNB + RQQmEyTn1hn/eA== + -----END CERTIFICATE----- ---- [float] @@ -478,7 +528,7 @@ certificate: | ==== `key: "/etc/server/cert.key"` The server certificate key used for authentication is required. -The key option support embedding of the private key: +The key option supports embedding of the private key: [source,yaml] ---- From 2539cdc2577b7aad659f78ea670308a7d38dfe13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Tue, 31 Oct 2023 19:47:16 +0100 Subject: [PATCH 21/22] [Heartbeat] Bump nodejs minor version (#36961) Bump NodeJS version bundled with Heartbeat to 18.18.2. Co-authored-by: Tiago Queiroz --------- Co-authored-by: Tiago Queiroz --- CHANGELOG.next.asciidoc | 1 + dev-tools/packaging/templates/docker/Dockerfile.tmpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index fd5543cb927..2d2364180d0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -119,6 +119,7 @@ is collected by it. - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Fix retries to trigger on a down monitor with no previous state. {pull}36842[36842] +- Bump NodeJS minor version to 18.18.2. {pull}36961[36961] - Fix monitor duration calculation with retries. {pull}36900[36900] *Metricbeat* diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 305687d33e0..9a792ecc1e0 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -123,7 +123,7 @@ RUN echo \ ENV ELASTIC_SYNTHETICS_CAPABLE=true ENV TZ=UTC ENV SUITES_DIR={{ $beatHome }}/suites -ENV NODE_VERSION=18.16.0 +ENV NODE_VERSION=18.18.2 ENV PATH="$NODE_PATH/node/bin:$PATH" # Install the latest version of @elastic/synthetics forcefully ignoring the previously # cached node_modules, heartbeat then calls the global executable to run test suites From 32cdaf67df53913568afc290c922c83e3154c960 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:11:32 -0400 Subject: [PATCH 22/22] chore: Update snapshot.yml (#37003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 7e210e41426..cb047a327df 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-bb5042c3-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-0fcb9daf-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-bb5042c3-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-0fcb9daf-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-bb5042c3-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-0fcb9daf-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing"