From 9da2831fe3c67a66178eb6240187ed02315a1b85 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 23 Jul 2024 17:03:42 -0400 Subject: [PATCH 1/9] Update to Go 1.22.5 (#40082) * Update to go 1.22.4. * Link to PR. * Fix bad merge. * Fix some regex. * Fix config_test.go * Comment out ping. * Update to go 1.22.5. * Un-comment code. * Update go.mod. * Add encrypt=false to mssql py test. * Make CI happy * Fix NOTICE.txt. --------- Co-authored-by: subham sarkar Co-authored-by: Julien Lind (cherry picked from commit c0f8ae3b257b13cd80036c8da9e1d607444572bd) # Conflicts: # auditbeat/module/file_integrity/config_test.go # dev-tools/mage/crossbuild.go # go.mod # libbeat/docs/version.asciidoc # libbeat/processors/actions/drop_fields_test.go # metricbeat/Dockerfile --- .go-version | 2 +- .golangci.yml | 8 +- CHANGELOG.next.asciidoc | 3 + auditbeat/Dockerfile | 2 +- .../module/file_integrity/config_test.go | 9 +- .../kubernetes/filebeat/Dockerfile.debug | 2 +- .../kubernetes/heartbeat/Dockerfile.debug | 2 +- .../kubernetes/metricbeat/Dockerfile.debug | 2 +- dev-tools/mage/crossbuild.go | 16 +++ go.mod | 4 + heartbeat/Dockerfile | 2 +- libbeat/docs/version.asciidoc | 6 + .../processors/actions/drop_fields_test.go | 122 ++++++++++++++++++ metricbeat/Dockerfile | 5 + metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/nats/_meta/Dockerfile | 2 +- metricbeat/module/vsphere/_meta/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/functionbeat/Dockerfile | 2 +- .../module/mssql/_meta/supported-versions.yml | 2 +- .../module/mssql/docker-compose.yml | 4 +- .../metricbeat/module/stan/_meta/Dockerfile | 2 +- 22 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 libbeat/processors/actions/drop_fields_test.go diff --git a/.go-version b/.go-version index 26d7b6e756e0..da9594fd66f3 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.21.12 +1.22.5 diff --git a/.golangci.yml b/.golangci.yml index f00d1496fa09..78d266ea0df6 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -102,7 +102,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.21.12" + go: "1.22.5" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -120,19 +120,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.21.12" + go: "1.22.5" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.21.12" + go: "1.22.5" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.21.12" + go: "1.22.5" gosec: excludes: diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index be9b8ede3ccf..590b2df04cd6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -10,6 +10,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* +- Update Go version to 1.22.5. {pull}40082[40082] + +*Auditbeat* *Auditbeat* diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 910945115a38..f81aac4e5be5 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ diff --git a/auditbeat/module/file_integrity/config_test.go b/auditbeat/module/file_integrity/config_test.go index c0097fc086e1..0410488ae9b3 100644 --- a/auditbeat/module/file_integrity/config_test.go +++ b/auditbeat/module/file_integrity/config_test.go @@ -52,10 +52,15 @@ func TestConfig(t *testing.T) { assert.EqualValues(t, 1024*1024*1024, c.MaxFileSizeBytes) assert.EqualValues(t, 1024*1024*10, c.ScanRateBytesPerSec) assert.Len(t, c.ExcludeFiles, 2) - assert.EqualValues(t, `\.DS_Store(?-m:$)`, c.ExcludeFiles[0].String()) - assert.EqualValues(t, `\.swp(?-m:$)`, c.ExcludeFiles[1].String()) + assert.EqualValues(t, `(?-m:\.DS_Store$)`, c.ExcludeFiles[0].String()) + assert.EqualValues(t, `(?-m:\.swp$)`, c.ExcludeFiles[1].String()) assert.Len(t, c.IncludeFiles, 1) +<<<<<<< HEAD assert.EqualValues(t, `\.ssh/(?-m:$)`, c.IncludeFiles[0].String()) +======= + assert.EqualValues(t, `(?-m:\.ssh/$)`, c.IncludeFiles[0].String()) + assert.Len(t, c.FileParsers, 2) +>>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) } func TestConfigInvalid(t *testing.T) { diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index 6dd31c7b7b9c..9f7a6fe27e44 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.12 as builder +FROM golang:1.22.5 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index c843cd01e717..b05b4b46b350 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.12 as builder +FROM golang:1.22.5 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index a2a867e79714..5725175a53ab 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.12 as builder +FROM golang:1.22.5 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 7faa6a9dfe3b..8c7eec070ca8 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -309,6 +309,22 @@ func (b GolangCrossBuilder) Build() error { "--env", fmt.Sprintf("DEV=%v", DevBuild), "-v", repoInfo.RootDir+":"+mountPoint, "-w", workDir, +<<<<<<< HEAD +======= + ) + + // Ensure the proper platform is passed + // This fixes an issue where during arm64 linux build for the currently used docker image + // docker.elastic.co/beats-dev/golang-crossbuild:1.22.5-arm the image for amd64 arch is pulled + // and causes problems when using native arch tools on the binaries that are built for arm64 arch. + if strings.HasPrefix(b.Platform, "linux/") { + args = append(args, + "--platform", b.Platform, + ) + } + + args = append(args, +>>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) image, "--build-cmd", buildCmd+" "+b.Target, "-p", b.Platform, diff --git a/go.mod b/go.mod index fb7faa80ef6a..fd9dcdf3c42a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,10 @@ module github.com/elastic/beats/v7 +<<<<<<< HEAD go 1.21 +======= +go 1.22.5 +>>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) require ( cloud.google.com/go/bigquery v1.52.0 diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index ca0116f6e9dd..753825ecc115 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index da61339498be..3c4c3613b275 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,12 @@ +<<<<<<< HEAD :stack-version: 7.17.22 :doc-branch: 7.17 :go-version: 1.21.12 +======= +:stack-version: 8.15.0 +:doc-branch: main +:go-version: 1.22.5 +>>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/processors/actions/drop_fields_test.go b/libbeat/processors/actions/drop_fields_test.go new file mode 100644 index 000000000000..d49e4561fdd9 --- /dev/null +++ b/libbeat/processors/actions/drop_fields_test.go @@ -0,0 +1,122 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/common/match" + config2 "github.com/elastic/elastic-agent-libs/config" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestDropFieldRun(t *testing.T) { + event := &beat.Event{ + Fields: mapstr.M{ + "field": "value", + }, + Meta: mapstr.M{ + "meta_field": "value", + }, + } + + t.Run("supports a normal field", func(t *testing.T) { + p := dropFields{ + Fields: []string{"field"}, + } + + newEvent, err := p.Run(event) + assert.NoError(t, err) + assert.Equal(t, mapstr.M{}, newEvent.Fields) + assert.Equal(t, event.Meta, newEvent.Meta) + }) + + t.Run("supports a metadata field", func(t *testing.T) { + p := dropFields{ + Fields: []string{"@metadata.meta_field"}, + } + + newEvent, err := p.Run(event) + assert.NoError(t, err) + assert.Equal(t, mapstr.M{}, newEvent.Meta) + assert.Equal(t, event.Fields, newEvent.Fields) + }) + + t.Run("supports a regexp field", func(t *testing.T) { + event = &beat.Event{ + Fields: mapstr.M{ + "field_1": mapstr.M{ + "subfield_1": "sf_1_value", + "subfield_2": mapstr.M{ + "subfield_2_1": "sf_2_1_value", + "subfield_2_2": "sf_2_2_value", + }, + "subfield_3": mapstr.M{ + "subfield_3_1": "sf_3_1_value", + "subfield_3_2": "sf_3_2_value", + }, + }, + "field_2": "f_2_value", + }, + } + + p := dropFields{ + RegexpFields: []match.Matcher{match.MustCompile("field_2$"), match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), match.MustCompile("field_1\\.subfield_3(.*)")}, + Fields: []string{}, + } + + newEvent, err := p.Run(event) + assert.NoError(t, err) + assert.Equal(t, mapstr.M{ + "field_1": mapstr.M{ + "subfield_1": "sf_1_value", + }, + }, newEvent.Fields) + }) +} + +func TestNewDropFields(t *testing.T) { + t.Run("detects regexp fields and assign to RegexpFields property", func(t *testing.T) { + c := config2.MustNewConfigFrom(map[string]interface{}{ + "fields": []string{"/field_.*1/", "/second/", "third"}, + }) + + procInt, err := newDropFields(c) + assert.NoError(t, err) + + processor, ok := procInt.(*dropFields) + assert.True(t, ok) + assert.Equal(t, []string{"third"}, processor.Fields) + assert.Equal(t, "", processor.RegexpFields[0].String()) + assert.Equal(t, "(?-s:field_.*1)", processor.RegexpFields[1].String()) + }) + + t.Run("returns error when regexp field is badly written", func(t *testing.T) { + c := config2.MustNewConfigFrom(map[string]interface{}{ + "fields": []string{"/[//"}, + }) + + _, err := newDropFields(c) + + assert.Equal(t, "wrong configuration in drop_fields[0]=/[//. error parsing regexp: missing closing ]: `[/`", err.Error()) + }) +} diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 5ca4be5b9369..0f4665b3676f 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,5 +1,10 @@ +<<<<<<< HEAD FROM golang:1.21.12 COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ +======= +FROM golang:1.22.5 +COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ +>>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) RUN \ apt update \ diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 1c5adb9cf5bc..caa7dd400849 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index ea9a8b9ca535..111e733538db 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.21.12 AS build-env +FROM golang:1.22.5 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index db1b32f5967f..8fcf83b2ded5 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.21.12 +FROM golang:1.22.5 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 1e9ff2a10e63..46e77bccef77 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index ab993ab63bca..57f39f1e3ea2 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ diff --git a/x-pack/metricbeat/module/mssql/_meta/supported-versions.yml b/x-pack/metricbeat/module/mssql/_meta/supported-versions.yml index e9f409a28750..6d68c61adeb5 100644 --- a/x-pack/metricbeat/module/mssql/_meta/supported-versions.yml +++ b/x-pack/metricbeat/module/mssql/_meta/supported-versions.yml @@ -1,2 +1,2 @@ variants: - - MSSQL_VERSION: 2017-GA-ubuntu + - MSSQL_VERSION: 2017-CU31-GDR2-ubuntu-18.04 diff --git a/x-pack/metricbeat/module/mssql/docker-compose.yml b/x-pack/metricbeat/module/mssql/docker-compose.yml index 9fce8257135e..eccea7bda053 100644 --- a/x-pack/metricbeat/module/mssql/docker-compose.yml +++ b/x-pack/metricbeat/module/mssql/docker-compose.yml @@ -2,10 +2,10 @@ version: '2.3' services: mssql: - image: docker.elastic.co/integrations-ci/beats-mssql:${MSSQL_VERSION:-2017-GA-ubuntu}-1 + image: docker.elastic.co/integrations-ci/beats-mssql:${MSSQL_VERSION:-2017-CU31-GDR2-ubuntu-18.04}-1 build: context: ./_meta args: - MSSQL_VERSION: ${MSSQL_VERSION:-2017-GA-ubuntu} + MSSQL_VERSION: ${MSSQL_VERSION:-2017-CU31-GDR2-ubuntu-18.04} ports: - 1433 diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index 583c2929ebcb..6192a19570ef 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.21.12 AS build-env +FROM golang:1.22.5 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build . From cb7c46f97657c7a77db0c75321a35ac6273dc006 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 24 Jul 2024 16:10:40 -0400 Subject: [PATCH 2/9] Fix conflicts. --- auditbeat/module/file_integrity/config_test.go | 5 ----- dev-tools/mage/crossbuild.go | 16 ---------------- go.mod | 4 ---- libbeat/docs/version.asciidoc | 6 ------ metricbeat/Dockerfile | 5 ----- 5 files changed, 36 deletions(-) diff --git a/auditbeat/module/file_integrity/config_test.go b/auditbeat/module/file_integrity/config_test.go index 0410488ae9b3..5533f8dc38b2 100644 --- a/auditbeat/module/file_integrity/config_test.go +++ b/auditbeat/module/file_integrity/config_test.go @@ -55,12 +55,7 @@ func TestConfig(t *testing.T) { assert.EqualValues(t, `(?-m:\.DS_Store$)`, c.ExcludeFiles[0].String()) assert.EqualValues(t, `(?-m:\.swp$)`, c.ExcludeFiles[1].String()) assert.Len(t, c.IncludeFiles, 1) -<<<<<<< HEAD - assert.EqualValues(t, `\.ssh/(?-m:$)`, c.IncludeFiles[0].String()) -======= assert.EqualValues(t, `(?-m:\.ssh/$)`, c.IncludeFiles[0].String()) - assert.Len(t, c.FileParsers, 2) ->>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) } func TestConfigInvalid(t *testing.T) { diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 8c7eec070ca8..7faa6a9dfe3b 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -309,22 +309,6 @@ func (b GolangCrossBuilder) Build() error { "--env", fmt.Sprintf("DEV=%v", DevBuild), "-v", repoInfo.RootDir+":"+mountPoint, "-w", workDir, -<<<<<<< HEAD -======= - ) - - // Ensure the proper platform is passed - // This fixes an issue where during arm64 linux build for the currently used docker image - // docker.elastic.co/beats-dev/golang-crossbuild:1.22.5-arm the image for amd64 arch is pulled - // and causes problems when using native arch tools on the binaries that are built for arm64 arch. - if strings.HasPrefix(b.Platform, "linux/") { - args = append(args, - "--platform", b.Platform, - ) - } - - args = append(args, ->>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) image, "--build-cmd", buildCmd+" "+b.Target, "-p", b.Platform, diff --git a/go.mod b/go.mod index fd9dcdf3c42a..297ac0885ebb 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,6 @@ module github.com/elastic/beats/v7 -<<<<<<< HEAD -go 1.21 -======= go 1.22.5 ->>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) require ( cloud.google.com/go/bigquery v1.52.0 diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 3c4c3613b275..2e133e564ca7 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,12 +1,6 @@ -<<<<<<< HEAD :stack-version: 7.17.22 :doc-branch: 7.17 -:go-version: 1.21.12 -======= -:stack-version: 8.15.0 -:doc-branch: main :go-version: 1.22.5 ->>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 0f4665b3676f..d4adc21397fc 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,10 +1,5 @@ -<<<<<<< HEAD -FROM golang:1.21.12 -COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ -======= FROM golang:1.22.5 COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ ->>>>>>> c0f8ae3b25 (Update to Go 1.22.5 (#40082)) RUN \ apt update \ From 5eaaf2c65d3802d01b23799258c00e8bd9147f78 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 24 Jul 2024 16:11:37 -0400 Subject: [PATCH 3/9] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 590b2df04cd6..4a720df32eac 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -14,8 +14,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Auditbeat* -*Auditbeat* - *Filebeat* - Fix high IO and handling of a corrupted registry log file. {pull}35893[35893] From 279b3373004cd404bb515ee974df19ca29dd02ca Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 24 Jul 2024 16:12:42 -0400 Subject: [PATCH 4/9] Remove libbeat/processors/actions/drop_fields_test.go. --- .../processors/actions/drop_fields_test.go | 122 ------------------ 1 file changed, 122 deletions(-) delete mode 100644 libbeat/processors/actions/drop_fields_test.go diff --git a/libbeat/processors/actions/drop_fields_test.go b/libbeat/processors/actions/drop_fields_test.go deleted file mode 100644 index d49e4561fdd9..000000000000 --- a/libbeat/processors/actions/drop_fields_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package actions - -import ( - "testing" - - "github.com/elastic/beats/v7/libbeat/common/match" - config2 "github.com/elastic/elastic-agent-libs/config" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func TestDropFieldRun(t *testing.T) { - event := &beat.Event{ - Fields: mapstr.M{ - "field": "value", - }, - Meta: mapstr.M{ - "meta_field": "value", - }, - } - - t.Run("supports a normal field", func(t *testing.T) { - p := dropFields{ - Fields: []string{"field"}, - } - - newEvent, err := p.Run(event) - assert.NoError(t, err) - assert.Equal(t, mapstr.M{}, newEvent.Fields) - assert.Equal(t, event.Meta, newEvent.Meta) - }) - - t.Run("supports a metadata field", func(t *testing.T) { - p := dropFields{ - Fields: []string{"@metadata.meta_field"}, - } - - newEvent, err := p.Run(event) - assert.NoError(t, err) - assert.Equal(t, mapstr.M{}, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) - }) - - t.Run("supports a regexp field", func(t *testing.T) { - event = &beat.Event{ - Fields: mapstr.M{ - "field_1": mapstr.M{ - "subfield_1": "sf_1_value", - "subfield_2": mapstr.M{ - "subfield_2_1": "sf_2_1_value", - "subfield_2_2": "sf_2_2_value", - }, - "subfield_3": mapstr.M{ - "subfield_3_1": "sf_3_1_value", - "subfield_3_2": "sf_3_2_value", - }, - }, - "field_2": "f_2_value", - }, - } - - p := dropFields{ - RegexpFields: []match.Matcher{match.MustCompile("field_2$"), match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), match.MustCompile("field_1\\.subfield_3(.*)")}, - Fields: []string{}, - } - - newEvent, err := p.Run(event) - assert.NoError(t, err) - assert.Equal(t, mapstr.M{ - "field_1": mapstr.M{ - "subfield_1": "sf_1_value", - }, - }, newEvent.Fields) - }) -} - -func TestNewDropFields(t *testing.T) { - t.Run("detects regexp fields and assign to RegexpFields property", func(t *testing.T) { - c := config2.MustNewConfigFrom(map[string]interface{}{ - "fields": []string{"/field_.*1/", "/second/", "third"}, - }) - - procInt, err := newDropFields(c) - assert.NoError(t, err) - - processor, ok := procInt.(*dropFields) - assert.True(t, ok) - assert.Equal(t, []string{"third"}, processor.Fields) - assert.Equal(t, "", processor.RegexpFields[0].String()) - assert.Equal(t, "(?-s:field_.*1)", processor.RegexpFields[1].String()) - }) - - t.Run("returns error when regexp field is badly written", func(t *testing.T) { - c := config2.MustNewConfigFrom(map[string]interface{}{ - "fields": []string{"/[//"}, - }) - - _, err := newDropFields(c) - - assert.Equal(t, "wrong configuration in drop_fields[0]=/[//. error parsing regexp: missing closing ]: `[/`", err.Error()) - }) -} From 09bc57d84fbcb12fa5a6d0a885c6bb15ddaf50c1 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 25 Jul 2024 10:12:59 -0400 Subject: [PATCH 5/9] Fix test_autodiscover. --- filebeat/tests/system/test_autodiscover.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 62dd79164372..d7a96ca86fa2 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -24,24 +24,8 @@ def test_docker(self): autodiscover={ 'docker': { 'cleanup_timeout': '0s', - 'templates': f''' - - condition: - equals.docker.container.name: {container.name} - config: - - type: log - paths: - - %s/${{data.docker.container.name}}.log - ''' % self.working_dir, - }, - }, - ) - - proc = self.start_beat() - self._test(container) - - self.wait_until(lambda: self.log_contains('Stopping runner: input')) - proc.check_kill_and_wait() - + 'templates': f''' - condition: + equals.docker.container.name: {container.name} config: - type: log paths: - %s/${{data.docker.container.name}}.log ''' % self.working_dir, }, }, ) proc = self.start_beat() self._test(container) self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") From 17f399cf27e0b4ad6eccb0a9f581833048ead1cc Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 26 Jul 2024 12:41:20 -0400 Subject: [PATCH 6/9] Fixes. --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 10 ++-------- x-pack/elastic-agent/magefile.go | 10 +++++----- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 10d99499e45a..a5e886529bb5 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -7229,11 +7229,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@ -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-licenser -Version: v0.4.0 +Version: v0.4.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.4.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.4.2/LICENSE: Apache License diff --git a/go.mod b/go.mod index 81a638169569..04df81f13be6 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/elastic/elastic-agent-client/v7 v7.8.1 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.3.1 - github.com/elastic/go-licenser v0.4.0 + github.com/elastic/go-licenser v0.4.2 github.com/elastic/go-lookslike v0.3.0 github.com/elastic/go-lumber v0.1.0 github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 diff --git a/go.sum b/go.sum index 9fa4625c0dd1..96413fc048b9 100644 --- a/go.sum +++ b/go.sum @@ -315,8 +315,8 @@ github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa2 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38= github.com/elastic/go-libaudit/v2 v2.3.1 h1:BCPhB3BZTl2YBPJy3XHIu4DFfbbbWEmLMLlK/aJ7U/0= github.com/elastic/go-libaudit/v2 v2.3.1/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo= -github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI= -github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A= +github.com/elastic/go-licenser v0.4.2/go.mod h1:W8eH6FaZDR8fQGm+7FnVa7MxI1b/6dAqxz+zPB8nm5c= github.com/elastic/go-lookslike v0.3.0 h1:HDI/DQ65V85ZqM7D/sbxcK2wFFnh3+7iFvBk2v2FTHs= github.com/elastic/go-lookslike v0.3.0/go.mod h1:AhH+rdJux5RlVjs+6ej4jkvYyoNRkj2crxmqeHlj3hA= github.com/elastic/go-lumber v0.1.0 h1:HUjpyg36v2HoKtXlEC53EJ3zDFiDRn65d7B8dBHNius= @@ -901,7 +901,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 h1:0gYLpmzecnaDCoeWxSfEJ7J1b6B/67+NV++4HKQXx+Y= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= @@ -1000,7 +999,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -1035,7 +1033,6 @@ golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= @@ -1114,9 +1111,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1188,7 +1183,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 0b028c0f582b..1cd82a8894ff 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -133,12 +133,12 @@ func (Dev) Package() { // InstallGoLicenser install go-licenser to check license of the files. func (Prepare) InstallGoLicenser() error { - return GoGet(goLicenserRepo) + return GoInstall(goLicenserRepo) } // InstallGoLint for the code. func (Prepare) InstallGoLint() error { - return GoGet(goLintRepo) + return GoInstall(goLintRepo) } // All build all the things for the current projects. @@ -424,9 +424,9 @@ func RunGo(args ...string) error { return sh.RunV(mg.GoCmd(), args...) } -// GoGet fetch a remote dependencies. -func GoGet(link string) error { - _, err := sh.Exec(map[string]string{"GO111MODULE": "off"}, os.Stdout, os.Stderr, "go", "get", link) +// GoInstall installs a tool by calling `go install +func GoInstall(link string) error { + _, err := sh.Exec(map[string]string{}, os.Stdout, os.Stderr, "go", "install", link) return err } From 28e0253ca997e42305eb7869d4ae23b705897289 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 29 Jul 2024 09:13:09 -0400 Subject: [PATCH 7/9] Fix weird local update of test_autodiscover.py. --- filebeat/tests/system/test_autodiscover.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index d7a96ca86fa2..62dd79164372 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -24,8 +24,24 @@ def test_docker(self): autodiscover={ 'docker': { 'cleanup_timeout': '0s', - 'templates': f''' - condition: - equals.docker.container.name: {container.name} config: - type: log paths: - %s/${{data.docker.container.name}}.log ''' % self.working_dir, }, }, ) proc = self.start_beat() self._test(container) self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() + 'templates': f''' + - condition: + equals.docker.container.name: {container.name} + config: + - type: log + paths: + - %s/${{data.docker.container.name}}.log + ''' % self.working_dir, + }, + }, + ) + + proc = self.start_beat() + self._test(container) + + self.wait_until(lambda: self.log_contains('Stopping runner: input')) + proc.check_kill_and_wait() + @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") From b7aa75d474f7b2aadc1a241925356ba553143b17 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 29 Jul 2024 10:01:10 -0400 Subject: [PATCH 8/9] Fix bad references to go 1.21.12. --- filebeat/Dockerfile | 2 +- libbeat/Dockerfile | 2 +- x-pack/libbeat/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index 6297f03d8e86..9c9be8507e6b 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index 4784669d2508..c006beccfb9f 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 9f8279eb12d7..b7ab1c150317 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.12 +FROM golang:1.22.5 RUN \ apt-get update \ From 571f22aa3338621f923954b567bd195f2be19956 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 29 Jul 2024 10:11:19 -0400 Subject: [PATCH 9/9] More... --- .golangci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 78d266ea0df6..0e717cb59b1f 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -41,9 +41,7 @@ linters: - unused # checks Go code for unused constants, variables, functions and types - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string - ineffassign # detects when assignments to existing variables are not used - - structcheck # finds unused struct fields - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code - - varcheck # Finds unused global variables and constants - asciicheck # simple linter to check that your code does not contain non-ASCII identifiers - bodyclose # checks whether HTTP response body is closed successfully - durationcheck # check for two durations multiplied together