Merge remote-tracking branch 'upstream/main' into feat/macos-ul

CHANGELOG.asciidoc

@@ -441,6 +441,11 @@ filebeat.inputs:
 === Beats version 8.14.3
 https://github.com/elastic/beats/compare/v8.14.2\...v8.14.3[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Bugfixes
 
 *Filebeat*
@@ -471,6 +476,11 @@ https://github.com/elastic/beats/compare/v8.14.2\...v8.14.3[View commits]
 === Beats version 8.14.2
 https://github.com/elastic/beats/compare/v8.14.1\...v8.14.2[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Breaking changes
 
 *Filebeat*
@@ -507,6 +517,11 @@ https://github.com/elastic/beats/compare/v8.14.1\...v8.14.2[View commits]
 === Beats version 8.14.1
 https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Bugfixes
 
 *Heartbeat*
@@ -518,6 +533,11 @@ https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits]
 === Beats version 8.14.0
 https://github.com/elastic/beats/compare/v8.13.4\...v8.14.0[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Breaking changes
 
 *Filebeat*

CHANGELOG.next.asciidoc

@@ -50,8 +50,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
 - The performance of ingesting SQS data with the S3 input has improved by up to 60x for queues with many small events. `max_number_of_messages` config for SQS mode is now ignored, as the new design no longer needs a manual cap on messages. Instead, use `number_of_workers` to scale ingestion rate in both S3 and SQS modes. The increased efficiency may increase network bandwidth consumption, which can be throttled by lowering `number_of_workers`. It may also increase number of events stored in memory, which can be throttled by lowering the configured size of the internal queue. {pull}40699[40699]
 - Fixes filestream logging the error "filestream input with ID 'ID' already exists, this will lead to data duplication[...]" on Kubernetes when using autodiscover. {pull}41585[41585]
-
 - Add kafka compression support for ZSTD.
+- Filebeat fails to start if there is any input with a duplicated ID. It logs the duplicated IDs and the offending inputs configurations. {pull}41731[41731]
 
 *Heartbeat*
 
@@ -116,12 +116,14 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add `translate_ldap_attribute` processor. {pull}41472[41472]
 - Remove unnecessary debug logs during idle connection teardown {issue}40824[40824]
 - Fix incorrect cloud provider identification in add_cloud_metadata processor using provider priority mechanism {pull}41636[41636]
+- Prevent panic if libbeat processors are loaded more than once. {issue}41475[41475] {pull}41857[51857]
 
 *Auditbeat*
 
 - auditd: Request status from a separate socket to avoid data congestion {pull}41207[41207]
 - auditd: Use ECS `event.type: end` instead of `stop` for SERVICE_STOP, DAEMON_ABORT, and DAEMON_END messages. {pull}41558[41558]
 - auditd: Update syscall names for Linux 6.11. {pull}41558[41558]
+- hasher: Geneneral improvements and fixes. {pull}41863[41863]
 
 *Filebeat*
 
@@ -185,6 +187,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix missing key in streaming input logging. {pull}41600[41600]
 - Improve S3 object size metric calculation to support situations where Content-Length is not available. {pull}41755[41755]
 - Fix handling of http_endpoint request exceeding memory limits. {issue}41764[41764] {pull}41765[41765]
+- Rate limiting fixes in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41583[41583]
 
 *Heartbeat*
 
@@ -217,6 +220,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Kubernetes metadata sometimes not being present after startup {pull}41216[41216]
 - Do not report non-existant 0 values for RSS metrics in docker/memory {pull}41449[41449]
 - Log Cisco Meraki `getDevicePerformanceScores` errors without stopping metrics collection. {pull}41622[41622]
+- Don't skip first bucket value in GCP metrics metricset for distribution type metrics {pull}41822[41822]
 
 
 *Osquerybeat*
@@ -261,6 +265,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add linux capabilities to processes in the system/process. {pull}37453[37453]
 - Add linux capabilities to processes in the system/process. {pull}37453[37453]
 - Add process.entity_id, process.group.name and process.group.id in add_process_metadata processor. Make fim module with kprobes backend to always add an appropriately configured add_process_metadata processor to enrich file events {pull}38776[38776]
+- Split module/system/process into common and provider bits. {pull}41868[41868]
 
 *Auditbeat*
 
@@ -346,8 +351,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add support for Journald in the System module. {pull}41555[41555]
 - Add ability to remove request trace logs from http_endpoint input. {pull}40005[40005]
 - Add ability to remove request trace logs from entityanalytics input. {pull}40004[40004]
+- Refactor & cleanup with updates to default values and documentation. {pull}41834[41834]
 - Update CEL mito extensions to v1.16.0. {pull}41727[41727]
 - Add `unifiedlogs` input for MacOS. {pull}41791[41791]
+- Add evaluation state dump debugging option to CEL input. {pull}41335[41335]
+- Added support for retry configuration in GCS input. {issue}11580[11580] {pull}41862[41862]
+- Improve S3 polling mode states registry when using list prefix option. {pull}41869[41869]
 
 *Auditbeat*
 
@@ -401,6 +410,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add support for location label as an optional configuration parameter in GCP metrics metricset. {issue}41550[41550] {pull}41626[41626]
 
 *Metricbeat*
+- Add benchmark module {pull}41801[41801]
 
 
 *Osquerybeat*
@@ -419,6 +429,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Language setting also added to decode xml wineventlog processor {pull}41525[41525]
 - Format embedded messages in the experimental api {pull}41525[41525]
 - Implement exclusion range support for event_id. {issue}38623[38623] {pull}41639[41639]
+- Make the experimental API GA and rename it to winlogbeat-raw {issue}39580[39580] {pull}41770[41770]
 
 
 *Functionbeat*

auditbeat/helper/hasher/hasher.go

@@ -26,7 +26,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"os"
 	"strings"
 	"time"
 
@@ -124,7 +123,7 @@ type FileTooLargeError struct {
 
 // Error returns the error message for FileTooLargeError.
 func (e FileTooLargeError) Error() string {
-	return fmt.Sprintf("hasher: file size %d exceeds max file size", e.fileSize)
+	return fmt.Sprintf("size %d exceeds max file size", e.fileSize)
 }
 
 // Config contains the configuration of a FileHasher.
@@ -174,28 +173,46 @@ type FileHasher struct {
 
 // NewFileHasher creates a new FileHasher.
 func NewFileHasher(c Config, done <-chan struct{}) (*FileHasher, error) {
+	var limit rate.Limit
+
+	if c.ScanRateBytesPerSec == 0 {
+		limit = rate.Inf
+	} else {
+		limit = rate.Limit(c.ScanRateBytesPerSec)
+	}
+
 	return &FileHasher{
 		config: c,
 		limiter: rate.NewLimiter(
-			rate.Limit(c.ScanRateBytesPerSec), // Rate
-			int(c.MaxFileSizeBytes),           // Burst
+			limit,                   // Rate
+			int(c.MaxFileSizeBytes), // Burst
 		),
 		done: done,
 	}, nil
 }
 
 // HashFile hashes the contents of a file.
 func (hasher *FileHasher) HashFile(path string) (map[HashType]Digest, error) {
-	info, err := os.Stat(path)
+	f, err := file.ReadOpen(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to stat file %v: %w", path, err)
+		return nil, fmt.Errorf("open: %w", err)
+	}
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("stat: %w", err)
+	}
+	if !info.Mode().IsRegular() {
+		return nil, fmt.Errorf("not a regular file")
+
 	}
 
 	// Throttle reading and hashing rate.
 	if len(hasher.config.HashTypes) > 0 {
 		err = hasher.throttle(info.Size())
 		if err != nil {
-			return nil, fmt.Errorf("failed to hash file %v: %w", path, err)
+			return nil, err
 		}
 	}
 
@@ -210,15 +227,9 @@ func (hasher *FileHasher) HashFile(path string) (map[HashType]Digest, error) {
 	}
 
 	if len(hashes) > 0 {
-		f, err := file.ReadOpen(path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to open file for hashing: %w", err)
-		}
-		defer f.Close()
-
 		hashWriter := multiWriter(hashes)
 		if _, err := io.Copy(hashWriter, f); err != nil {
-			return nil, fmt.Errorf("failed to calculate file hashes: %w", err)
+			return nil, err
 		}
 
 		nameToHash := make(map[HashType]Digest, len(hashes))
@@ -233,6 +244,10 @@ func (hasher *FileHasher) HashFile(path string) (map[HashType]Digest, error) {
 }
 
 func (hasher *FileHasher) throttle(fileSize int64) error {
+	// Burst is ignored if limit is infinite, so check it manually
+	if hasher.limiter.Limit() == rate.Inf && int(fileSize) > hasher.limiter.Burst() {
+		return FileTooLargeError{fileSize}
+	}
 	reservation := hasher.limiter.ReserveN(time.Now(), int(fileSize))
 	if !reservation.OK() {
 		// File is bigger than the max file size

dev-tools/mage/integtest_docker.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"go/build"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -99,11 +98,9 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 	if err != nil {
 		return err
 	}
-	dockerRepoRoot := filepath.Join("/go/src", repo.CanonicalRootImportPath)
-	dockerGoCache := filepath.Join(dockerRepoRoot, "build/docker-gocache")
 	magePath := filepath.Join("/go/src", repo.CanonicalRootImportPath, repo.SubDir, "build/mage-linux-"+GOARCH)
-	goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod/cache/download")
-	dockerGoPkgCache := "/gocache"
+	goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod")
+	dockerGoPkgCache := "/go/pkg/mod"
 
 	// Execute the inside of docker-compose.
 	args := []string{"-p", DockerComposeProjectName(), "run",
@@ -114,10 +111,8 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 		// compose.EnsureUp needs to know the environment type.
 		"-e", "STACK_ENVIRONMENT=" + StackEnvironment,
 		"-e", "TESTING_ENVIRONMENT=" + StackEnvironment,
-		"-e", "GOCACHE=" + dockerGoCache,
 		// Use the host machine's pkg cache to minimize external downloads.
-		"-v", goPkgCache + ":" + dockerGoPkgCache + ":ro",
-		"-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct",
+		"-v", goPkgCache + ":" + dockerGoPkgCache,
 	}
 	args, err = addUidGidEnvArgs(args)
 	if err != nil {
@@ -356,7 +351,7 @@ func StartIntegTestContainers() error {
 
 func StopIntegTestContainers() error {
 	// Docker-compose rm is noisy. So only pass through stderr when in verbose.
-	out := ioutil.Discard
+	out := io.Discard
 	if mg.Verbose() {
 		out = os.Stderr
 	}
@@ -368,7 +363,7 @@ func StopIntegTestContainers() error {
 
 	_, err = sh.Exec(
 		composeEnv,
-		ioutil.Discard,
+		io.Discard,
 		out,
 		"docker-compose",
 		"-p", DockerComposeProjectName(),

dev-tools/mage/kubernetes/kind.go

@@ -19,7 +19,8 @@ package kubernetes
 
 import (
 	"fmt"
-	"io/ioutil"
+	"go/build"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -66,8 +67,8 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 	}
 
 	clusterName := kubernetesClusterName()
-	stdOut := ioutil.Discard
-	stdErr := ioutil.Discard
+	stdOut := io.Discard
+	stdErr := io.Discard
 	if mg.Verbose() {
 		stdOut = os.Stdout
 		stdErr = os.Stderr
@@ -86,9 +87,29 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 		return err
 	}
 
+	cfg, err := os.CreateTemp("", "kind-")
+	if err != nil {
+		return err
+	}
+	if _, err := cfg.WriteString(fmt.Sprintf(`
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: %s
+        containerPath: /go/pkg/mod
+`, filepath.Join(build.Default.GOPATH, "pkg", "mod"))); err != nil {
+		return err
+	}
+	if err := cfg.Close(); err != nil {
+		return err
+	}
+
 	args := []string{
 		"create",
 		"cluster",
+		"--config", cfg.Name(),
 		"--name", clusterName,
 		"--kubeconfig", kubeConfig,
 		"--wait",
@@ -116,8 +137,8 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 
 // Teardown destroys the kubernetes cluster.
 func (m *KindIntegrationTestStep) Teardown(env map[string]string) error {
-	stdOut := ioutil.Discard
-	stdErr := ioutil.Discard
+	stdOut := io.Discard
+	stdErr := io.Discard
 	if mg.Verbose() {
 		stdOut = os.Stdout
 		stdErr = os.Stderr