Merge branch 'main' into document-unavailable-unknown-ignore-types

CHANGELOG-developer.next.asciidoc

@@ -89,7 +89,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Fixed some race conditions in tests {pull}36185[36185]
 - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525]
 - Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650]
-- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714] 
+- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714]
+- Fix copy arguments for strict aligned architectures. {pull}36976[36976]
 
 ==== Added
 

CHANGELOG.next.asciidoc

@@ -113,11 +113,14 @@ is collected by it.
 - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
 - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772]
 - Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857]
+- Fix handling of response errors in HTTPJSON and CEL request trace logging. {pull}36956[36956]
 
 *Heartbeat*
 
 - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702]
 - Fix retries to trigger on a down monitor with no previous state. {pull}36842[36842]
+- Bump NodeJS minor version to 18.18.2. {pull}36961[36961]
+- Fix monitor duration calculation with retries. {pull}36900[36900]
 
 *Metricbeat*
 
@@ -141,6 +144,7 @@ is collected by it.
 - Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413]
 - Add log error when statsd server fails to start {pull}36477[36477]
 - Fix CassandraConnectionClosures metric configuration {pull}34742[34742]
+- Fix event mapping implementation for statsd module {pull}36925[36925]
 
 *Osquerybeat*
 
@@ -171,9 +175,13 @@ is collected by it.
 - allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788]
 - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884]
 - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments
+- elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843]
+- Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969]
 
 *Auditbeat*
 
+- Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851]
+- Fix copy arguments for strict aligned architectures. {pull}36976[36976]
 
 *Filebeat*
 
@@ -248,6 +256,9 @@ is collected by it.
 
 *Auditbeat*
 
+- Upgrade go-libaudit to v2.4.0. {issue}36776[36776] {pull}36964[36964]
+- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each dataset instance. {pull}36971[36971]
+
 *Libbeat*
 
 *Heartbeat*
@@ -262,6 +273,7 @@ is collected by it.
 - Add GCP Carbon Footprint metricbeat data {pull}34820[34820]
 - Add event loop utilization metric to Kibana module {pull}35020[35020]
 - Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms {pull}36647[36647]
+- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971]
 
 
 *Osquerybeat*
@@ -297,6 +309,7 @@ is collected by it.
 
 *Filebeat*
 
+- Deprecate rsa2elk Filebeat modules. {issue}36125[36125] {pull}36887[36887]
 
 *Heartbeat*
 

NOTICE.txt

@@ -2970,11 +2970,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/apache/arrow/go/v12
-Version: v12.0.1-0.20230605094802-c153c6d36ccf
+Version: v12.0.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/apache/arrow/go/[email protected]-0.20230605094802-c153c6d36ccf/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/apache/arrow/go/[email protected]/LICENSE.txt:
 
 
                                  Apache License
@@ -12712,11 +12712,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.6.0
+Version: v0.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -13659,11 +13659,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-libaudit/v2
-Version: v2.3.3
+Version: v2.4.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.3.3/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.4.0/LICENSE.txt:
 
 
                                  Apache License
@@ -24738,11 +24738,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/crypto
-Version: v0.12.0
+Version: v0.14.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.12.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -24849,11 +24849,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
-Version: v0.12.0
+Version: v0.17.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.12.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.17.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -24960,11 +24960,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/sys
-Version: v0.12.0
+Version: v0.13.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.12.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.13.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -24997,11 +24997,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/text
-Version: v0.12.0
+Version: v0.13.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.12.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -51107,11 +51107,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
-Version: v0.11.0
+Version: v0.13.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.11.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 

auditbeat/auditbeat.reference.yml

@@ -522,6 +522,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means use the default of 60s. The
+  # format is a Go language duration (example 60s is 60 seconds).
+  # idle_connection_timeout: 60s
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

auditbeat/docs/modules/auditd.asciidoc

@@ -212,6 +212,9 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are
 supported and will expand in lexicographical order. The format is the same as
 that of the `audit_rules` field.
 
+*`ignore_errors`*:: This setting allows errors during rule loading and parsing
+to be ignored, but logged as warnings.
+
 *`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to
 prevent backpressure from propagating to the kernel and impacting audited
 processes.

auditbeat/docs/modules/file_integrity.asciidoc

@@ -121,7 +121,7 @@ units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`,
 *`max_file_size`*:: The maximum size of a file in bytes for which
 {beatname_uc} will compute hashes and run file parsers. Files larger than this
 size will not be hashed or analysed by configured file parsers. The default
-value is 100 MiB. For convenience units can be specified as a suffix to the
+value is 100 MiB. For convenience, units can be specified as a suffix to the
 value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`,
 `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`.
 

auditbeat/module/auditd/_meta/docs.asciidoc

@@ -205,6 +205,9 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are
 supported and will expand in lexicographical order. The format is the same as
 that of the `audit_rules` field.
 
+*`ignore_errors`*:: This setting allows errors during rule loading and parsing
+to be ignored, but logged as warnings.
+
 *`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to
 prevent backpressure from propagating to the kernel and impacting audited
 processes.

auditbeat/module/auditd/audit_linux.go

@@ -979,8 +979,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) {
 		if c.SocketType == "" {
 			return "", fmt.Errorf("failed to create audit client: %w", err)
 		}
-		// Ignore errors if a socket type has been specified. It will fail during
-		// further setup and its necessary for unit tests to pass
+		// Ignore errors if a socket type has been specified.
 		return c.SocketType, nil
 	}
 	defer client.Close()

auditbeat/module/auditd/config_linux.go → auditbeat/module/auditd/config.go

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build unix
+
 package auditd
 
 import (
@@ -30,6 +32,7 @@ import (
 
 	"github.com/joeshaw/multierror"
 
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/go-libaudit/v2/rule"
 	"github.com/elastic/go-libaudit/v2/rule/flags"
 )
@@ -46,6 +49,7 @@ type Config struct {
 	RuleFiles    []string `config:"audit_rule_files"`    // List of rule files.
 	SocketType   string   `config:"socket_type"`         // Socket type to use with the kernel (unicast or multicast).
 	Immutable    bool     `config:"immutable"`           // Sets kernel audit config immutable.
+	IgnoreErrors bool     `config:"ignore_errors"`       // Ignore errors when reading and parsing rules, equivalent to auditctl -i.
 
 	// Tuning options (advanced, use with care)
 	ReassemblerMaxInFlight uint32        `config:"reassembler.max_in_flight"`
@@ -120,11 +124,19 @@ func (c Config) rules() []auditRule {
 }
 
 func (c *Config) loadRules() error {
+	var log *logp.Logger
+	if c.IgnoreErrors {
+		log = logp.NewLogger(moduleName)
+	}
+
 	var paths []string
 	for _, pattern := range c.RuleFiles {
 		absPattern, err := filepath.Abs(pattern)
 		if err != nil {
-			return fmt.Errorf("unable to get the absolute path for %s: %w", pattern, err)
+			if log == nil {
+				return fmt.Errorf("unable to get the absolute path for %s: %w", pattern, err)
+			}
+			log.Warnf("unable to get the absolute path for %s: %v", pattern, err)
 		}
 		files, err := filepath.Glob(absPattern)
 		if err != nil {
@@ -136,7 +148,7 @@ func (c *Config) loadRules() error {
 
 	knownRules := ruleSet{}
 
-	rules, err := readRules(bytes.NewBufferString(c.RulesBlob), "(audit_rules at auditbeat.yml)", knownRules)
+	rules, err := readRules(bytes.NewBufferString(c.RulesBlob), "(audit_rules at auditbeat.yml)", knownRules, log)
 	if err != nil {
 		return err
 	}
@@ -145,9 +157,13 @@ func (c *Config) loadRules() error {
 	for _, filename := range paths {
 		fHandle, err := os.Open(filename)
 		if err != nil {
-			return fmt.Errorf("unable to open rule file '%s': %w", filename, err)
+			if log == nil {
+				return fmt.Errorf("unable to open rule file '%s': %w", filename, err)
+			}
+			log.Warnf("unable to open rule file '%s': %v", filename, err)
+			continue
 		}
-		rules, err = readRules(fHandle, filename, knownRules)
+		rules, err = readRules(fHandle, filename, knownRules, log)
 		if err != nil {
 			return err
 		}
@@ -170,7 +186,11 @@ func (c Config) failureMode() (uint32, error) {
 	}
 }
 
-func readRules(reader io.Reader, source string, knownRules ruleSet) (rules []auditRule, err error) {
+// readRules reads the audit rules from reader, adding them to knownRules. If
+// log is nil, errors will result in an empty rules set being returned. Otherwise
+// errors will be logged as warnings and any successfully parsed rules will be
+// returned.
+func readRules(reader io.Reader, source string, knownRules ruleSet, log *logp.Logger) (rules []auditRule, err error) {
 	var errs multierror.Errors
 
 	s := bufio.NewScanner(reader)
@@ -207,8 +227,11 @@ func readRules(reader io.Reader, source string, knownRules ruleSet) (rules []aud
 		rules = append(rules, rule)
 	}
 
-	if len(errs) > 0 {
-		return nil, fmt.Errorf("failed loading rules: %w", errs.Err())
+	if len(errs) != 0 {
+		if log == nil {
+			return nil, fmt.Errorf("failed loading rules: %w", errs.Err())
+		}
+		log.Warnf("errors loading rules: %v", errs.Err())
 	}
 	return rules, nil
 }