Merge branch 'main' into openai-metrics

.github/CODEOWNERS

@@ -26,9 +26,9 @@ CHANGELOG*
 /.github/CODEOWNERS @elastic/beats-tech-leads
 /auditbeat/ @elastic/sec-linux-platform
 /deploy/ @elastic/elastic-agent-data-plane
-/deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring
+/deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/elastic-agent-control-plane
 /dev-tools/ @elastic/elastic-agent-data-plane
-/dev-tools/kubernetes @elastic/obs-ds-hosted-services
+/dev-tools/kubernetes @elastic/elastic-agent-data-plane @elastic/elastic-agent-control-plane
 /docs/ @elastic/elastic-agent-data-plane
 /filebeat @elastic/elastic-agent-data-plane
 /filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
@@ -57,10 +57,11 @@ CHANGELOG*
 /heartbeat/ @elastic/obs-ds-hosted-services
 /journalbeat @elastic/elastic-agent-data-plane
 /libbeat/ @elastic/elastic-agent-data-plane
+/libbeat/autodiscover/providers/kubernetes @elastic/elastic-agent-data-plane @elastic/elastic-agent-control-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
 /libbeat/management @elastic/elastic-agent-control-plane
 /libbeat/processors/add_cloud_metadata @elastic/obs-ds-hosted-services
-/libbeat/processors/add_kubernetes_metadata @elastic/obs-cloudnative-monitoring
+/libbeat/processors/add_kubernetes_metadata @elastic/elastic-agent-data-plane
 /libbeat/processors/cache/ @elastic/security-service-integrations
 /libbeat/processors/community_id/ @elastic/sec-deployment-and-devices
 /libbeat/processors/decode_xml/ @elastic/security-service-integrations

CHANGELOG.next.asciidoc

@@ -128,6 +128,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - auditd: Update syscall names for Linux 6.11. {pull}41558[41558]
 - hasher: Geneneral improvements and fixes. {pull}41863[41863]
 - hasher: Add a cached hasher for upcoming backend. {pull}41952[41952]
+- Split common tty definitions. {pull}42004[42004]
 
 *Filebeat*
 
@@ -193,6 +194,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix handling of http_endpoint request exceeding memory limits. {issue}41764[41764] {pull}41765[41765]
 - Rate limiting fixes in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41583[41583]
 - Redact authorization headers in HTTPJSON debug logs. {pull}41920[41920]
+- Further rate limiting fix in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 
 *Heartbeat*
 
@@ -365,6 +367,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add support for SSL and Proxy configurations for websoket type in streaming input. {pull}41934[41934]
 - AWS S3 input registry cleanup for untracked s3 objects. {pull}41694[41694]
 - The environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true` enables internal logs tracer for the azure-eventhub input. {issue}41931[41931] {pull}41932[41932]
+- Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 
 *Auditbeat*
 
@@ -416,6 +419,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Only watch metadata for ReplicaSets in metricbeat k8s module {pull}41289[41289]
 - Add support for region/zone for Vertex AI service in GCP module {pull}41551[41551]
 - Add support for location label as an optional configuration parameter in GCP metrics metricset. {issue}41550[41550] {pull}41626[41626]
+- Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944]
+- Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965]
 
 *Metricbeat*
 - Add benchmark module {pull}41801[41801]

NOTICE.txt

@@ -12699,11 +12699,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.11.4
+Version: v0.11.5
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].4/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].5/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/helper/tty/tty.go

@@ -0,0 +1,76 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tty
+
+type TTYType int
+
+const (
+	TTYUnknown TTYType = iota
+	Pts
+	TTY
+	TTYConsole
+)
+
+const (
+	ptsMinMajor     = 136
+	ptsMaxMajor     = 143
+	ttyMajor        = 4
+	consoleMaxMinor = 63
+	ttyMaxMinor     = 255
+)
+
+type TTYDev struct {
+	Minor   uint32
+	Major   uint32
+	Winsize TTYWinsize
+	Termios TTYTermios
+}
+
+type TTYWinsize struct {
+	Rows uint16
+	Cols uint16
+}
+
+type TTYTermios struct {
+	CIflag uint32
+	COflag uint32
+	CLflag uint32
+	CCflag uint32
+}
+
+// interactiveFromTTY returns if this is an interactive tty device.
+func InteractiveFromTTY(tty TTYDev) bool {
+	return TTYUnknown != GetTTYType(tty.Major, tty.Minor)
+}
+
+// getTTYType returns the type of a TTY device based on its major and minor numbers.
+func GetTTYType(major uint32, minor uint32) TTYType {
+	if major >= ptsMinMajor && major <= ptsMaxMajor {
+		return Pts
+	}
+
+	if ttyMajor == major {
+		if minor <= consoleMaxMinor {
+			return TTYConsole
+		} else if minor > consoleMaxMinor && minor <= ttyMaxMinor {
+			return TTY
+		}
+	}
+
+	return TTYUnknown
+}

dev-tools/packaging/templates/ironbank/auditbeat/hardening_manifest.yaml

@@ -52,9 +52,6 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - email: "[email protected]"
-    name: "Nassim Kammah"
-    username: "nassim.kammah"
   - email: "[email protected]"
     name: "Julien lind"
     username: "julien.lind"

dev-tools/packaging/templates/ironbank/filebeat/hardening_manifest.yaml

@@ -52,9 +52,6 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - email: "[email protected]"
-    name: "Nassim Kammah"
-    username: "nassim.kammah"
   - email: "[email protected]"
     name: "Julien lind"
     username: "julien.lind"

dev-tools/packaging/templates/ironbank/heartbeat/hardening_manifest.yaml

@@ -52,9 +52,6 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - email: "[email protected]"
-    name: "Nassim Kammah"
-    username: "nassim.kammah"
   - email: "[email protected]"
     name: "Julien lind"
     username: "julien.lind"

dev-tools/packaging/templates/ironbank/metricbeat/hardening_manifest.yaml

@@ -52,9 +52,6 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - email: "[email protected]"
-    name: "Nassim Kammah"
-    username: "nassim.kammah"
   - email: "[email protected]"
     name: "Julien lind"
     username: "julien.lind"

dev-tools/packaging/templates/ironbank/packetbeat/hardening_manifest.yaml

@@ -52,9 +52,6 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - email: "[email protected]"
-    name: "Nassim Kammah"
-    username: "nassim.kammah"
   - email: "[email protected]"
     name: "Julien lind"
     username: "julien.lind"

go.mod

@@ -178,7 +178,7 @@ require (
 	github.com/elastic/ebpfevents v0.6.0
 	github.com/elastic/elastic-agent-autodiscover v0.9.0
 	github.com/elastic/elastic-agent-libs v0.17.4
-	github.com/elastic/elastic-agent-system-metrics v0.11.4
+	github.com/elastic/elastic-agent-system-metrics v0.11.5
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/elastic/go-quark v0.2.0
 	github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727

go.sum

@@ -324,8 +324,8 @@ github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7b
 github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
 github.com/elastic/elastic-agent-libs v0.17.4 h1:kWK5Kn2EQjM97yHqbeXv+cFAIti4IiI9Qj8huM+lZzE=
 github.com/elastic/elastic-agent-libs v0.17.4/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
-github.com/elastic/elastic-agent-system-metrics v0.11.4 h1:Z/8CML5RKvGpi6/QUFok1K3EriBAv2kUAXnsk8hCifk=
-github.com/elastic/elastic-agent-system-metrics v0.11.4/go.mod h1:TTW2ysv78uHBQ68hG8TXiaX1m6f29ZHgGWb8XONYsU8=
+github.com/elastic/elastic-agent-system-metrics v0.11.5 h1:JSjXFEn8uYZ9hoC/GxZNMgJ622UoP96sjYP/49/Uvuo=
+github.com/elastic/elastic-agent-system-metrics v0.11.5/go.mod h1:nzkrGajQA29YNcfP62gfzhxX9an3/xdQ3RmfQNw9YTI=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
 github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=

metricbeat/docs/fields.asciidoc

@@ -32162,6 +32162,27 @@ type: keyword
 
 --
 
+*`elasticsearch.index.tier_preference`*::
++
+--
+type: keyword
+
+--
+
+*`elasticsearch.index.creation_date`*::
++
+--
+type: date
+
+--
+
+*`elasticsearch.index.version`*::
++
+--
+type: keyword
+
+--
+
 *`elasticsearch.index.name`*::
 +
 --

metricbeat/docs/modules/system.asciidoc

@@ -265,6 +265,11 @@ metricbeat.modules:
 
   # Filter systemd services based on a name pattern
   #service.pattern_filter: ["ssh*", "nfs*"]
+
+  # This option enables the use of performance counters to collect data for cpu/core metricset.
+  # Only effective for Windows.
+  # You should use this option if running beats on machins with more than 64 cores.
+  #use_performance_counters: true
 ----
 
 [float]

metricbeat/metricbeat.reference.yml

@@ -142,6 +142,11 @@ metricbeat.modules:
   # Filter systemd services based on a name pattern
   #service.pattern_filter: ["ssh*", "nfs*"]
 
+  # This option enables the use of performance counters to collect data for cpu/core metricset.
+  # Only effective for Windows.
+  # You should use this option if running beats on machins with more than 64 cores.
+  #use_performance_counters: true
+
 #------------------------------ Aerospike Module ------------------------------
 - module: aerospike
   metricsets: ["namespace"]

metricbeat/module/elasticsearch/cluster_stats/data.go

@@ -231,7 +231,7 @@ func eventMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.I
 	}
 
 	clusterStateMetrics := []string{"version", "master_node", "nodes", "routing_table"}
-	clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics)
+	clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics, []string{})
 	if err != nil {
 		return fmt.Errorf("failed to get cluster state from Elasticsearch: %w", err)
 	}

metricbeat/module/elasticsearch/elasticsearch.go

@@ -288,13 +288,21 @@ func GetLicense(http *helper.HTTP, resetURI string) (*License, error) {
 }
 
 // GetClusterState returns cluster state information.
-func GetClusterState(http *helper.HTTP, resetURI string, metrics []string) (mapstr.M, error) {
+func GetClusterState(http *helper.HTTP, resetURI string, metrics []string, filterPaths []string) (mapstr.M, error) {
+	queryParams := []string{"local=true"}
 	clusterStateURI := "_cluster/state"
 	if len(metrics) > 0 {
 		clusterStateURI += "/" + strings.Join(metrics, ",")
 	}
 
-	content, err := fetchPath(http, resetURI, clusterStateURI, "local=true")
+	if len(filterPaths) > 0 {
+		filterPathQueryParam := "filter_path=" + strings.Join(filterPaths, ",")
+		queryParams = append(queryParams, filterPathQueryParam)
+	}
+
+	queryString := strings.Join(queryParams, "&")
+
+	content, err := fetchPath(http, resetURI, clusterStateURI, queryString)
 	if err != nil {
 		return nil, err
 	}
@@ -304,6 +312,28 @@ func GetClusterState(http *helper.HTTP, resetURI string, metrics []string) (maps
 	return clusterState, err
 }
 
+func GetIndexSettings(http *helper.HTTP, resetURI string, indexPattern string, filterPaths []string) (mapstr.M, error) {
+
+	queryParams := []string{"local=true", "expand_wildcards=hidden,all"}
+	indicesSettingsURI := indexPattern + "/_settings"
+
+	if len(filterPaths) > 0 {
+		filterPathQueryParam := "filter_path=" + strings.Join(filterPaths, ",")
+		queryParams = append(queryParams, filterPathQueryParam)
+	}
+
+	queryString := strings.Join(queryParams, "&")
+
+	content, err := fetchPath(http, resetURI, indicesSettingsURI, queryString)
+	if err != nil {
+		return nil, err
+	}
+
+	var indicesSettings map[string]interface{}
+	err = json.Unmarshal(content, &indicesSettings)
+	return indicesSettings, err
+}
+
 // GetClusterSettingsWithDefaults returns cluster settings.
 func GetClusterSettingsWithDefaults(http *helper.HTTP, resetURI string, filterPaths []string) (mapstr.M, error) {
 	return GetClusterSettings(http, resetURI, true, filterPaths)