diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2a9a71c13246..7ffa6e338eb6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -408,6 +408,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add support for location label as an optional configuration parameter in GCP metrics metricset. {issue}41550[41550] {pull}41626[41626] - Add support for podman metrics in docker module. {pull}41889[41889] - Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944] +- Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965] *Metricbeat* - Add benchmark module {pull}41801[41801] diff --git a/NOTICE.txt b/NOTICE.txt index ec0eaa95417c..df341f41dcae 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13617,11 +13617,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.11.4 +Version: v0.11.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.11.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.11.5/LICENSE.txt: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 6c6f683111b3..4ad15a8cce3f 100644 --- a/go.mod +++ b/go.mod @@ -185,7 +185,7 @@ require ( github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.9.0 github.com/elastic/elastic-agent-libs v0.17.4 - github.com/elastic/elastic-agent-system-metrics v0.11.4 + github.com/elastic/elastic-agent-system-metrics v0.11.5 github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/elastic/go-quark v0.2.0 github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 diff --git a/go.sum b/go.sum index 40b76c076ef3..4164bd72f3eb 100644 --- a/go.sum +++ b/go.sum @@ -340,8 +340,8 @@ github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7b github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= github.com/elastic/elastic-agent-libs v0.17.4 h1:kWK5Kn2EQjM97yHqbeXv+cFAIti4IiI9Qj8huM+lZzE= github.com/elastic/elastic-agent-libs v0.17.4/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= -github.com/elastic/elastic-agent-system-metrics v0.11.4 h1:Z/8CML5RKvGpi6/QUFok1K3EriBAv2kUAXnsk8hCifk= -github.com/elastic/elastic-agent-system-metrics v0.11.4/go.mod h1:TTW2ysv78uHBQ68hG8TXiaX1m6f29ZHgGWb8XONYsU8= +github.com/elastic/elastic-agent-system-metrics v0.11.5 h1:JSjXFEn8uYZ9hoC/GxZNMgJ622UoP96sjYP/49/Uvuo= +github.com/elastic/elastic-agent-system-metrics v0.11.5/go.mod h1:nzkrGajQA29YNcfP62gfzhxX9an3/xdQ3RmfQNw9YTI= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 2fc3930d8444..164abe5c91e3 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -265,6 +265,11 @@ metricbeat.modules: # Filter systemd services based on a name pattern #service.pattern_filter: ["ssh*", "nfs*"] + + # This option enables the use of performance counters to collect data for cpu/core metricset. + # Only effective for Windows. + # You should use this option if running beats on machins with more than 64 cores. + #use_performance_counters: true ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f1412be4b6f2..6cb4352b87d6 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -142,6 +142,11 @@ metricbeat.modules: # Filter systemd services based on a name pattern #service.pattern_filter: ["ssh*", "nfs*"] + # This option enables the use of performance counters to collect data for cpu/core metricset. + # Only effective for Windows. + # You should use this option if running beats on machins with more than 64 cores. + #use_performance_counters: true + #------------------------------ Aerospike Module ------------------------------ - module: aerospike metricsets: ["namespace"] diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 974df87cb0bc..777a6444ecae 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -81,3 +81,8 @@ # Filter systemd services based on a name pattern #service.pattern_filter: ["ssh*", "nfs*"] + + # This option enables the use of performance counters to collect data for cpu/core metricset. + # Only effective for Windows. + # You should use this option if running beats on machins with more than 64 cores. + #use_performance_counters: true diff --git a/metricbeat/module/system/core/_meta/docs.asciidoc b/metricbeat/module/system/core/_meta/docs.asciidoc index e70e55f0db71..751c4759a9c3 100644 --- a/metricbeat/module/system/core/_meta/docs.asciidoc +++ b/metricbeat/module/system/core/_meta/docs.asciidoc @@ -14,6 +14,10 @@ This metricset is available on: *`core.metrics`*:: This option controls what metrics are reported for each CPU core. The value is a list and two metric types are supported - `percentages` and `ticks`. The default value is `core.metrics: [percentages]`. +*`use_performance_counters`*:: This option enables the use of performance counters to +collect data for the CPU/core metricset. It is only effective on Windows. +You should use this option if running beats on machins with more than 64 cores. +The default value is `use_performance_counters: true` + [source,yaml] ---- @@ -21,4 +25,5 @@ metricbeat.modules: - module: system metricsets: [core] core.metrics: [percentages, ticks] + #use_performance_counters: true ---- diff --git a/metricbeat/module/system/core/config.go b/metricbeat/module/system/core/config.go index 8ac1d2f9575a..940a23282165 100644 --- a/metricbeat/module/system/core/config.go +++ b/metricbeat/module/system/core/config.go @@ -34,8 +34,9 @@ const ( // Config for the system core metricset. type Config struct { - Metrics []string `config:"core.metrics"` - CPUTicks *bool `config:"cpu_ticks"` // Deprecated. + Metrics []string `config:"core.metrics"` + CPUTicks *bool `config:"cpu_ticks"` // Deprecated. + UserPerformanceCounters bool `config:"use_performance_counters"` } // Validate validates the core config. @@ -65,5 +66,6 @@ func (c Config) Validate() (metrics.MetricOpts, error) { } var defaultConfig = Config{ - Metrics: []string{percentages}, + Metrics: []string{percentages}, + UserPerformanceCounters: true, } diff --git a/metricbeat/module/system/core/core.go b/metricbeat/module/system/core/core.go index 1bf2f3f3a3db..fc5e7b9e3945 100644 --- a/metricbeat/module/system/core/core.go +++ b/metricbeat/module/system/core/core.go @@ -41,6 +41,7 @@ type MetricSet struct { mb.BaseMetricSet opts metrics.MetricOpts cores *metrics.Monitor + sys resolve.Resolver } // New returns a new core MetricSet. @@ -58,11 +59,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if config.CPUTicks != nil && *config.CPUTicks { config.Metrics = append(config.Metrics, "ticks") } - sys := base.Module().(resolve.Resolver) + sys, ok := base.Module().(resolve.Resolver) + if !ok { + return nil, fmt.Errorf("unexpected module type: %T", base.Module()) + } + + cpuOpts := make([]metrics.OptionFunc, 0) + if config.UserPerformanceCounters { + cpuOpts = append(cpuOpts, metrics.WithWindowsPerformanceCounter()) + } + cpu, err := metrics.New(sys, cpuOpts...) + if err != nil { + return nil, fmt.Errorf("error initializing system.cpu metricset: %w", err) + } + return &MetricSet{ BaseMetricSet: base, opts: opts, - cores: metrics.New(sys), + cores: cpu, + sys: sys, }, nil } @@ -109,6 +124,5 @@ func (m *MetricSet) Diagnostics() []diagnostics.DiagnosticSetup { } func (m *MetricSet) getDiagData() []byte { - sys := m.BaseMetricSet.Module().(resolve.Resolver) - return diagnostics.GetRawFileOrErrorString(sys, "/proc/stat") + return diagnostics.GetRawFileOrErrorString(m.sys, "/proc/stat") } diff --git a/metricbeat/module/system/cpu/_meta/docs.asciidoc b/metricbeat/module/system/cpu/_meta/docs.asciidoc index fb83c41e500a..b0d9da584591 100644 --- a/metricbeat/module/system/cpu/_meta/docs.asciidoc +++ b/metricbeat/module/system/cpu/_meta/docs.asciidoc @@ -15,6 +15,10 @@ This metricset is available on: is a list and three metric types are supported - `percentages`, `normalized_percentages`, and `ticks`. The default value is `cpu.metrics: [percentages]`. +*`use_performance_counters`*:: This option enables the use of performance counters to +collect data for the CPU/core metricset. It is only effective on Windows. +You should use this option if running beats on machins with more than 64 cores. +The default value is `use_performance_counters: true` + [source,yaml] ---- @@ -22,4 +26,5 @@ metricbeat.modules: - module: system metricsets: [cpu] cpu.metrics: [percentages, normalized_percentages, ticks] + #use_performance_counters: true ---- diff --git a/metricbeat/module/system/cpu/config.go b/metricbeat/module/system/cpu/config.go index ef9d78fe0ced..7cfffed57a53 100644 --- a/metricbeat/module/system/cpu/config.go +++ b/metricbeat/module/system/cpu/config.go @@ -35,8 +35,9 @@ const ( // Config for the system cpu metricset. type Config struct { - Metrics []string `config:"cpu.metrics"` - CPUTicks *bool `config:"cpu_ticks"` // Deprecated. + Metrics []string `config:"cpu.metrics"` + CPUTicks *bool `config:"cpu_ticks"` // Deprecated. + UserPerformanceCounters bool `config:"use_performance_counters"` } // Validate validates the cpu config. @@ -69,5 +70,6 @@ func (c Config) Validate() (metrics.MetricOpts, error) { } var defaultConfig = Config{ - Metrics: []string{percentages, normalizedPercentages}, + Metrics: []string{percentages, normalizedPercentages}, + UserPerformanceCounters: true, } diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 8eb06c2427bd..ace37b809e85 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -44,6 +44,7 @@ type MetricSet struct { mb.BaseMetricSet opts metrics.MetricOpts cpu *metrics.Monitor + sys resolve.Resolver } // New is a mb.MetricSetFactory that returns a cpu.MetricSet. @@ -61,11 +62,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if config.CPUTicks != nil && *config.CPUTicks { config.Metrics = append(config.Metrics, "ticks") } - sys := base.Module().(resolve.Resolver) + sys, ok := base.Module().(resolve.Resolver) + if !ok { + return nil, fmt.Errorf("unexpected module type: %T", base.Module()) + } + + cpuOpts := make([]metrics.OptionFunc, 0) + if config.UserPerformanceCounters { + cpuOpts = append(cpuOpts, metrics.WithWindowsPerformanceCounter()) + } + cpu, err := metrics.New(sys, cpuOpts...) + + if err != nil { + return nil, fmt.Errorf("error initializing system.cpu metricset: %w", err) + } return &MetricSet{ BaseMetricSet: base, opts: opts, - cpu: metrics.New(sys), + cpu: cpu, + sys: sys, }, nil } @@ -125,13 +140,11 @@ func (m *MetricSet) Diagnostics() []diagnostics.DiagnosticSetup { } func (m *MetricSet) fetchRawCPU() []byte { - sys := m.BaseMetricSet.Module().(resolve.Resolver) - return diagnostics.GetRawFileOrErrorString(sys, "/proc/stat") + return diagnostics.GetRawFileOrErrorString(m.sys, "/proc/stat") } func (m *MetricSet) fetchCPUInfo() []byte { - sys := m.BaseMetricSet.Module().(resolve.Resolver) - return diagnostics.GetRawFileOrErrorString(sys, "/proc/cpuinfo") + return diagnostics.GetRawFileOrErrorString(m.sys, "/proc/cpuinfo") } // copyFieldsOrDefault copies the field specified by key to the given map. It will diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 8c4250caecea..707b6419d9ed 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -142,6 +142,11 @@ metricbeat.modules: # Filter systemd services based on a name pattern #service.pattern_filter: ["ssh*", "nfs*"] + # This option enables the use of performance counters to collect data for cpu/core metricset. + # Only effective for Windows. + # You should use this option if running beats on machins with more than 64 cores. + #use_performance_counters: true + #------------------------------- ActiveMQ Module ------------------------------- - module: activemq metricsets: ['broker', 'queue', 'topic']