metricbeat: collect number of threads per process

Upgrades elastic-agent-system-metrics to v0.5.0 in order to collect the number of threads per process.
elastic · Sep 29, 2023 · 98b2b4d · 98b2b4d
1 parent 9e503b4
commit 98b2b4d
Show file tree

Hide file tree

Showing 13 changed files with 43 additions and 17 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -21,6 +21,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Metricbeat*
 
+ - System module now collects the number of threads per process.
+The elastic-agent-system-metrics was updated to v0.5.0 as the number of threads
+is collected by it.
 
 *Osquerybeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -13025,12 +13025,12 @@ these terms.
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.6.1
+Dependency : github.com/AndersonQ/elastic-agent-system-metrics
+Version: v0.0.0-20230926055627-a33e2d04a748
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.6.1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/!anderson!q/elastic-agent-system-metrics@v0.0.0-20230926055627-a33e2d04a748/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/go.mod b/go.mod
@@ -407,3 +407,5 @@ replace (
 
 // Exclude this version because the version has an invalid checksum.
 exclude github.com/docker/distribution v2.8.0+incompatible
+
+replace github.com/elastic/elastic-agent-system-metrics => github.com/AndersonQ/elastic-agent-system-metrics v0.0.0-20230929151341-2f28d53433fc
diff --git a/go.sum b/go.sum
@@ -79,6 +79,8 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4=
 github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA=
+github.com/AndersonQ/elastic-agent-system-metrics v0.0.0-20230929151341-2f28d53433fc h1:f7V2y9K6VV+h6jxYB3gkumxLjY+tYXyn5ZKopsEs7N4=
+github.com/AndersonQ/elastic-agent-system-metrics v0.0.0-20230929151341-2f28d53433fc/go.mod h1:e2jSj1Fnnyhl21p2g52qhYCyyizX9Lup3z2m/3E9kvg=
 github.com/Azure/azure-amqp-common-go/v3 v3.2.1 h1:uQyDk81yn5hTP1pW4Za+zHzy97/f4vDz9o1d/exI4j4=
 github.com/Azure/azure-amqp-common-go/v3 v3.2.1/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI=
 github.com/Azure/azure-event-hubs-go/v3 v3.3.15 h1:2yAik9gS5Qjs4Z6EM1B6d0tfZJ506B/0oA08JlRQxb8=
@@ -657,8 +659,6 @@ github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a
 github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
-github.com/elastic/elastic-agent-system-metrics v0.6.1 h1:LCN1lvQTkdUuU/rKlpKyVMDU/G/I8/iZWCaW6K+mo4o=
-github.com/elastic/elastic-agent-system-metrics v0.6.1/go.mod h1:Bj8XM/uNKm553blQHkGNEICRLGnVEtw8yttmV5vBngA=
 github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
 github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=

diff --git a/libbeat/generator/fields/module_fields_collector.go b/libbeat/generator/fields/module_fields_collector.go
@@ -19,6 +19,7 @@ package fields
 
 import (
 	"io/ioutil"
+	"os"
 	"path/filepath"
 )
 
@@ -30,9 +31,9 @@ var indentByModule = map[string]int{
 	"protos":     8,
 }
 
-// GetModules returns a the list of modules for the given modules directory
+// GetModules returns a list of modules for the given modules directory
 func GetModules(modulesDir string) ([]string, error) {
-	moduleInfos, err := ioutil.ReadDir(modulesDir)
+	moduleInfos, err := os.ReadDir(modulesDir)
 	if err != nil {
 		return nil, err
 	}

diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
@@ -717,6 +717,7 @@ def extract_fields(doc_list, name):
             if not os.path.isfile(path):
                 path = os.path.abspath(os.path.dirname(
                     __file__) + "../../../../_meta/fields.common.yml")
+
             with open(path, encoding="utf-8") as f2:
                 content = f2.read()
 
@@ -788,7 +789,7 @@ def output_count(self, pred, output_file=None):
     def assert_fields_are_documented(self, evt):
         """
         Assert that all keys present in evt are documented in fields.yml.
-        This reads from the global fields.yml, means `make collect` has to be run before the check.
+        This reads from the global fields.yml, means `mage fields` has to be run before the check.
         """
         expected_fields, dict_fields, aliases = self.load_fields()
         flat = self.flatten_object(evt, dict_fields)

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -62020,6 +62020,16 @@ alias to: process.pgid
 
 --
 
+*`system.process.num_threads`*::
++
+--
+Number of threads in the process
+
+
+type: integer
+
+--
+
 *`system.process.cmdline`*::
 +
 --

diff --git a/metricbeat/module/system/fields.go b/metricbeat/module/system/fields.go
diff --git a/metricbeat/module/system/process/_meta/data.json b/metricbeat/module/system/process/_meta/data.json
@@ -250,10 +250,11 @@
                 "share": 14946304,
                 "size": 1725992960
             },
-            "state": "sleeping"
+            "state": "sleeping",
+            "num_threads": 42
         }
     },
     "user": {
         "name": "alexk"
     }
-}
+}
diff --git a/metricbeat/module/system/process/_meta/fields.yml b/metricbeat/module/system/process/_meta/fields.yml
@@ -24,6 +24,10 @@
       type: alias
       path: process.pgid
       migration: true
+    - name: num_threads
+      type: integer
+      description: >
+        Number of threads in the process
     - name: cmdline
       type: keyword
       description: >
@@ -345,7 +349,7 @@
             - name: stats.user.pct
               type: scaled_float
               description: time the cgroup spent in user space, as a percentage of total CPU time
-            
+
             - name: stats.user.norm.pct
               type: scaled_float
               description: time the cgroup spent in user space, as a percentage of total CPU time, normalized by CPU count.
@@ -419,7 +423,7 @@
               type: long
               format: bytes
               description: memory max threshhold
-            
+
             - name: mem.events
               type: group
               description: number of times the controller tripped a given usage level

diff --git a/metricbeat/module/system/process/process.go b/metricbeat/module/system/process/process.go
@@ -32,7 +32,7 @@ import (
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
 )
 
-var debugf = logp.MakeDebug("system.process")
+var debugf = logp.NewLogger("system.process").Debugf
 
 func init() {
 	mb.Registry.MustAddMetricSet("system", "process", New,

diff --git a/metricbeat/module/system/process/process_test.go b/metricbeat/module/system/process/process_test.go
@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 	_ "github.com/elastic/beats/v7/metricbeat/module/system"
@@ -32,7 +33,9 @@ import (
 )
 
 func TestFetch(t *testing.T) {
-	logp.DevelopmentSetup()
+	err := logp.DevelopmentSetup()
+	require.NoError(t, err, "could not set the logger to DevelopmentSetup")
+
 	f := mbtest.NewReportingMetricSetV2Error(t, getConfig())
 	events, errs := mbtest.ReportingFetchV2Error(f)
 	assert.Empty(t, errs)
@@ -44,6 +47,7 @@ func TestFetch(t *testing.T) {
 	assert.Empty(t, errs)
 	assert.NotEmpty(t, events)
 
+	t.Logf("fetched %d events, showing events[0]:", len(events))
 	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
 		events[0].BeatEvent("system", "process").Fields.StringToPrint())
 }

diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py
@@ -112,7 +112,7 @@
 # for some kernel level processes. fd is also part of the system process, but
 # is not available on all OSes and requires root to read for all processes.
 # cgroup is only available on linux.
-SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state"]
+SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state", "num_threads"]
 
 
 class Test(metricbeat.BaseTest):
@@ -439,7 +439,7 @@ def test_process(self):
     @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os")
     def test_process_unix(self):
         """
-        Test system/process output for fields specific of unix systems.
+        Test system/process output checking it has got all expected fields specific of unix systems and no extra ones.
         """
 
         self.render_config_template(
Original file line number	Diff line number	Diff line change
Expand Up		@@ -407,3 +407,5 @@ replace (

		// Exclude this version because the version has an invalid checksum.
		exclude github.com/docker/distribution v2.8.0+incompatible

		replace github.com/elastic/elastic-agent-system-metrics => github.com/AndersonQ/elastic-agent-system-metrics v0.0.0-20230929151341-2f28d53433fc