[vSphere][network] Add support for new metrics in network metricset (e…

…lastic#40559) * initial commit for network metricset * mage update * added changelog entry * fix changelog * resolve review comments * add support for DVPG * make generic changes - eventMapping() -> mapEvent() - MetricSet -> {Host|Network}MetricSet * resolved review comments * assetsName -> assetsNames * make update * resolved review comment: bytesMultiplier * resolve review comments * resolve review comments * update changelog entry * update sample event json * minor changes --------- Co-authored-by: Ishleen Kaur <[email protected]>
niraj-elastic · Aug 28, 2024 · 93ee5ca · 93ee5ca
1 parent 7960af6
commit 93ee5ca
Show file tree

Hide file tree

Showing 25 changed files with 664 additions and 67 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -316,7 +316,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new memory/cgroup metrics to Kibana module {pull}37232[37232]
 - Add SSL support to mysql module {pull}37997[37997]
 - Add SSL support for aerospike module {pull}38126[38126]
-
+- Add new metricset network for the vSphere module. {pull}40559[40559]
 
 *Metricbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -67416,6 +67416,105 @@ type: keyword
 Number of virtual machines on the host.
 
 
+type: long
+
+--
+
+[float]
+=== network
+
+Network-related information.
+
+
+
+*`vsphere.network.accessible`*::
++
+--
+Indicates whether at least one host is configured to provide this network.
+
+
+type: boolean
+
+--
+
+*`vsphere.network.config.status`*::
++
+--
+Indicates whether the system has detected a configuration issue.
+
+
+type: keyword
+
+--
+
+
+*`vsphere.network.host.names`*::
++
+--
+Names of the hosts connected to this network.
+
+
+type: keyword
+
+--
+
+*`vsphere.network.host.count`*::
++
+--
+Number of hosts connected to this network.
+
+
+type: long
+
+--
+
+*`vsphere.network.name`*::
++
+--
+Name of the network.
+
+
+type: keyword
+
+--
+
+*`vsphere.network.status`*::
++
+--
+General health of the Network.
+
+
+type: keyword
+
+--
+
+*`vsphere.network.type`*::
++
+--
+Type of the network (e.g., Network(Standard), DistributedVirtualport).
+
+
+type: keyword
+
+--
+
+
+*`vsphere.network.vm.names`*::
++
+--
+Names of the virtual machines connected to this network.
+
+
+type: keyword
+
+--
+
+*`vsphere.network.vm.count`*::
++
+--
+Number of virtual machines connected to this network.
+
+
 type: long
 
 --

diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc
@@ -11,7 +11,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5.
 
-By default it enables the metricsets `datastore`, `host` and `virtualmachine`.
+By default it enables the metricsets `network`, `datastore`, `host` and `virtualmachine`.
 
 [float]
 === Dashboard
@@ -35,7 +35,7 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: vsphere
   enabled: true
-  metricsets: ["datastore", "host", "virtualmachine"]
+  metricsets: ["datastore", "host", "virtualmachine", "network"]
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]
@@ -57,12 +57,16 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-vsphere-host,host>>
 
+* <<metricbeat-metricset-vsphere-network,network>>
+
 * <<metricbeat-metricset-vsphere-virtualmachine,virtualmachine>>
 
 include::vsphere/datastore.asciidoc[]
 
 include::vsphere/host.asciidoc[]
 
+include::vsphere/network.asciidoc[]
+
 include::vsphere/virtualmachine.asciidoc[]
 
 :edit_url!:
diff --git a/metricbeat/docs/modules/vsphere/network.asciidoc b/metricbeat/docs/modules/vsphere/network.asciidoc
@@ -0,0 +1,29 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/vsphere/network/_meta/docs.asciidoc
+
+
+[[metricbeat-metricset-vsphere-network]]
+=== vSphere network metricset
+
+beta[]
+
+include::../../../module/vsphere/network/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+:edit_url:
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-vsphere,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/vsphere/network/_meta/data.json[]
+----
+:edit_url!:
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -311,8 +311,9 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-module-uwsgi,uWSGI>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-uwsgi-status,status>>   
 |<<metricbeat-module-vsphere,vSphere>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.3+| .3+|  |<<metricbeat-metricset-vsphere-datastore,datastore>>   
+.4+| .4+|  |<<metricbeat-metricset-vsphere-datastore,datastore>>   
 |<<metricbeat-metricset-vsphere-host,host>>   
+|<<metricbeat-metricset-vsphere-network,network>> beta[]  
 |<<metricbeat-metricset-vsphere-virtualmachine,virtualmachine>>   
 |<<metricbeat-module-windows,Windows>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .2+| .2+|  |<<metricbeat-metricset-windows-perfmon,perfmon>>   

diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -1007,7 +1007,7 @@ metricbeat.modules:
 #------------------------------- VSphere Module -------------------------------
 - module: vsphere
   enabled: true
-  metricsets: ["datastore", "host", "virtualmachine"]
+  metricsets: ["datastore", "host", "virtualmachine", "network"]
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]

diff --git a/metricbeat/module/vsphere/_meta/README.md b/metricbeat/module/vsphere/_meta/README.md
@@ -31,6 +31,7 @@ Now setup your metricbeat config to connect to Govcsim:
     - datastore
     - host
     - virtualmachine
+    - network
   enabled: true
   period: 5s
   hosts: ["https://127.0.0.1:8989/sdk"]

diff --git a/metricbeat/module/vsphere/_meta/config.reference.yml b/metricbeat/module/vsphere/_meta/config.reference.yml
@@ -1,6 +1,6 @@
 - module: vsphere
   enabled: true
-  metricsets: ["datastore", "host", "virtualmachine"]
+  metricsets: ["datastore", "host", "virtualmachine", "network"]
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]

diff --git a/metricbeat/module/vsphere/_meta/config.yml b/metricbeat/module/vsphere/_meta/config.yml
@@ -3,6 +3,7 @@
   #  - datastore
   #  - host
   #  - virtualmachine
+  #  - network
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]

diff --git a/metricbeat/module/vsphere/_meta/docs.asciidoc b/metricbeat/module/vsphere/_meta/docs.asciidoc
@@ -1,6 +1,6 @@
 The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5.
 
-By default it enables the metricsets `datastore`, `host` and `virtualmachine`.
+By default it enables the metricsets `network`, `datastore`, `host` and `virtualmachine`.
 
 [float]
 === Dashboard

diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go
diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go
@@ -23,7 +23,8 @@ import (
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
-func (m *MetricSet) eventMapping(hs mo.HostSystem, data *metricData) mapstr.M {
+func (m *HostMetricSet) mapEvent(hs mo.HostSystem, data *metricData) mapstr.M {
+	const bytesMultiplier int64 = 1024 * 1024
 	event := mapstr.M{
 		"name":   hs.Summary.Config.Name,
 		"status": hs.Summary.OverallStatus,
@@ -35,7 +36,7 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, data *metricData) mapstr.M {
 
 	if hw := hs.Summary.Hardware; hw != nil {
 		totalCPU := int64(hw.CpuMhz) * int64(hw.NumCpuCores)
-		usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024
+		usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * bytesMultiplier
 		event.Put("cpu.total.mhz", totalCPU)
 		event.Put("cpu.free.mhz", totalCPU-int64(hs.Summary.QuickStats.OverallCpuUsage))
 		event.Put("memory.used.bytes", usedMemory)
@@ -45,28 +46,29 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, data *metricData) mapstr.M {
 		m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest")
 	}
 
-	if len(data.assetsName.outputVmNames) > 0 {
-		event.Put("vm.names", data.assetsName.outputVmNames)
-		event.Put("vm.count", len(data.assetsName.outputVmNames))
+	if len(data.assetNames.outputVmNames) > 0 {
+		event.Put("vm.names", data.assetNames.outputVmNames)
+		event.Put("vm.count", len(data.assetNames.outputVmNames))
 	}
 
-	if len(data.assetsName.outputDsNames) > 0 {
-		event.Put("datastore.names", data.assetsName.outputDsNames)
-		event.Put("datastore.count", len(data.assetsName.outputDsNames))
+	if len(data.assetNames.outputDsNames) > 0 {
+		event.Put("datastore.names", data.assetNames.outputDsNames)
+		event.Put("datastore.count", len(data.assetNames.outputDsNames))
 	}
 
-	if len(data.assetsName.outputNetworkNames) > 0 {
-		event.Put("network_names", data.assetsName.outputNetworkNames)
-		event.Put("network.names", data.assetsName.outputNetworkNames)
-		event.Put("network.count", len(data.assetsName.outputNetworkNames))
+	if len(data.assetNames.outputNetworkNames) > 0 {
+		event.Put("network_names", data.assetNames.outputNetworkNames)
+		event.Put("network.names", data.assetNames.outputNetworkNames)
+		event.Put("network.count", len(data.assetNames.outputNetworkNames))
 	}
 
 	return event
 }
 
 func mapPerfMetricToEvent(event mapstr.M, perfMetricMap map[string]interface{}) {
+	const bytesMultiplier int64 = 1024
 	if val, exist := perfMetricMap["disk.capacity.usage.average"]; exist {
-		event.Put("disk.capacity.usage.bytes", val.(int64)*1000)
+		event.Put("disk.capacity.usage.bytes", val.(int64)*bytesMultiplier)
 	}
 	if val, exist := perfMetricMap["disk.deviceLatency.average"]; exist {
 		event.Put("disk.devicelatency.average.ms", val)
@@ -75,23 +77,23 @@ func mapPerfMetricToEvent(event mapstr.M, perfMetricMap map[string]interface{})
 		event.Put("disk.latency.total.ms", val)
 	}
 	if val, exist := perfMetricMap["disk.usage.average"]; exist {
-		event.Put("disk.total.bytes", val.(int64)*1000)
+		event.Put("disk.total.bytes", val.(int64)*bytesMultiplier)
 	}
 	if val, exist := perfMetricMap["disk.read.average"]; exist {
-		event.Put("disk.read.bytes", val.(int64)*1000)
+		event.Put("disk.read.bytes", val.(int64)*bytesMultiplier)
 	}
 	if val, exist := perfMetricMap["disk.write.average"]; exist {
-		event.Put("disk.write.bytes", val.(int64)*1000)
+		event.Put("disk.write.bytes", val.(int64)*bytesMultiplier)
 	}
 
 	if val, exist := perfMetricMap["net.transmitted.average"]; exist {
-		event.Put("network.bandwidth.transmitted.bytes", val.(int64)*1000)
+		event.Put("network.bandwidth.transmitted.bytes", val.(int64)*bytesMultiplier)
 	}
 	if val, exist := perfMetricMap["net.received.average"]; exist {
-		event.Put("network.bandwidth.received.bytes", val.(int64)*1000)
+		event.Put("network.bandwidth.received.bytes", val.(int64)*bytesMultiplier)
 	}
 	if val, exist := perfMetricMap["net.usage.average"]; exist {
-		event.Put("network.bandwidth.total.bytes", val.(int64)*1000)
+		event.Put("network.bandwidth.total.bytes", val.(int64)*bytesMultiplier)
 	}
 
 	if val, exist := perfMetricMap["net.packetsTx.summation"]; exist {