From e1bbe05be24b8a158ff722fb3f97caaf887d4c00 Mon Sep 17 00:00:00 2001 From: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:23:00 +0530 Subject: [PATCH] [vSphere][virtualmachine] Add metrics to virtualmachine metricset (#40485) * Add metrics to virtualmachine * Lint changes * Address lint errors * Update data file * Address review comments * Resolve lint issues * Address lint error * Update the field definitions * Update network code and address review comment * mage update * mage update * Update tests * Update metricbeat/module/vsphere/virtualmachine/virtualmachine.go Co-authored-by: niraj-elastic <124254029+niraj-elastic@users.noreply.github.com> * Address review comments * Change event --------- Co-authored-by: niraj-elastic <124254029+niraj-elastic@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 88 +++++++++-- metricbeat/module/vsphere/fields.go | 2 +- .../vsphere/virtualmachine/_meta/data.json | 75 ++++++---- .../vsphere/virtualmachine/_meta/fields.yml | 58 ++++++-- .../module/vsphere/virtualmachine/data.go | 76 ++++++++++ .../vsphere/virtualmachine/data_test.go | 116 +++++++++++++++ .../vsphere/virtualmachine/virtualmachine.go | 138 +++++++----------- .../virtualmachine/virtualmachine_test.go | 5 + 9 files changed, 421 insertions(+), 138 deletions(-) create mode 100644 metricbeat/module/vsphere/virtualmachine/data.go create mode 100644 metricbeat/module/vsphere/virtualmachine/data_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5e4079eaffb1..a1cd76900c9a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Allow metricsets to report their status via control v2 protocol. {pull}40025[40025] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation - Add support for Kibana status metricset in v8 format {pull}40275[40275] +- Add metrics for the vSphere Virtualmachine metricset. {pull}40485[40485] - Add new metrics for the vSphere Datastore metricset. {pull}40441[40441] - Update metrics for the vSphere Host metricset. {pull}40429[40429] - Mark system process metricsets as running if metrics are partially available {pull}40565[40565] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 57357afaadc9..b870cef6c5aa 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -67430,7 +67430,7 @@ virtualmachine *`vsphere.virtualmachine.host.id`*:: + -- -Host id +Host id. type: keyword @@ -67440,7 +67440,7 @@ type: keyword *`vsphere.virtualmachine.host.hostname`*:: + -- -Host name of the host +Hostname of the host. type: keyword @@ -67450,7 +67450,7 @@ type: keyword *`vsphere.virtualmachine.name`*:: + -- -Virtual Machine name +Virtual Machine name. type: keyword @@ -67460,7 +67460,7 @@ type: keyword *`vsphere.virtualmachine.os`*:: + -- -Virtual Machine Operating System name +Virtual Machine Operating System name. type: keyword @@ -67470,7 +67470,7 @@ type: keyword *`vsphere.virtualmachine.cpu.used.mhz`*:: + -- -Used CPU in Mhz +Used CPU in Mhz. type: long @@ -67480,7 +67480,7 @@ type: long *`vsphere.virtualmachine.cpu.total.mhz`*:: + -- -Total CPU in Mhz +Total CPU in Mhz. type: long @@ -67490,7 +67490,7 @@ type: long *`vsphere.virtualmachine.cpu.free.mhz`*:: + -- -Available CPU in Mhz +Available CPU in Mhz. type: long @@ -67500,7 +67500,7 @@ type: long *`vsphere.virtualmachine.memory.used.guest.bytes`*:: + -- -Used Memory of Guest in bytes +Used Memory of Guest in bytes. type: long @@ -67512,7 +67512,7 @@ format: bytes *`vsphere.virtualmachine.memory.used.host.bytes`*:: + -- -Used Memory of Host in bytes +Used Memory of Host in bytes. type: long @@ -67524,7 +67524,7 @@ format: bytes *`vsphere.virtualmachine.memory.total.guest.bytes`*:: + -- -Total Memory of Guest in bytes +Total Memory of Guest in bytes. type: long @@ -67536,7 +67536,7 @@ format: bytes *`vsphere.virtualmachine.memory.free.guest.bytes`*:: + -- -Free Memory of Guest in bytes +Free Memory of Guest in bytes. type: long @@ -67548,7 +67548,7 @@ format: bytes *`vsphere.virtualmachine.custom_fields`*:: + -- -Custom fields +Custom fields. type: object @@ -67558,13 +67558,75 @@ type: object *`vsphere.virtualmachine.network_names`*:: + -- -Network names +Network names. + + +type: keyword + +-- + + +*`vsphere.virtualmachine.datastore.names`*:: ++ +-- +Names of the datastore associated to this virtualmachine. + + +type: keyword + +-- + +*`vsphere.virtualmachine.datastore.count`*:: ++ +-- +Number of datastores associated to this virtualmachine. + + +type: long + +-- + + +*`vsphere.virtualmachine.network.names`*:: ++ +-- +Names of the networks associated to this virtualmachine. + + +type: keyword + +-- + +*`vsphere.virtualmachine.network.count`*:: ++ +-- +Number of networks associated to this virtualmachine. + + +type: long + +-- + +*`vsphere.virtualmachine.status`*:: ++ +-- +Overall health and status of a virtual machine. type: keyword -- +*`vsphere.virtualmachine.uptime`*:: ++ +-- +The uptime of the VM in seconds. + + +type: long + +-- + [[exported-fields-windows]] == Windows fields diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index 20b57227030f..038eeb12d069 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmk1v2zgTx+/5FIPe695zeIAiD3ZTYNMW67TXgJbGFtd8EciRDffTL4Z6sWxLjmWT7voQBJYy/x+HM8MhmY+wxt0jbHxZoMMHAJKk8BE+bObhmw8PADn6zMmSpDWP8L8HAIDmKWibV4r/zKFC4fERVuIBYClR5f4xvPoRjNDYl+AP7Up+2dmqbL4ZUDk01DeWCxKebGdu2OSo2ebRgJHDcbSfY4w+SiZKkUnazZYOcbbYEfqD91owZc3q6MEZNv784RAhGAS7BCpwTzw7enlpnRb0CKfyJ5hkSaionK9sMT5o5TGPyvnDY54Is8xoENJnQmH+tlRWHL/wDux3dBkaEitk2A6004WgO4Jd1n88CL70jDZIu8bd1rp8YpRKhX7nCXWwMhsULaynWWYrM+ym6XP5tdILdOwaNu3PqPKvwxF01XD/kp5YVigVQog1gtwIg7RlrPidk3UcD18+fYMna8hZBWK1crgShDl8MWVFn75VVFYE30p0gg15jiSYY2ZNPkzIP+P55/9dpLLdYUWHIm5m/y0opAkblmYV0gWWzupbszyQKkFosrZy6ljQnzcYplNozotQkqRGhgIRhMG2kzgwFpAGtFRK+jC1I9HnSVAVMfrnwd475bNb8XWilP/5MjLcjU6d7j9fziX71kmK2wO0kc2Wu8gme2tc16C/I7CD8pWRfdIGcvG9pQN85uItTe2vjqZtbtFspLNGo6HZbS1iWdVtgi5+xexlnr7/YGe9PP8aDkfWbeY2mnDd7F2iHPrheMKhG35Pd58SietAfrDMjZSDPU2aOtjZ92BN142MsEi/nvWaVrGKW6hei37C+6ZV6brVrHIODakdLJDrWGaNr3Tox8G6Fh8NMd3kchYGl+NGZtjWNFEXoeRVTRKQWKPnopxZXSokBGFg/jT/wl9oYfK6rJTFzstMKKhBL1vAw8gS1elnuSrQEzTmYSNUhSAyZ70Pgc7iPmw0eJq6bvdy8OiNXjsJpsuBej8X2qX9UiL9GlBkBfiBvvfigIq/mo/hsxKh6Rb2OPzxd/rzSoe6w3zB5Rzc9YruBA+kB81hErSn02vU1qU6UHkJxm+nS3WOEosvzfHJzXRxd5vP7e57ZGuLtLVu/Ra5Gfhamz239DfKqdsQczlJmhaksX5BA9JyLITJtzKnYkZOGK+58qVZJLgkgSDYFjIr6p3TVnjoyUJeOe5GmFwaQrcRagavheQVpXTo0ZAPTzvqdu/bDmd6Apz4wWGGcnNfJ7Sa/w0PJFup2vzozzmvWd3w62WrLWeD3rh6fKXI1kj+IM7TZGGj1B/m+SRs0brQS8vVyVwEhc5Zd0+3bSUVUKtOd2FDeydP9lGv82pItZiQddtizqFeRqgrRTITnu4w9Z3W9bmzx00896es02a+59fkk3/Cehli7mxZYn6Hie+vBK07W/VJqIknvVuhboK8X7afxYt9DfFaIFhucJSCAoWiopEIvXF3SsHL+DuHuXvGqiQ5sjW57jQs+L4xe8jVHJyECjkV83dfcSS71NlIR5VQoEVWSDN2pNlh1G83Lz8cg0y4Axi1NP2cP6DKY/ffur09Mnggxj8S7ajbNv/ojiXVTv5nM/0v9TycWm9lbcTQPxZtbs3NCub1fzSMUqS+0jmyer8bnTPCkS90Pm+EVGKh8D3t/oHWqkJPyY617BL+ZIFuQzh579dnDRmaELW5vbyNtI6m+G49OMyM5tcQgvFh+wfDt7NmlSer3+qVYpDQLv7Bk/8Yq798u6GcPQVhGBC+54How78BAAD//4VlTHc=" + return "eJzUmktv27gTwO/5FIPe695z+ANF/thNgU1TrNNeA0YaW1zzIZAjG+6nXwz1sCxLjh+k2/WhKGxl5sfhPEl9hBVu72HtywId3gGQJIX38GE9D998uAPI0WdOliStuYf/3QEANL+Ctnml+M8cKhQe72Ep7gAWElXu78OjH8EIjX0V/KFtyQ87W5XNNyNa9gX1heWChCfbiRsXOSm2+WlEyP462s8Qo4+SiVJkkrazhUOcvW0J/d5zLZiyZjn44Qgbf/5wiBAEgl0AFbgjng0eXlinBd3DofoDTLIkVFTOF5YYH7TymEfl/O4xT4RZZjQK6TOhMH9dKCuGD7wD+w1dhobEEhm2A+30QtA7gV3WfzwKvvCMNkq7wu3GuvxML5UK/dYT6iBlNqq0sJ5mma3MuJnO38uvlX5Dx6Zh0f6IVv7vuAddtNy/pCdWK5QKLsQ6groJBmnLWP47J+vYH758eoYHa8hZBWK5dLgUhDl8MWVFn54rKiuC5xKdYEGePQnmmFmTjxPyv/Hs8//OU1nuuEaHIm5k/y0ohAkLlmYZwgUWzuprozyQKkFosjZz6ljQn9cYtlNojouQkqRGhgIRFINtN3FkLSANaKmU9GFrJ7zPk6AqovfPg7x30mdX8XWikP/xNLHctU4d7j+ejgX7xkmK2wO0ns2SO88me61f16C/wrGD5gs9+6AN5OR7TQf4yMlbmtpeHU3b3KJZS2eNRkOz61rEsqrbBF38jNnLPHz7zsZ6evw57o6st9nbaIrrZu8UzaEfjqc4dMPv6d2FROI8kO+VuYl0sKNJkwc7+R6s6bqRCRbpV7Ne0yqWcRPVS9EPeN+0Kl23mlXOoSG1hTfkPJZZ4ysd+nGwrsVHQ0x3djoLi8txLTNsc5qok1DyrCYJSKzQc1LOrC4VEoIwMH+Yf+EvtDB5nVbKYutlJhTUoKcV8LCyRHn6US4L9ASNeFgLVSGIzFnvg6Ozch8GDd6mrts9HTx6o9dugulioJ7nQru0KyXSrwBFVoAf6XtPdqj41XwKnzURmq6wx+GPP+nPKx3yDvMFk7Nz1xXdCV5ID5rdJOg+n16jti7VgcpTEH49XapzlFh8aY5PrqaLO20+ttP3xGiLtLFu9Rq5Gfhaiz1W+hvNqdsQczpJmhakkX5CA9JyvAmTb2ROxYycMF5z5ktTJDglgSDYFDIr6slpIzz01EJeOe5GmFwaQrcWagYvheSKUjr0aMiHXzvqdvZtl3N+ABzYwWGGcn1bI7Q6fw8LJKtUbXz095xrVrf8umy16WzUGhevrxTZCsnv+XmaKGw09Zd5PAhbtM710nJ1ak6CQuesu6XZNpIKqLWeb8KG9kaW7KNeZtUQajEh67bFHEM9jVBXimQmPN1g6ztdl8fODjfx3h+ynrfzPbsm3/wD1tMQc2fLEvMbbHy/ErTmbLWfhZp407sKdRXk7aL9KF7sa4iXAsFyg6MUFCgUFY2K0Bt3pxRcxt85zN0xViXJidHkstOwYPtG7D5Xc3ASMuS5mL/6iiPZpc5aOqqEAi2yQpqpI80Oo366efhuCHLGHcCkpPPP+QOqHJr/2vFWTgRV0Mb/xB+pWWLb5h+Z6KLq/dHs/1O9EUemehvR+Ydam3tzs4R5/U7DNEbqW53il93qHNMc+Vbn81pIJd4Uvqu8f6y1rNBTssMtu4A/WUGkM7gQQAlZm0vMKMeZ8Q27d6gZz7LBDePT9k+II8BmlSerX+uaMYpo3/7Bg3fH6i9fr0hrD0FxU6x+t7PRsfcuYbJ0w0TRhUElGq7hvXWcsJawHhZ98MILCO9tJsN7V+HSRvpBIzF0lp5TjLRPcMxnTyUduxc+k3PgHP+5DeqOo3/r/bmUMtU49bw/SgmT741Tg878RuPTbnBqhpLd4DTyRtDdvwEAAP//8HhtJQ==" } diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/data.json b/metricbeat/module/vsphere/virtualmachine/_meta/data.json index 2dea5c913633..d04a8efb42db 100644 --- a/metricbeat/module/vsphere/virtualmachine/_meta/data.json +++ b/metricbeat/module/vsphere/virtualmachine/_meta/data.json @@ -1,5 +1,5 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", + "@timestamp": "2024-08-12T04:51:25.851Z", "event": { "dataset": "vsphere.virtualmachine", "duration": 115000, @@ -15,35 +15,58 @@ }, "vsphere": { "virtualmachine": { - "cpu": { - "used": { - "mhz": 0 - } + "network.names": [ + "PROD_VCF" + ], + "os": "Microsoft Windows 10 (64-bit)", + "datastore": { + "count": 1 }, - "host.hostname": "localhost.localdomain", - "host.id": "ha-host", "memory": { - "free": { - "guest": { - "bytes": 33554432 - } + "used": { + "guest": { + "bytes": 0 }, - "total": { - "guest": { - "bytes": 33554432 - } - }, - "used": { - "guest": { - "bytes": 0 - }, - "host": { - "bytes": 0 - } + "host": { + "bytes": 0 + } + }, + "total": { + "guest": { + "bytes": 4294967296 } + }, + "free": { + "guest": { + "bytes": 4294967296 + } + } + }, + "network": { + "count": 1 + }, + "host.hostname": "phx-w1c1", + "name": "phx-14", + "cpu": { + "free": { + "mhz": 0 + }, + "used": { + "mhz": 0 + }, + "total": { + "mhz": 0 + } }, - "name": "ha-host_VM0", - "os": "otherGuest" + "uptime": 0, + "status": "green", + "network_names": [ + "PROD_VCF" + ], + "datastore.names": [ + "VxRailtec-Virtual-SAN-Datastore-247df-bc1d-5aad2" + ], + "host.id": "host-20" + } } } -} \ No newline at end of file diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml b/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml index 723d611b2e1a..6720da187652 100644 --- a/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml +++ b/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml @@ -7,57 +7,89 @@ - name: host.id type: keyword description: > - Host id + Host id. - name: host.hostname type: keyword description: > - Host name of the host + Hostname of the host. - name: name type: keyword description: > - Virtual Machine name + Virtual Machine name. - name: os type: keyword description: > - Virtual Machine Operating System name + Virtual Machine Operating System name. - name: cpu.used.mhz type: long description: > - Used CPU in Mhz + Used CPU in Mhz. - name: cpu.total.mhz type: long description: > - Total CPU in Mhz + Total CPU in Mhz. - name: cpu.free.mhz type: long description: > - Available CPU in Mhz + Available CPU in Mhz. - name: memory.used.guest.bytes type: long description: > - Used Memory of Guest in bytes + Used Memory of Guest in bytes. format: bytes - name: memory.used.host.bytes type: long description: > - Used Memory of Host in bytes + Used Memory of Host in bytes. format: bytes - name: memory.total.guest.bytes type: long description: > - Total Memory of Guest in bytes + Total Memory of Guest in bytes. format: bytes - name: memory.free.guest.bytes type: long description: > - Free Memory of Guest in bytes + Free Memory of Guest in bytes. format: bytes - name: custom_fields type: object object_type: keyword description: > - Custom fields + Custom fields. - name: network_names type: keyword description: > - Network names + Network names. + - name: datastore + type: group + fields: + - name: names + type: keyword + description: > + Names of the datastore associated to this virtualmachine. + - name: count + type: long + description: > + Number of datastores associated to this virtualmachine. + - name: network + type: group + fields: + - name: names + type: keyword + description: > + Names of the networks associated to this virtualmachine. + - name: count + type: long + description: > + Number of networks associated to this virtualmachine. + - name: status + type: keyword + description: > + Overall health and status of a virtual machine. + - name: uptime + type: long + description: > + The uptime of the VM in seconds. + + diff --git a/metricbeat/module/vsphere/virtualmachine/data.go b/metricbeat/module/vsphere/virtualmachine/data.go new file mode 100644 index 000000000000..5c814105f90b --- /dev/null +++ b/metricbeat/module/vsphere/virtualmachine/data.go @@ -0,0 +1,76 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package virtualmachine + +import ( + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func (m *MetricSet) mapEvent(data VMData) mapstr.M { + const bytesMultiplier = int64(1024 * 1024) + usedMemory := int64(data.VM.Summary.QuickStats.GuestMemoryUsage) * bytesMultiplier + usedCPU := data.VM.Summary.QuickStats.OverallCpuUsage + totalCPU := data.VM.Summary.Config.CpuReservation + totalMemory := int64(data.VM.Summary.Config.MemorySizeMB) * bytesMultiplier + + freeCPU := max(0, totalCPU-usedCPU) + freeMemory := max(0, totalMemory-usedMemory) + + event := mapstr.M{ + "name": data.VM.Summary.Config.Name, + "os": data.VM.Summary.Config.GuestFullName, + "uptime": data.VM.Summary.QuickStats.UptimeSeconds, + "status": data.VM.Summary.OverallStatus, + "host.id": data.HostID, + "host.hostname": data.HostName, + "cpu": mapstr.M{ + "used": mapstr.M{"mhz": usedCPU}, + "total": mapstr.M{"mhz": totalCPU}, + "free": mapstr.M{"mhz": freeCPU}, + }, + "memory": mapstr.M{ + "used": mapstr.M{ + "guest": mapstr.M{"bytes": usedMemory}, + "host": mapstr.M{"bytes": int64(data.VM.Summary.QuickStats.HostMemoryUsage) * bytesMultiplier}, + }, + "total": mapstr.M{ + "guest": mapstr.M{"bytes": totalMemory}, + }, + "free": mapstr.M{ + "guest": mapstr.M{"bytes": freeMemory}, + }, + }, + "network": mapstr.M{ + "count": len(data.NetworkNames), + }, + "datastore": mapstr.M{ + "count": len(data.DatastoreNames), + }, + } + if len(data.CustomFields) > 0 { + event["custom_fields"] = data.CustomFields + } + if len(data.NetworkNames) > 0 { + event["network_names"] = data.NetworkNames + event["network.names"] = data.NetworkNames + } + if len(data.DatastoreNames) > 0 { + event["datastore.names"] = data.DatastoreNames + } + return event +} diff --git a/metricbeat/module/vsphere/virtualmachine/data_test.go b/metricbeat/module/vsphere/virtualmachine/data_test.go new file mode 100644 index 000000000000..351daea7c118 --- /dev/null +++ b/metricbeat/module/vsphere/virtualmachine/data_test.go @@ -0,0 +1,116 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package virtualmachine + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestEventMapping(t *testing.T) { + var m MetricSet + + VirtualMachineTest := mo.VirtualMachine{ + Summary: types.VirtualMachineSummary{ + OverallStatus: types.ManagedEntityStatus("green"), + Config: types.VirtualMachineConfigSummary{ + Name: "localhost.localdomain", + GuestFullName: "otherGuest", + MemorySizeMB: 70, + CpuReservation: 2294, // MHz + }, + QuickStats: types.VirtualMachineQuickStats{ + UptimeSeconds: 10, + OverallCpuUsage: 30, // MHz + GuestMemoryUsage: 40, // MB + HostMemoryUsage: 50, // MB + }, + }, + } + + data := VMData{ + VM: VirtualMachineTest, + HostID: "host-1234", + HostName: "test-host", + NetworkNames: []string{"network-1", "network-2"}, + DatastoreNames: []string{"ds1", "ds2"}, + CustomFields: mapstr.M{ + "customField1": "value1", + "customField2": "value2", + }, + } + + event := m.mapEvent(data) + + // Expected event structure + expectedEvent := mapstr.M{ + "name": "localhost.localdomain", + "os": "otherGuest", + "uptime": int32(10), + "status": types.ManagedEntityStatus("green"), + "host.id": "host-1234", + "host.hostname": "test-host", + "cpu": mapstr.M{ + "used": mapstr.M{"mhz": int32(30)}, + "total": mapstr.M{"mhz": int32(2294)}, + "free": mapstr.M{"mhz": int32(2264)}, + }, + "memory": mapstr.M{ + "used": mapstr.M{ + "guest": mapstr.M{ + "bytes": int64(40 * 1024 * 1024), + }, + "host": mapstr.M{ + "bytes": int64(50 * 1024 * 1024), + }, + }, + "total": mapstr.M{ + "guest": mapstr.M{ + "bytes": int64(70 * 1024 * 1024), + }, + }, + "free": mapstr.M{ + "guest": mapstr.M{ + "bytes": int64(30 * 1024 * 1024), + }, + }, + }, + "network": mapstr.M{ + "count": 2, + }, + "datastore": mapstr.M{ + "count": 2, + }, + "custom_fields": mapstr.M{ + "customField1": "value1", + "customField2": "value2", + }, + "network.names": []string{"network-1", "network-2"}, + "network_names": []string{"network-1", "network-2"}, + "datastore.names": []string{"ds1", "ds2"}, + } + + // Assert that the output event matches the expected event + assert.Exactly(t, expectedEvent, event) + +} diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go index fd1a20ee3a72..107aabbc4bd3 100644 --- a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go +++ b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go @@ -49,6 +49,15 @@ type MetricSet struct { GetCustomFields bool } +type VMData struct { + VM mo.VirtualMachine + HostID string + HostName string + NetworkNames []string + DatastoreNames []string + CustomFields mapstr.M +} + // New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms, err := vsphere.NewMetricSet(base) @@ -80,12 +89,12 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { client, err := govmomi.NewClient(ctx, m.HostURL, m.Insecure) if err != nil { - return fmt.Errorf("error in NewClient: %w", err) + return fmt.Errorf("virtualmachine: error in NewClient: %w", err) } defer func() { if err := client.Logout(ctx); err != nil { - m.Logger().Debug(fmt.Errorf("error trying to logout from vshphere: %w", err)) + m.Logger().Debugf("Error logging out from vsphere: %v", err) } }() @@ -97,7 +106,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { var err error customFieldsMap, err = setCustomFieldsMap(ctx, c) if err != nil { - return fmt.Errorf("error in setCustomFieldsMap: %w", err) + return fmt.Errorf("virtualmachine: error in setCustomFieldsMap: %w", err) } } @@ -106,72 +115,33 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { v, err := mgr.CreateContainerView(ctx, c.ServiceContent.RootFolder, []string{"VirtualMachine"}, true) if err != nil { - return fmt.Errorf("error in CreateContainerView: %w", err) + return fmt.Errorf("virtualmachine: error in CreateContainerView: %w", err) } defer func() { if err := v.Destroy(ctx); err != nil { - m.Logger().Debug(fmt.Errorf("error trying to destroy view from vshphere: %w", err)) + m.Logger().Debug("Error destroying view from vsphere %w", err) } }() // Retrieve summary property for all machines var vmt []mo.VirtualMachine - err = v.Retrieve(ctx, []string{"VirtualMachine"}, []string{"summary"}, &vmt) + err = v.Retrieve(ctx, []string{"VirtualMachine"}, []string{"summary", "datastore"}, &vmt) if err != nil { - return fmt.Errorf("error in Retrieve: %w", err) + return fmt.Errorf("virtualmachine: error in Retrieve: %w", err) } + pc := property.DefaultCollector(c) for _, vm := range vmt { - usedMemory := int64(vm.Summary.QuickStats.GuestMemoryUsage) * 1024 * 1024 - usedCPU := vm.Summary.QuickStats.OverallCpuUsage - event := mapstr.M{ - "name": vm.Summary.Config.Name, - "os": vm.Summary.Config.GuestFullName, - "cpu": mapstr.M{ - "used": mapstr.M{ - "mhz": usedCPU, - }, - }, - "memory": mapstr.M{ - "used": mapstr.M{ - "guest": mapstr.M{ - "bytes": usedMemory, - }, - "host": mapstr.M{ - "bytes": int64(vm.Summary.QuickStats.HostMemoryUsage) * 1024 * 1024, - }, - }, - }, - } - - totalCPU := vm.Summary.Config.CpuReservation - if totalCPU > 0 { - freeCPU := totalCPU - usedCPU - // Avoid negative values if reported used CPU is slightly over total configured. - if freeCPU < 0 { - freeCPU = 0 - } - event.Put("cpu.total.mhz", totalCPU) - event.Put("cpu.free.mhz", freeCPU) - } - - totalMemory := int64(vm.Summary.Config.MemorySizeMB) * 1024 * 1024 - if totalMemory > 0 { - freeMemory := totalMemory - usedMemory - // Avoid negative values if reported used memory is slightly over total configured. - if freeMemory < 0 { - freeMemory = 0 - } - event.Put("memory.total.guest.bytes", totalMemory) - event.Put("memory.free.guest.bytes", freeMemory) - } + var hostID, hostName string + var networkNames, datastoreNames []string + var customFields mapstr.M if host := vm.Summary.Runtime.Host; host != nil { - event["host.id"] = host.Value + hostID = host.Value hostSystem, err := getHostSystem(ctx, c, host.Reference()) if err == nil { - event["host.hostname"] = hostSystem.Summary.Config.Name + hostName = hostSystem.Summary.Config.Name } else { m.Logger().Debug(err.Error()) } @@ -181,32 +151,45 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { "from host/guest") } - // Get custom fields (attributes) values if get_custom_fields is true. + // Retrieve custom fields if enabled if m.GetCustomFields && vm.Summary.CustomValue != nil { - customFields := getCustomFields(vm.Summary.CustomValue, customFieldsMap) - - if len(customFields) > 0 { - event["custom_fields"] = customFields - } - } else { + customFields = getCustomFields(vm.Summary.CustomValue, customFieldsMap) + } + if len(customFields) <= 0 { m.Logger().Debug("custom fields not activated or custom values not found/parse in Summary data. This " + "is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete " + "information returned from host/guest") } - + // Retrieve network names if vm.Summary.Vm != nil { - networkNames, err := getNetworkNames(ctx, c, vm.Summary.Vm.Reference()) + networkNames, err = getNetworkNames(ctx, c, vm.Summary.Vm.Reference()) if err != nil { m.Logger().Debug(err.Error()) + } + } + + // Retrieve the datastore names associated with the Virtualmachine + for _, datastoreRef := range vm.Datastore { + var ds mo.Datastore + err = pc.RetrieveOne(ctx, datastoreRef, []string{"name"}, &ds) + if err == nil { + datastoreNames = append(datastoreNames, ds.Name) } else { - if len(networkNames) > 0 { - event["network_names"] = networkNames - } + m.Logger().Debug("error retrieving datastore name for VM %s: %v", vm.Summary.Config.Name, err) } } + data := VMData{ + VM: vm, + HostID: hostID, + HostName: hostName, + NetworkNames: networkNames, + DatastoreNames: datastoreNames, + CustomFields: customFields, + } + reporter.Event(mb.Event{ - MetricSetFields: event, + MetricSetFields: m.mapEvent(data), }) } @@ -232,38 +215,23 @@ func getNetworkNames(ctx context.Context, c *vim25.Client, ref types.ManagedObje ctx, cancel := context.WithCancel(ctx) defer cancel() - var outputNetworkNames []string - pc := property.DefaultCollector(c) var vm mo.VirtualMachine err := pc.RetrieveOne(ctx, ref, []string{"network"}, &vm) if err != nil { - return nil, fmt.Errorf("error retrieving virtual machine information: %v", err) + return nil, fmt.Errorf("error retrieving virtual machine information: %w", err) } if len(vm.Network) == 0 { return nil, errors.New("no networks found") } - var networkRefs []types.ManagedObjectReference - for _, obj := range vm.Network { - if obj.Type == "Network" { - networkRefs = append(networkRefs, obj) - } - } - - // If only "Distributed port group" was found, for example. - if len(networkRefs) == 0 { - return nil, errors.New("no networks found") - } - var nets []mo.Network - err = pc.Retrieve(ctx, networkRefs, []string{"name"}, &nets) - if err != nil { - return nil, fmt.Errorf("error retrieving network from virtual machine: %v", err) + if err := pc.Retrieve(ctx, vm.Network, []string{"name"}, &nets); err != nil { + return nil, fmt.Errorf("error retrieving network from virtual machine: %w", err) } - + outputNetworkNames := make([]string, 0, len(nets)) for _, net := range nets { name := strings.Replace(net.Name, ".", "_", -1) outputNetworkNames = append(outputNetworkNames, name) @@ -298,7 +266,7 @@ func getHostSystem(ctx context.Context, c *vim25.Client, ref types.ManagedObject var hs mo.HostSystem err := pc.RetrieveOne(ctx, ref, []string{"summary"}, &hs) if err != nil { - return nil, fmt.Errorf("error retrieving host information: %v", err) + return nil, fmt.Errorf("error retrieving host information: %w", err) } return &hs, nil } diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine_test.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine_test.go index e43345e5e532..6566aa72e569 100644 --- a/metricbeat/module/vsphere/virtualmachine/virtualmachine_test.go +++ b/metricbeat/module/vsphere/virtualmachine/virtualmachine_test.go @@ -50,7 +50,12 @@ func TestFetchEventContents(t *testing.T) { assert.EqualValues(t, "ha-host", event["host.id"]) assert.EqualValues(t, "localhost.localdomain", event["host.hostname"]) + assert.EqualValues(t, "green", event["status"]) + assert.EqualValues(t, 0, event["uptime"]) assert.True(t, strings.Contains(event["name"].(string), "ha-host_VM")) + expectedDatastoreNames := []string{"LocalDS_0"} + actualDatastoreNames := event["datastore.names"].([]string) + assert.EqualValues(t, expectedDatastoreNames, actualDatastoreNames) cpu := event["cpu"].(mapstr.M)