-
Notifications
You must be signed in to change notification settings - Fork 452
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[vSphere][datastorecluster] Add new Datastream Datastore Cluster (#11089
) * initial commit for datastore cluster datastream * resolve review comments * update changelog entry * update README.md * udpate README * update changelog entry * reorder changelog * fix changelog * udpate docker-compose * fix conflicts * fix description of period * update and fix system test for datastorecluster * update processor URL * remove agent fields
- Loading branch information
1 parent
3189c70
commit 8966518
Showing
16 changed files
with
472 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,4 +6,4 @@ RUN apk add --no-cache curl git | |
| RUN go install github.com/vmware/govmomi/[email protected] | ||
|
|
||
| HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8989/ | ||
| CMD vcsim -l :8989 | ||
| CMD vcsim -pod 1 -l :8989 | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
...sphere/data_stream/datastorecluster/_dev/test/pipeline/test-datastorecluster-metrics.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| { | ||
| "events": [ | ||
| { | ||
| "vsphere": { | ||
| "datastorecluster": { | ||
| "name": "datastore_cluster1", | ||
| "capacity": { | ||
| "bytes": 8795019280384 | ||
| }, | ||
| "free_space": { | ||
| "bytes": 8788836876288 | ||
| }, | ||
| "datastore": { | ||
| "count": 1, | ||
| "names": [ | ||
| "LocalDS_0" | ||
| ] | ||
| } | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } |
26 changes: 26 additions & 0 deletions
26
...ream/datastorecluster/_dev/test/pipeline/test-datastorecluster-metrics.json-expected.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| { | ||
| "expected": [ | ||
| { | ||
| "ecs": { | ||
| "version": "8.11.0" | ||
| }, | ||
| "vsphere": { | ||
| "datastorecluster": { | ||
| "capacity": { | ||
| "bytes": 8795019280384 | ||
| }, | ||
| "datastore": { | ||
| "count": 1, | ||
| "names": [ | ||
| "LocalDS_0" | ||
| ] | ||
| }, | ||
| "free_space": { | ||
| "bytes": 8788836876288 | ||
| }, | ||
| "name": "datastore_cluster1" | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } |
9 changes: 9 additions & 0 deletions
9
packages/vsphere/data_stream/datastorecluster/_dev/test/system/test-default-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| service: vsphere-metrics | ||
| vars: | ||
| hosts: | ||
| - "{{Hostname}}:8989/sdk" | ||
| period: 10s | ||
| username: "user" | ||
| password: "password" | ||
| # If insecure is true, don't verify the server's certificate chain | ||
| insecure: true |
20 changes: 20 additions & 0 deletions
20
packages/vsphere/data_stream/datastorecluster/agent/stream/stream.yml.hbs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| metricsets: ["datastorecluster"] | ||
| hosts: | ||
| {{#each hosts}} | ||
| - {{this}} | ||
| {{/each}} | ||
| period: {{period}} | ||
| username: {{username}} | ||
| password: {{password}} | ||
| # If insecure is true, don't verify the server's certificate chain | ||
| insecure: {{insecure}} | ||
|
|
||
| tags: | ||
| {{#each tags as |tag|}} | ||
| - {{tag}} | ||
| {{/each}} | ||
|
|
||
| {{#if processors}} | ||
| processors: | ||
| {{processors}} | ||
| {{/if}} |
10 changes: 10 additions & 0 deletions
10
packages/vsphere/data_stream/datastorecluster/elasticsearch/ingest_pipeline/default.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| --- | ||
| description: Pipeline for datastorecluster | ||
| processors: | ||
| - set: | ||
| field: ecs.version | ||
| value: '8.11.0' | ||
| on_failure: | ||
| - set: | ||
| field: error.message | ||
| value: '{{ _ingest.on_failure_message }}' |
20 changes: 20 additions & 0 deletions
20
packages/vsphere/data_stream/datastorecluster/fields/agent.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| - name: host | ||
| title: Host | ||
| group: 2 | ||
| description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' | ||
| type: group | ||
| fields: | ||
| - name: name | ||
| level: core | ||
| type: keyword | ||
| dimension: true | ||
| ignore_above: 1024 | ||
| description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' | ||
| - name: agent | ||
| title: Agent | ||
| type: group | ||
| fields: | ||
| - name: id | ||
| type: keyword | ||
| ignore_above: 1024 | ||
| dimension: true |
20 changes: 20 additions & 0 deletions
20
packages/vsphere/data_stream/datastorecluster/fields/base-fields.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| - name: data_stream.type | ||
| type: constant_keyword | ||
| description: Data stream type. | ||
| - name: data_stream.dataset | ||
| type: constant_keyword | ||
| description: Data stream dataset. | ||
| - name: data_stream.namespace | ||
| type: constant_keyword | ||
| description: Data stream namespace. | ||
| - name: '@timestamp' | ||
| type: date | ||
| description: Event timestamp. | ||
| - name: event.module | ||
| type: constant_keyword | ||
| description: Event module | ||
| value: vsphere | ||
| - name: event.dataset | ||
| type: constant_keyword | ||
| description: Event dataset | ||
| value: vsphere.datastorecluster |
21 changes: 21 additions & 0 deletions
21
packages/vsphere/data_stream/datastorecluster/fields/ecs.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| - name: service.address | ||
| external: ecs | ||
| dimension: true | ||
| - external: ecs | ||
| name: cloud.instance.id | ||
| dimension: true | ||
| - external: ecs | ||
| name: cloud.provider | ||
| dimension: true | ||
| - external: ecs | ||
| name: container.id | ||
| dimension: true | ||
| - external: ecs | ||
| name: cloud.account.id | ||
| dimension: true | ||
| - external: ecs | ||
| name: cloud.region | ||
| dimension: true | ||
| - external: ecs | ||
| name: cloud.availability_zone | ||
| dimension: true |
57 changes: 57 additions & 0 deletions
57
packages/vsphere/data_stream/datastorecluster/fields/fields.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| - name: vsphere.datastorecluster | ||
| type: group | ||
| description: > | ||
| Datastore Cluster | ||
| fields: | ||
| - name: capacity.bytes | ||
| type: long | ||
| metric_type: gauge | ||
| unit: byte | ||
| description: > | ||
| Total capacity of this storage pod, in bytes. | ||
| format: bytes | ||
| - name: datastore | ||
| type: group | ||
| fields: | ||
| - name: names | ||
| type: keyword | ||
| description: List of all the Datastore names associated with the Datastore Cluster. | ||
| - name: count | ||
| type: long | ||
| metric_type: gauge | ||
| description: Number of datastores in the Datastore Cluster. | ||
| - name: free_space.bytes | ||
| type: long | ||
| metric_type: gauge | ||
| unit: byte | ||
| description: > | ||
| Total free space on this storage pod, in bytes. | ||
| format: bytes | ||
| - name: name | ||
| type: keyword | ||
| # Reason to add as a dimension field: unique identifier of the host | ||
| dimension: true | ||
| description: The Datastore Cluster name. | ||
| - name: triggered_alarms | ||
| type: group | ||
| description: > | ||
| List of all the triggered alarms. | ||
| fields: | ||
| - name: status | ||
| type: keyword | ||
| description: Status of the alarm. | ||
| - name: triggered_time | ||
| type: date | ||
| description: Time when the alarm was triggered. | ||
| - name: description | ||
| type: keyword | ||
| description: Description of the alarm. | ||
| - name: entity_name | ||
| type: keyword | ||
| description: Name of the entity associated with the alarm. | ||
| - name: name | ||
| type: keyword | ||
| description: Name of the alarm. | ||
| - name: id | ||
| type: keyword | ||
| description: Unique identifier for the alarm. |
29 changes: 29 additions & 0 deletions
29
packages/vsphere/data_stream/datastorecluster/manifest.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| title: "vSphere DatastoreCluster metrics" | ||
| type: metrics | ||
| streams: | ||
| - input: vsphere/metrics | ||
| title: vSphere Datastore Cluster metrics | ||
| description: vSphere Datastore Cluster metrics | ||
| vars: | ||
| - name: period | ||
| type: text | ||
| title: Period | ||
| description: The period for fetching metrics, e.g., every 1s, 1m, 1h. | ||
| default: 20s | ||
| - name: tags | ||
| type: text | ||
| title: Tags | ||
| multi: true | ||
| required: true | ||
| show_user: false | ||
| default: | ||
| - vsphere-datastorecluster | ||
| - name: processors | ||
| type: yaml | ||
| title: Processors | ||
| multi: false | ||
| required: false | ||
| show_user: false | ||
| description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. | ||
| elasticsearch: | ||
| index_mode: "time_series" |
79 changes: 79 additions & 0 deletions
79
packages/vsphere/data_stream/datastorecluster/sample_event.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| { | ||
| "@timestamp": "2024-09-22T05:28:46.315Z", | ||
| "agent": { | ||
| "ephemeral_id": "8b7e9ea4-0517-4e98-a795-b6fe529f4a2f", | ||
| "id": "7737279e-51e9-4d90-a0d0-2c12dc4446bf", | ||
| "name": "elastic-agent-23128", | ||
| "type": "metricbeat", | ||
| "version": "8.15.2" | ||
| }, | ||
| "data_stream": { | ||
| "dataset": "vsphere.datastorecluster", | ||
| "namespace": "65218", | ||
| "type": "metrics" | ||
| }, | ||
| "ecs": { | ||
| "version": "8.11.0" | ||
| }, | ||
| "elastic_agent": { | ||
| "id": "7737279e-51e9-4d90-a0d0-2c12dc4446bf", | ||
| "snapshot": true, | ||
| "version": "8.15.2" | ||
| }, | ||
| "event": { | ||
| "agent_id_status": "verified", | ||
| "dataset": "vsphere.datastorecluster", | ||
| "duration": 10772332, | ||
| "ingested": "2024-09-22T05:28:49Z", | ||
| "module": "vsphere" | ||
| }, | ||
| "host": { | ||
| "architecture": "x86_64", | ||
| "containerized": true, | ||
| "hostname": "elastic-agent-23128", | ||
| "id": "57723763cd1b4ff48e54a505de4ebe6c", | ||
| "ip": [ | ||
| "192.168.240.2", | ||
| "192.168.255.5" | ||
| ], | ||
| "mac": [ | ||
| "02-42-C0-A8-F0-02", | ||
| "02-42-C0-A8-FF-05" | ||
| ], | ||
| "name": "elastic-agent-23128", | ||
| "os": { | ||
| "codename": "focal", | ||
| "family": "debian", | ||
| "kernel": "3.10.0-1160.118.1.el7.x86_64", | ||
| "name": "Ubuntu", | ||
| "platform": "ubuntu", | ||
| "type": "linux", | ||
| "version": "20.04.6 LTS (Focal Fossa)" | ||
| } | ||
| }, | ||
| "metricset": { | ||
| "name": "datastorecluster", | ||
| "period": 20000 | ||
| }, | ||
| "service": { | ||
| "address": "https://svc-vsphere-metrics:8989/sdk", | ||
| "type": "vsphere" | ||
| }, | ||
| "tags": [ | ||
| "vsphere-datastorecluster" | ||
| ], | ||
| "vsphere": { | ||
| "datastorecluster": { | ||
| "capacity": { | ||
| "bytes": 0 | ||
| }, | ||
| "datastore": { | ||
| "count": 0 | ||
| }, | ||
| "free_space": { | ||
| "bytes": 0 | ||
| }, | ||
| "name": "DC0_POD0" | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.