From 8860463da70f86c827808a1ae95f4c8d2ee5674f Mon Sep 17 00:00:00 2001 From: ritalwar Date: Fri, 11 Aug 2023 10:55:03 +0530 Subject: [PATCH 1/7] [Azure][Storage_account] Add dimension to the storage account datastream --- packages/azure_metrics/changelog.yml | 5 +++++ .../storage_account/fields/agent.yml | 9 +++++++++ .../storage_account/fields/package-fields.yml | 20 +++++++++++++------ packages/azure_metrics/docs/README.md | 4 +++- .../azure_metrics/docs/storage_account.md | 4 +++- packages/azure_metrics/manifest.yml | 2 +- 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/packages/azure_metrics/changelog.yml b/packages/azure_metrics/changelog.yml index 8c9732228a1..77dd48619a2 100644 --- a/packages/azure_metrics/changelog.yml +++ b/packages/azure_metrics/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.0.20" + changes: + - description: Add dimension to the storage account datastream + type: enhancement + link: tbd - version: "1.0.16" changes: - description: Added categories and/or subcategories. diff --git a/packages/azure_metrics/data_stream/storage_account/fields/agent.yml b/packages/azure_metrics/data_stream/storage_account/fields/agent.yml index da4e652c53b..bf4208ac2f7 100644 --- a/packages/azure_metrics/data_stream/storage_account/fields/agent.yml +++ b/packages/azure_metrics/data_stream/storage_account/fields/agent.yml @@ -45,6 +45,7 @@ - name: region level: extended type: keyword + dimension: true ignore_above: 1024 description: Region in which this host is running. example: us-east-1 @@ -196,3 +197,11 @@ description: > OS codename, if any. +- name: agent + title: Agent + type: group + fields: + - name: id + type: keyword + ignore_above: 1024 + dimension: true \ No newline at end of file diff --git a/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml b/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml index 28fa99283bd..62fcaaa3ee4 100644 --- a/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml +++ b/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml @@ -4,6 +4,7 @@ fields: - name: timegrain type: keyword + dimension: true description: > The Azure metric timegrain @@ -25,6 +26,7 @@ - name: id type: keyword + dimension: true description: > The id of the resource @@ -42,6 +44,7 @@ - name: namespace type: keyword + dimension: true description: > The namespace selected @@ -55,12 +58,17 @@ description: > The application ID - - name: dimensions.* - type: object - object_type: keyword - object_type_mapping_type: "*" - description: > - Azure metric dimensions. + - name: dimensions + type: group + fields: + - name: response_type + type: keyword + dimension: true + description: Response Type such as Success, ClientOtherError, etc. + - name: api_name + type: keyword + dimension: true + description: API name such as EntityGroupTransaction, GetBlobServiceProperties, etc. - name: metrics.*.* type: object diff --git a/packages/azure_metrics/docs/README.md b/packages/azure_metrics/docs/README.md index 15ef445b01f..5649e6e4726 100644 --- a/packages/azure_metrics/docs/README.md +++ b/packages/azure_metrics/docs/README.md @@ -251,8 +251,10 @@ so the `period` for `storage_account` should be `300s` or multiples of `300s`. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| agent.id | | keyword | | azure.application_id | The application ID | keyword | -| azure.dimensions.\* | Azure metric dimensions. | object | +| azure.dimensions.api_name | API name such as EntityGroupTransaction, GetBlobServiceProperties, etc. | keyword | +| azure.dimensions.response_type | Response Type such as Success, ClientOtherError, etc. | keyword | | azure.metrics.\*.\* | Metrics returned. | object | | azure.namespace | The namespace selected | keyword | | azure.resource.group | The resource group | keyword | diff --git a/packages/azure_metrics/docs/storage_account.md b/packages/azure_metrics/docs/storage_account.md index 8c80240d902..923744faca3 100644 --- a/packages/azure_metrics/docs/storage_account.md +++ b/packages/azure_metrics/docs/storage_account.md @@ -73,8 +73,10 @@ Authentication: we are handling authentication on our side (creating/renewing th | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| agent.id | | keyword | | azure.application_id | The application ID | keyword | -| azure.dimensions.\* | Azure metric dimensions. | object | +| azure.dimensions.api_name | API name such as EntityGroupTransaction, GetBlobServiceProperties, etc. | keyword | +| azure.dimensions.response_type | Response Type such as Success, ClientOtherError, etc. | keyword | | azure.metrics.\*.\* | Metrics returned. | object | | azure.namespace | The namespace selected | keyword | | azure.resource.group | The resource group | keyword | diff --git a/packages/azure_metrics/manifest.yml b/packages/azure_metrics/manifest.yml index ddcaa7eacba..d56aa33bb0f 100644 --- a/packages/azure_metrics/manifest.yml +++ b/packages/azure_metrics/manifest.yml @@ -1,6 +1,6 @@ name: azure_metrics title: Azure Resource Metrics -version: 1.0.16 +version: 1.0.20 release: ga description: Collect metrics from Azure resources with Elastic Agent. type: integration From 80f72d44c4e51ebf494815d4588ade79b2b0f325 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Fri, 11 Aug 2023 11:00:29 +0530 Subject: [PATCH 2/7] Update changelog.yml --- packages/azure_metrics/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure_metrics/changelog.yml b/packages/azure_metrics/changelog.yml index 77dd48619a2..faf92674102 100644 --- a/packages/azure_metrics/changelog.yml +++ b/packages/azure_metrics/changelog.yml @@ -2,7 +2,7 @@ changes: - description: Add dimension to the storage account datastream type: enhancement - link: tbd + link: https://github.com/elastic/integrations/pull/7356 - version: "1.0.16" changes: - description: Added categories and/or subcategories. From 6822d93051ff0983441de7bcde8e299b17ee7d48 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Thu, 17 Aug 2023 11:30:21 +0530 Subject: [PATCH 3/7] Update dimensions. --- packages/azure_metrics/changelog.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 4 ++++ .../storage_account/fields/package-fields.yml | 23 ++++++++++--------- .../data_stream/storage_account/manifest.yml | 2 +- packages/azure_metrics/docs/README.md | 4 ++-- .../azure_metrics/docs/storage_account.md | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/packages/azure_metrics/changelog.yml b/packages/azure_metrics/changelog.yml index faf92674102..ad4835416bc 100644 --- a/packages/azure_metrics/changelog.yml +++ b/packages/azure_metrics/changelog.yml @@ -1,6 +1,6 @@ - version: "1.0.20" changes: - - description: Add dimension to the storage account datastream + - description: Add dimensions to the storage account datastream type: enhancement link: https://github.com/elastic/integrations/pull/7356 - version: "1.0.16" diff --git a/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml b/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml index 71c383dbd29..593421f00a2 100644 --- a/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml @@ -8,6 +8,10 @@ processors: field: azure.storage target_field: azure.storage_account ignore_missing: true + - fingerprint: + fields: ["azure.dimensions"] + target_field: 'azure.dimensions.fingerprint' + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml b/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml index 62fcaaa3ee4..8d654c42628 100644 --- a/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml +++ b/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml @@ -58,17 +58,18 @@ description: > The application ID - - name: dimensions - type: group - fields: - - name: response_type - type: keyword - dimension: true - description: Response Type such as Success, ClientOtherError, etc. - - name: api_name - type: keyword - dimension: true - description: API name such as EntityGroupTransaction, GetBlobServiceProperties, etc. + - name: dimensions.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Azure metric dimensions. + + - name: dimensions.fingerprint + type: keyword + description: | + Autogenerated ID representing the fingerprint of the azure.dimensions object + dimension: true - name: metrics.*.* type: object diff --git a/packages/azure_metrics/data_stream/storage_account/manifest.yml b/packages/azure_metrics/data_stream/storage_account/manifest.yml index 36b843448ff..8479f0b3bed 100644 --- a/packages/azure_metrics/data_stream/storage_account/manifest.yml +++ b/packages/azure_metrics/data_stream/storage_account/manifest.yml @@ -31,4 +31,4 @@ streams: required: false show_user: true title: Storage Account - description: Collect Storage Account metrics + description: Collect Storage Account metrics \ No newline at end of file diff --git a/packages/azure_metrics/docs/README.md b/packages/azure_metrics/docs/README.md index 5649e6e4726..5b252d020a1 100644 --- a/packages/azure_metrics/docs/README.md +++ b/packages/azure_metrics/docs/README.md @@ -253,8 +253,8 @@ so the `period` for `storage_account` should be `300s` or multiples of `300s`. | @timestamp | Event timestamp. | date | | agent.id | | keyword | | azure.application_id | The application ID | keyword | -| azure.dimensions.api_name | API name such as EntityGroupTransaction, GetBlobServiceProperties, etc. | keyword | -| azure.dimensions.response_type | Response Type such as Success, ClientOtherError, etc. | keyword | +| azure.dimensions.\* | Azure metric dimensions. | object | +| azure.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the azure.dimensions object | keyword | | azure.metrics.\*.\* | Metrics returned. | object | | azure.namespace | The namespace selected | keyword | | azure.resource.group | The resource group | keyword | diff --git a/packages/azure_metrics/docs/storage_account.md b/packages/azure_metrics/docs/storage_account.md index 923744faca3..3651bbfc60b 100644 --- a/packages/azure_metrics/docs/storage_account.md +++ b/packages/azure_metrics/docs/storage_account.md @@ -75,8 +75,8 @@ Authentication: we are handling authentication on our side (creating/renewing th | @timestamp | Event timestamp. | date | | agent.id | | keyword | | azure.application_id | The application ID | keyword | -| azure.dimensions.api_name | API name such as EntityGroupTransaction, GetBlobServiceProperties, etc. | keyword | -| azure.dimensions.response_type | Response Type such as Success, ClientOtherError, etc. | keyword | +| azure.dimensions.\* | Azure metric dimensions. | object | +| azure.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the azure.dimensions object | keyword | | azure.metrics.\*.\* | Metrics returned. | object | | azure.namespace | The namespace selected | keyword | | azure.resource.group | The resource group | keyword | From 4a553251bafbcba7ca910f86d9af0753ff41a9cc Mon Sep 17 00:00:00 2001 From: ritalwar Date: Wed, 27 Sep 2023 13:48:17 +0530 Subject: [PATCH 4/7] Address review comments --- .../data_stream/storage_account/fields/agent.yml | 11 +---------- .../data_stream/storage_account/fields/ecs.yml | 3 +++ packages/azure_metrics/docs/README.md | 2 +- packages/azure_metrics/docs/storage_account.md | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/azure_metrics/data_stream/storage_account/fields/agent.yml b/packages/azure_metrics/data_stream/storage_account/fields/agent.yml index bf4208ac2f7..61a93a5e201 100644 --- a/packages/azure_metrics/data_stream/storage_account/fields/agent.yml +++ b/packages/azure_metrics/data_stream/storage_account/fields/agent.yml @@ -195,13 +195,4 @@ type: keyword example: "stretch" description: > - OS codename, if any. - -- name: agent - title: Agent - type: group - fields: - - name: id - type: keyword - ignore_above: 1024 - dimension: true \ No newline at end of file + OS codename, if any. \ No newline at end of file diff --git a/packages/azure_metrics/data_stream/storage_account/fields/ecs.yml b/packages/azure_metrics/data_stream/storage_account/fields/ecs.yml index c9ba9972270..0ef9d19c59d 100644 --- a/packages/azure_metrics/data_stream/storage_account/fields/ecs.yml +++ b/packages/azure_metrics/data_stream/storage_account/fields/ecs.yml @@ -15,3 +15,6 @@ external: ecs - name: host external: ecs +- name: agent.id + external: ecs + dimension: true diff --git a/packages/azure_metrics/docs/README.md b/packages/azure_metrics/docs/README.md index 3d9bd3a0b20..df8bb95ab35 100644 --- a/packages/azure_metrics/docs/README.md +++ b/packages/azure_metrics/docs/README.md @@ -287,7 +287,7 @@ so the `period` for `storage_account` should be `300s` or multiples of `300s`. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| agent.id | | keyword | +| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | azure.application_id | The application ID | keyword | | azure.dimensions.\* | Azure metric dimensions. | object | | azure.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the azure.dimensions object | keyword | diff --git a/packages/azure_metrics/docs/storage_account.md b/packages/azure_metrics/docs/storage_account.md index 3651bbfc60b..75151cc2b67 100644 --- a/packages/azure_metrics/docs/storage_account.md +++ b/packages/azure_metrics/docs/storage_account.md @@ -73,7 +73,7 @@ Authentication: we are handling authentication on our side (creating/renewing th | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| agent.id | | keyword | +| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | azure.application_id | The application ID | keyword | | azure.dimensions.\* | Azure metric dimensions. | object | | azure.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the azure.dimensions object | keyword | From a225cd202e284f5e6dfc8f609ed6556bdbf4af42 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Wed, 27 Sep 2023 15:47:09 +0530 Subject: [PATCH 5/7] Extend dimensions list to specify them individually, remove fingerprint processor. --- .../elasticsearch/ingest_pipeline/default.yml | 4 -- .../storage_account/fields/package-fields.yml | 47 ++++++++++++++----- packages/azure_metrics/docs/README.md | 10 +++- .../azure_metrics/docs/storage_account.md | 10 +++- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml b/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml index 593421f00a2..71c383dbd29 100644 --- a/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_metrics/data_stream/storage_account/elasticsearch/ingest_pipeline/default.yml @@ -8,10 +8,6 @@ processors: field: azure.storage target_field: azure.storage_account ignore_missing: true - - fingerprint: - fields: ["azure.dimensions"] - target_field: 'azure.dimensions.fingerprint' - ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml b/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml index 8d654c42628..1ce4abf333b 100644 --- a/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml +++ b/packages/azure_metrics/data_stream/storage_account/fields/package-fields.yml @@ -58,18 +58,41 @@ description: > The application ID - - name: dimensions.* - type: object - object_type: keyword - object_type_mapping_type: "*" - description: > - Azure metric dimensions. - - - name: dimensions.fingerprint - type: keyword - description: | - Autogenerated ID representing the fingerprint of the azure.dimensions object - dimension: true + - name: dimensions + type: group + fields: + - name: response_type + type: keyword + dimension: true + description: Transaction response type like Success, ClientOtherError, etc. + - name: api_name + type: keyword + dimension: true + description: The name of operation. + - name: geo_type + type: keyword + dimension: true + description: Transaction from Primary or Secondary cluster. The available values include Primary and Secondary. + - name: authentication + type: keyword + dimension: true + description: Authentication type used in transactions like OAuth. + - name: blob_type + type: keyword + dimension: true + description: Specifies the type of a blob. + - name: tier + type: keyword + dimension: true + description: Specifies access tier. + - name: file_share + type: keyword + dimension: true + description: Specifies file share. + - name: transaction_type + type: keyword + dimension: true + description: Type of transaction. The available values include User and System. - name: metrics.*.* type: object diff --git a/packages/azure_metrics/docs/README.md b/packages/azure_metrics/docs/README.md index df8bb95ab35..782da73b93d 100644 --- a/packages/azure_metrics/docs/README.md +++ b/packages/azure_metrics/docs/README.md @@ -289,8 +289,14 @@ so the `period` for `storage_account` should be `300s` or multiples of `300s`. | @timestamp | Event timestamp. | date | | agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | azure.application_id | The application ID | keyword | -| azure.dimensions.\* | Azure metric dimensions. | object | -| azure.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the azure.dimensions object | keyword | +| azure.dimensions.api_name | The name of operation. | keyword | +| azure.dimensions.authentication | Authentication type used in transactions like OAuth. | keyword | +| azure.dimensions.blob_type | Specifies the type of a blob. | keyword | +| azure.dimensions.file_share | Specifies file share. | keyword | +| azure.dimensions.geo_type | Transaction from Primary or Secondary cluster. The available values include Primary and Secondary. | keyword | +| azure.dimensions.response_type | Transaction response type like Success, ClientOtherError, etc. | keyword | +| azure.dimensions.tier | Specifies access tier. | keyword | +| azure.dimensions.transaction_type | Type of transaction. The available values include User and System. | keyword | | azure.metrics.\*.\* | Metrics returned. | object | | azure.namespace | The namespace selected | keyword | | azure.resource.group | The resource group | keyword | diff --git a/packages/azure_metrics/docs/storage_account.md b/packages/azure_metrics/docs/storage_account.md index 75151cc2b67..e6863dfa7f0 100644 --- a/packages/azure_metrics/docs/storage_account.md +++ b/packages/azure_metrics/docs/storage_account.md @@ -75,8 +75,14 @@ Authentication: we are handling authentication on our side (creating/renewing th | @timestamp | Event timestamp. | date | | agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | azure.application_id | The application ID | keyword | -| azure.dimensions.\* | Azure metric dimensions. | object | -| azure.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the azure.dimensions object | keyword | +| azure.dimensions.api_name | The name of operation. | keyword | +| azure.dimensions.authentication | Authentication type used in transactions like OAuth. | keyword | +| azure.dimensions.blob_type | Specifies the type of a blob. | keyword | +| azure.dimensions.file_share | Specifies file share. | keyword | +| azure.dimensions.geo_type | Transaction from Primary or Secondary cluster. The available values include Primary and Secondary. | keyword | +| azure.dimensions.response_type | Transaction response type like Success, ClientOtherError, etc. | keyword | +| azure.dimensions.tier | Specifies access tier. | keyword | +| azure.dimensions.transaction_type | Type of transaction. The available values include User and System. | keyword | | azure.metrics.\*.\* | Metrics returned. | object | | azure.namespace | The namespace selected | keyword | | azure.resource.group | The resource group | keyword | From e8a5d9642b177f61929c1d6d5b27d0a995ab101d Mon Sep 17 00:00:00 2001 From: ritalwar Date: Wed, 27 Sep 2023 19:24:01 +0530 Subject: [PATCH 6/7] Update kibana and package version --- packages/azure_metrics/changelog.yml | 2 +- packages/azure_metrics/manifest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/azure_metrics/changelog.yml b/packages/azure_metrics/changelog.yml index 51f2c676545..b3fc67de0aa 100644 --- a/packages/azure_metrics/changelog.yml +++ b/packages/azure_metrics/changelog.yml @@ -1,4 +1,4 @@ -- version: "1.0.44" +- version: "1.1.0" changes: - description: Add dimensions to the storage account datastream type: enhancement diff --git a/packages/azure_metrics/manifest.yml b/packages/azure_metrics/manifest.yml index 0e757d8b491..8a22dddb98c 100644 --- a/packages/azure_metrics/manifest.yml +++ b/packages/azure_metrics/manifest.yml @@ -1,6 +1,6 @@ name: azure_metrics title: Azure Resource Metrics -version: 1.0.44 +version: 1.1.0 release: ga description: Collect metrics from Azure resources with Elastic Agent. type: integration @@ -20,7 +20,7 @@ categories: - observability - azure conditions: - kibana.version: "^8.9.0" + kibana.version: "^8.10.0" vars: - name: client_id type: text From bceab5166fdbd8b3649c59b2757a0a0670478c52 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Tue, 3 Oct 2023 10:19:12 +0530 Subject: [PATCH 7/7] Update kibana version to 8.10.2 --- packages/azure_metrics/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure_metrics/manifest.yml b/packages/azure_metrics/manifest.yml index 8a22dddb98c..1480a55c984 100644 --- a/packages/azure_metrics/manifest.yml +++ b/packages/azure_metrics/manifest.yml @@ -20,7 +20,7 @@ categories: - observability - azure conditions: - kibana.version: "^8.10.0" + kibana.version: "^8.10.2" vars: - name: client_id type: text