From 4f0b28dee5367e624ec784622ed69390474e09b7 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Fri, 6 Oct 2023 16:59:30 +0200 Subject: [PATCH 01/11] Change in Metricbeat - Azure Timespan to get metric values --- x-pack/metricbeat/module/azure/client.go | 5 ++++- x-pack/metricbeat/module/azure/client_utils.go | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index e9595425be3..0da0c0fc054 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -111,7 +111,10 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] if t := convertTimegrainToDuration(metric.TimeGrain); t > interval*2 { interval = t } - endTime := time.Now().UTC() + + // Fetch in the range [{-2xINTERVAL},{-INTERVAL}) with a delay of {INTERVAL} + // It results in one data point {-2xINTERVAL} per call + endTime := time.Now().UTC().Add(interval * (-1)) startTime := endTime.Add(interval * (-2)) timespan := fmt.Sprintf("%s/%s", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index d96996c7215..1bd800069df 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -29,9 +29,10 @@ func mapMetricValues(metrics []armmonitor.Metric, previousMetrics []MetricValue, continue } // remove metric values that are not part of the timeline selected - if mv.TimeStamp.After(startTime) && mv.TimeStamp.Before(endTime) { - continue - } + // We don't need this filtering anymore, since we modify the timespan directly + //if mv.TimeStamp.After(startTime) && mv.TimeStamp.Before(endTime) { + // continue + //} // define the new metric value and match aggregations values var val MetricValue val.name = *v.Name.Value From 81b452ad8c67a276682ec5a53b4bc3f2f80321bc Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 9 Oct 2023 12:08:14 +0200 Subject: [PATCH 02/11] fix linter --- x-pack/metricbeat/module/azure/client.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 0da0c0fc054..daad3209e2b 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -9,9 +9,6 @@ import ( "strings" "time" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" ) @@ -173,7 +170,6 @@ func (client *Client) CreateMetric(resourceId string, subResourceId string, name // MapMetricByPrimaryAggregation will map the primary aggregation of the metric definition to the client metric func (client *Client) MapMetricByPrimaryAggregation(metrics []armmonitor.MetricDefinition, resourceId string, subResourceId string, namespace string, dim []Dimension, timegrain string) []Metric { - var clientMetrics []Metric metricGroups := make(map[string][]armmonitor.MetricDefinition) @@ -181,6 +177,8 @@ func (client *Client) MapMetricByPrimaryAggregation(metrics []armmonitor.MetricD metricGroups[string(*met.PrimaryAggregationType)] = append(metricGroups[string(*met.PrimaryAggregationType)], met) } + clientMetrics := make([]Metric, 0, len(metricGroups)) + for key, metricGroup := range metricGroups { var metricNames []string for _, metricName := range metricGroup { From 6a351e4bf3e44803399f204f109296ccf6f9c8be Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 9 Oct 2023 12:19:25 +0200 Subject: [PATCH 03/11] fix linter 2 --- x-pack/metricbeat/module/azure/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index daad3209e2b..314a0c9708c 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -9,6 +9,9 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" ) From 0752f9f47a2dc7e9d736b8582b5262828f703612 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Wed, 11 Oct 2023 13:49:00 +0200 Subject: [PATCH 04/11] fix interval --- x-pack/metricbeat/module/azure/client.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 314a0c9708c..51a9c36a85d 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -9,9 +9,6 @@ import ( "strings" "time" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" ) @@ -115,7 +112,7 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] // Fetch in the range [{-2xINTERVAL},{-INTERVAL}) with a delay of {INTERVAL} // It results in one data point {-2xINTERVAL} per call endTime := time.Now().UTC().Add(interval * (-1)) - startTime := endTime.Add(interval * (-2)) + startTime := endTime.Add(interval * (-1)) timespan := fmt.Sprintf("%s/%s", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) // build the 'filter' parameter which will contain any dimensions configured From 3385e014e9951245a56908c8a1aa36754ba8634c Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Wed, 11 Oct 2023 13:58:28 +0200 Subject: [PATCH 05/11] fix imports --- x-pack/metricbeat/module/azure/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 51a9c36a85d..756d4b0102a 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -9,6 +9,9 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" ) From 1042bf3dec16b2f377aec132f4c8a39e300d5121 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 16 Oct 2023 16:38:17 +0200 Subject: [PATCH 06/11] Algorithm base for timegrain > sampling period --- x-pack/metricbeat/module/azure/client.go | 54 +++++++++++++++++-- .../metricbeat/module/azure/client_utils.go | 2 +- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 756d4b0102a..ae17d6925f8 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -103,18 +103,26 @@ func (client *Client) InitResources(fn mapResourceMetrics) error { // GetMetricValues returns the specified metric data points for the specified resource ID/namespace. func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) []Metric { var resultedMetrics []Metric + + // Same end time for all metrics in the same batch + endTime := time.Now().UTC() + // loop over the set of metrics for _, metric := range metrics { // select period to collect metrics, will double the interval value in order to retrieve any missing values //if timegrain is larger than intervalx2 then interval will be assigned the timegrain value interval := client.Config.Period - if t := convertTimegrainToDuration(metric.TimeGrain); t > interval*2 { - interval = t + duration := convertTimegrainToDuration(metric.TimeGrain) + if t := duration; t > interval*2 { + //interval = t } + // Adjust end time based on timegrain + endTime = endTime.Add(interval * (-1)) + // Fetch in the range [{-2xINTERVAL},{-INTERVAL}) with a delay of {INTERVAL} // It results in one data point {-2xINTERVAL} per call - endTime := time.Now().UTC().Add(interval * (-1)) + // Adjust start time based on timegrain for the current metric startTime := endTime.Add(interval * (-1)) timespan := fmt.Sprintf("%s/%s", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) @@ -127,6 +135,44 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } filter = strings.Join(filterList, " AND ") } + + increment := time.Minute + + // TODO if timegrain is unit 1 day + + // Interval math for interval > timegrain + if duration > client.Config.Period { + + in_flag := false + + for d := startTime; d.After(endTime) == false; d = d.Add(increment) { + // if timegrain is in unit hours + if duration >= time.Hour { + // if hour mark is within the timespan + if d.Minute() == 0 { + in_flag = true + break + } + } + + // TODO if timegrain is in unit minutes + } + + // if the timegrain mark is not within the sampling timespan, remove that metric from the list in this batch and skip to the next one + if !in_flag { + // Remove metric from list + ind_to_remove := 0 + for i, currentMetric := range client.ResourceConfigurations.Metrics { + if matchMetrics(currentMetric, metric) { + ind_to_remove = i + break + } + } + client.ResourceConfigurations.Metrics = append(client.ResourceConfigurations.Metrics[:ind_to_remove], client.ResourceConfigurations.Metrics[ind_to_remove+1:]...) + continue + } + } + resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.ResourceSubId, metric.Namespace, metric.TimeGrain, timespan, metric.Names, metric.Aggregations, filter) if err != nil { @@ -136,7 +182,7 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } else { for i, currentMetric := range client.ResourceConfigurations.Metrics { if matchMetrics(currentMetric, metric) { - current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) + current := mapMetricValues(resp, currentMetric.Values) client.ResourceConfigurations.Metrics[i].Values = current if client.ResourceConfigurations.Metrics[i].TimeGrain == "" { client.ResourceConfigurations.Metrics[i].TimeGrain = timegrain diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index 1bd800069df..8184d1af95b 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -19,7 +19,7 @@ const DefaultTimeGrain = "PT5M" var instanceIdRegex = regexp.MustCompile(`.*?(\d+)$`) // mapMetricValues should map the metric values -func mapMetricValues(metrics []armmonitor.Metric, previousMetrics []MetricValue, startTime time.Time, endTime time.Time) []MetricValue { +func mapMetricValues(metrics []armmonitor.Metric, previousMetrics []MetricValue) []MetricValue { var currentMetrics []MetricValue // compare with the previously returned values and filter out any double records for _, v := range metrics { From 2c51fef0bce52421edf0d617a2e30295913976f1 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 16 Oct 2023 17:03:20 +0200 Subject: [PATCH 07/11] clean up --- x-pack/metricbeat/module/azure/client.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index ae17d6925f8..ef3ec4b5486 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -9,9 +9,6 @@ import ( "strings" "time" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" ) @@ -113,9 +110,6 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] //if timegrain is larger than intervalx2 then interval will be assigned the timegrain value interval := client.Config.Period duration := convertTimegrainToDuration(metric.TimeGrain) - if t := duration; t > interval*2 { - //interval = t - } // Adjust end time based on timegrain endTime = endTime.Add(interval * (-1)) @@ -145,7 +139,7 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] in_flag := false - for d := startTime; d.After(endTime) == false; d = d.Add(increment) { + for d := startTime; !d.After(endTime); d = d.Add(increment) { // if timegrain is in unit hours if duration >= time.Hour { // if hour mark is within the timespan From d8c37b7062aec344556cf299f1ff63005e21fb65 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 16 Oct 2023 17:24:34 +0200 Subject: [PATCH 08/11] import fix --- x-pack/metricbeat/module/azure/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index ef3ec4b5486..7dbf29cdd81 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -9,6 +9,9 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" ) From a96aedbfee0d1ff7e3a027b206b3428a4c60e57a Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 16 Oct 2023 20:34:50 +0200 Subject: [PATCH 09/11] Algoritm for timegrain > period - interval math --- x-pack/metricbeat/module/azure/client.go | 44 ++++++++++++++---------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 7dbf29cdd81..c3698c12c82 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -133,39 +133,47 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] filter = strings.Join(filterList, " AND ") } - increment := time.Minute - - // TODO if timegrain is unit 1 day - // Interval math for interval > timegrain if duration > client.Config.Period { - in_flag := false + inTimespan := false - for d := startTime; !d.After(endTime); d = d.Add(increment) { - // if timegrain is in unit hours - if duration >= time.Hour { - // if hour mark is within the timespan - if d.Minute() == 0 { - in_flag = true - break - } + var diffSec int64 = int64(endTime.Second() - startTime.Second()) + + var diffMin int64 = int64(endTime.Minute() - startTime.Minute()) + var diffMinDuration time.Duration = time.Duration(diffMin) * time.Minute + + var diffHour int64 = int64(endTime.Hour() - startTime.Hour()) + var diffHourDuration time.Duration = time.Duration(diffHour) * time.Hour + + // If timegrain is unit 1 day, 1 hour or 1 min + if duration == 24*time.Hour { + startOfDay := endTime.Truncate(24 * time.Hour) + if (startOfDay.Equal(startTime) || startOfDay.After(startTime)) && startOfDay.Before(endTime) { + inTimespan = true } - // TODO if timegrain is in unit minutes + } else if duration >= time.Hour { + if diffMin < 0 && diffHourDuration > 0 && diffHourDuration%duration == 0 { + inTimespan = true + } + } else { + if diffSec < 0 && diffMinDuration%duration == 0 { + inTimespan = true + } } // if the timegrain mark is not within the sampling timespan, remove that metric from the list in this batch and skip to the next one - if !in_flag { + if !inTimespan { // Remove metric from list - ind_to_remove := 0 + ind := 0 for i, currentMetric := range client.ResourceConfigurations.Metrics { if matchMetrics(currentMetric, metric) { - ind_to_remove = i + ind = i break } } - client.ResourceConfigurations.Metrics = append(client.ResourceConfigurations.Metrics[:ind_to_remove], client.ResourceConfigurations.Metrics[ind_to_remove+1:]...) + client.ResourceConfigurations.Metrics = append(client.ResourceConfigurations.Metrics[:ind], client.ResourceConfigurations.Metrics[ind+1:]...) continue } } From aad692e10777041601ad562492feb552c9723af5 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Mon, 16 Oct 2023 20:58:38 +0200 Subject: [PATCH 10/11] Style change --- x-pack/metricbeat/module/azure/client.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index c3698c12c82..f25a4f996c6 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -138,13 +138,13 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] inTimespan := false - var diffSec int64 = int64(endTime.Second() - startTime.Second()) + var diffSec = int64(endTime.Second() - startTime.Second()) - var diffMin int64 = int64(endTime.Minute() - startTime.Minute()) - var diffMinDuration time.Duration = time.Duration(diffMin) * time.Minute + var diffMin = int64(endTime.Minute() - startTime.Minute()) + var diffMinDuration = time.Duration(diffMin) * time.Minute - var diffHour int64 = int64(endTime.Hour() - startTime.Hour()) - var diffHourDuration time.Duration = time.Duration(diffHour) * time.Hour + var diffHour = int64(endTime.Hour() - startTime.Hour()) + var diffHourDuration = time.Duration(diffHour) * time.Hour // If timegrain is unit 1 day, 1 hour or 1 min if duration == 24*time.Hour { From 83ef3da576f8bc5c41c2757ec6b0481ec08a2e38 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva Date: Wed, 18 Oct 2023 18:15:42 +0200 Subject: [PATCH 11/11] Comment fix --- x-pack/metricbeat/module/azure/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index f25a4f996c6..7e80785c20a 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -133,7 +133,7 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] filter = strings.Join(filterList, " AND ") } - // Interval math for interval > timegrain + // Interval math for timegrain > period if duration > client.Config.Period { inTimespan := false