From 6a74b9f7e9e49006aaebd709f1b60b808c277d07 Mon Sep 17 00:00:00 2001 From: Vastin Date: Thu, 12 Oct 2023 15:52:29 -0500 Subject: [PATCH 1/4] Put metrics fields to bottom of EMF log entry. **Tracking Issues:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/26697 --- exporter/awsemfexporter/metric_translator.go | 25 +++++++++++++++++++ .../awsemfexporter/metric_translator_test.go | 4 +-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/exporter/awsemfexporter/metric_translator.go b/exporter/awsemfexporter/metric_translator.go index 26aba7bd368e..4b472ea7df60 100644 --- a/exporter/awsemfexporter/metric_translator.go +++ b/exporter/awsemfexporter/metric_translator.go @@ -431,11 +431,36 @@ func translateCWMetricToEMF(cWMetric *cWMetrics, config *Config) (*cwlogs.Event, } } + // remove metrics from fieldMap + metricsMap := make(map[string]interface{}) + for _, measurement := range cWMetric.measurements { + for _, metric := range measurement.Metrics { + metricName, exist := metric["Name"] + if exist { + v, ok := fieldMap[metricName] + if ok { + metricsMap[metricName] = v + delete(fieldMap, metricName) + } + } + } + } + pleMsg, err := json.Marshal(fieldMap) if err != nil { return nil, err } + // append metrics json to pleMsg + if len(metricsMap) > 0 { + metricsMsg, err := json.Marshal(metricsMap) + if err != nil { + return nil, err + } + metricsMsg[0] = ',' + pleMsg = append(pleMsg[:len(pleMsg)-1], metricsMsg...) + } + metricCreationTime := cWMetric.timestampMs logEvent := cwlogs.NewEvent( metricCreationTime, diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index 1ac411048e6d..bbd5b8120b04 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -403,7 +403,7 @@ func TestTranslateCWMetricToEMF(t *testing.T) { "Unit": "Count", }}, }}, - expectedEMFLogEvent: "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Version\":\"1\",\"_aws\":{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"Timestamp\":1596151098037},\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanCounter\":0,\"spanName\":\"test\"}", + expectedEMFLogEvent: "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Version\":\"1\",\"_aws\":{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"Timestamp\":1596151098037},\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanName\":\"test\",\"spanCounter\":0}", }, "WithMeasurementAndEMFV0": { emfVersion: "0", @@ -415,7 +415,7 @@ func TestTranslateCWMetricToEMF(t *testing.T) { "Unit": "Count", }}, }}, - expectedEMFLogEvent: "{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Timestamp\":\"1596151098037\",\"Version\":\"0\",\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanCounter\":0,\"spanName\":\"test\"}", + expectedEMFLogEvent: "{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Timestamp\":\"1596151098037\",\"Version\":\"0\",\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanName\":\"test\",\"spanCounter\":0}", }, "WithNoMeasurementAndEMFV1": { emfVersion: "1", From 594a2203a0c822164dc863d9742f4640770a5cd6 Mon Sep 17 00:00:00 2001 From: jjllee Date: Fri, 10 May 2024 20:50:21 -0700 Subject: [PATCH 2/4] cleanup cherry-pick - interface{} -> any --- exporter/awsemfexporter/metric_translator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/awsemfexporter/metric_translator.go b/exporter/awsemfexporter/metric_translator.go index 4b472ea7df60..f71b6a9731fe 100644 --- a/exporter/awsemfexporter/metric_translator.go +++ b/exporter/awsemfexporter/metric_translator.go @@ -432,7 +432,7 @@ func translateCWMetricToEMF(cWMetric *cWMetrics, config *Config) (*cwlogs.Event, } // remove metrics from fieldMap - metricsMap := make(map[string]interface{}) + metricsMap := make(map[string]any) for _, measurement := range cWMetric.measurements { for _, metric := range measurement.Metrics { metricName, exist := metric["Name"] From 788be8fd47f668a9ead1249aa7ebd32f261bd7ec Mon Sep 17 00:00:00 2001 From: jjllee Date: Fri, 10 May 2024 21:48:24 -0700 Subject: [PATCH 3/4] entry for .chloggen --- ..._awsemfexporter-put-metrics-to-bottom.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml diff --git a/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml b/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml new file mode 100644 index 000000000000..ccc0cc769d76 --- /dev/null +++ b/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: awsemfexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: AWS EMF Exporter to place metrics fields to the bottom of log entry to workaround CW log insight limitation + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] From 3799516d3c366479a7411add322f74b1f01d93a2 Mon Sep 17 00:00:00 2001 From: jjllee Date: Wed, 15 May 2024 11:16:43 -0700 Subject: [PATCH 4/4] update .chloggen entry --- .chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml b/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml index ccc0cc769d76..cffa1e840b5a 100644 --- a/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml +++ b/.chloggen/fix_awsemfexporter-put-metrics-to-bottom.yaml @@ -10,7 +10,7 @@ component: awsemfexporter note: AWS EMF Exporter to place metrics fields to the bottom of log entry to workaround CW log insight limitation # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. -issues: [] +issues: [33002] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document.