Merge branch 'amazon-contributing:aws-cwa-dev' into aws-cwa-dev

.chloggen/add-aws-http-error-event.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awsxrayexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Convert individual HTTP error events into exceptions within subsegments for AWS SDK spans and strip AWS.SDK prefix from remote aws service name
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [27232]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/aws_exporter_localrootspans.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awsxrayexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: AWS X-Ray exporter to make local root spans a segment for internal/service spans and subsegment + segment for client/producer/consumer spans.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [102]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/awscloudwatchlogsexporter/exporter.go

@@ -61,7 +61,7 @@ func newCwLogsPusher(expConfig *Config, params exp.CreateSettings) (*exporter, e
 	}
 
 	// create CWLogs client with aws session config
-	svcStructuredLog := cwlogs.NewClient(params.Logger, awsConfig, params.BuildInfo, expConfig.LogGroupName, expConfig.LogRetention, expConfig.Tags, session, false)
+	svcStructuredLog := cwlogs.NewClient(params.Logger, awsConfig, params.BuildInfo, expConfig.LogGroupName, expConfig.LogRetention, expConfig.Tags, session)
 	collectorIdentifier, err := uuid.NewRandom()
 
 	if err != nil {

exporter/awsemfexporter/emf_exporter.go

@@ -26,6 +26,10 @@ const (
 	// OutputDestination Options
 	outputDestinationCloudWatch = "cloudwatch"
 	outputDestinationStdout     = "stdout"
+
+	// Pulse EMF config
+	pulseMetricNamespace    = "AWS/APM"
+	pulseLogGroupNamePrefix = "/aws/apm/"
 )
 
 type emfExporter struct {
@@ -55,7 +59,16 @@ func newEmfExporter(config *Config, set exporter.CreateSettings) (*emfExporter,
 	}
 
 	// create CWLogs client with aws session config
-	svcStructuredLog := cwlogs.NewClient(set.Logger, awsConfig, set.BuildInfo, config.LogGroupName, config.LogRetention, config.Tags, session, isEnhancedContainerInsights(config))
+	svcStructuredLog := cwlogs.NewClient(set.Logger,
+		awsConfig,
+		set.BuildInfo,
+		config.LogGroupName,
+		config.LogRetention,
+		config.Tags,
+		session,
+		cwlogs.WithEnabledContainerInsights(isEnhancedContainerInsights(config)),
+		cwlogs.WithEnabledPulseApm(isPulseApmEnabled(config)),
+	)
 	collectorIdentifier, err := uuid.NewRandom()
 
 	if err != nil {
@@ -110,6 +123,11 @@ func (emf *emfExporter) pushMetricsData(_ context.Context, md pmetric.Metrics) e
 		if err != nil {
 			return err
 		}
+		// Drop a nil putLogEvent for EnhancedContainerInsights
+		if emf.config.EnhancedContainerInsights && putLogEvent == nil {
+			emf.config.logger.Debug("Dropping empty putLogEvents for EnhancedContainerInsights")
+			continue
+		}
 		// Currently we only support two options for "OutputDestination".
 		if strings.EqualFold(outputDestination, outputDestinationStdout) {
 			if putLogEvent != nil &&
@@ -203,3 +221,15 @@ func isEnhancedContainerInsights(_ *Config) bool {
 	return false // temporarily disable, also need to rename _config to config
 	// return config.EnhancedContainerInsights && !config.DisableMetricExtraction
 }
+
+func isPulseApmEnabled(config *Config) bool {
+	if config.LogGroupName == "" || config.Namespace == "" {
+		return false
+	}
+
+	if config.Namespace == pulseMetricNamespace && strings.HasPrefix(config.LogGroupName, pulseLogGroupNamePrefix) {
+		return true
+	}
+
+	return false
+}

exporter/awsemfexporter/emf_exporter_test.go

@@ -361,3 +361,58 @@ func TestIsEnhancedContainerInsights(t *testing.T) {
 	cfg.DisableMetricExtraction = true
 	assert.False(t, isEnhancedContainerInsights(cfg))
 }
+
+func TestIsPulseApmEnabled(t *testing.T) {
+
+	tests := []struct {
+		name            string
+		metricNameSpace string
+		logGroupName    string
+		expectedResult  bool
+	}{
+		{
+			"validPulseEMF",
+			"AWS/APM",
+			"/aws/apm/eks",
+			true,
+		},
+		{
+			"invalidPulseLogsGroup",
+			"AWS/APM",
+			"/nonaws/apm/eks",
+			false,
+		},
+		{
+			"invalidPulseMetricNamespace",
+			"NonAWS/APM",
+			"/aws/apm/eks",
+			false,
+		},
+		{
+			"invalidPulseEMF",
+			"NonAWS/APM",
+			"/nonaws/apm/eks",
+			false,
+		},
+		{
+			"defaultConfig",
+			"",
+			"",
+			false,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			factory := NewFactory()
+			cfg := factory.CreateDefaultConfig().(*Config)
+			if len(tc.metricNameSpace) > 0 {
+				cfg.Namespace = tc.metricNameSpace
+			}
+			if len(tc.logGroupName) > 0 {
+				cfg.LogGroupName = tc.logGroupName
+			}
+
+			assert.Equal(t, isPulseApmEnabled(cfg), tc.expectedResult)
+		})
+	}
+}

exporter/awsemfexporter/metric_translator.go

@@ -431,13 +431,41 @@ func translateCWMetricToEMF(cWMetric *cWMetrics, config *Config) (*cwlogs.Event,
 			*/
 			fieldMap["CloudWatchMetrics"] = cWMetric.measurements
 		}
+	} else if len(cWMetric.measurements) < 1 && config.EnhancedContainerInsights {
+		// Return nil if requests does not contain metrics when EnhancedContainerInsights is enabled
+		return nil, nil
+	}
+
+	// remove metrics from fieldMap
+	metricsMap := make(map[string]interface{})
+	for _, measurement := range cWMetric.measurements {
+		for _, metric := range measurement.Metrics {
+			metricName, exist := metric["Name"]
+			if exist {
+				v, ok := fieldMap[metricName]
+				if ok {
+					metricsMap[metricName] = v
+					delete(fieldMap, metricName)
+				}
+			}
+		}
 	}
 
 	pleMsg, err := json.Marshal(fieldMap)
 	if err != nil {
 		return nil, err
 	}
 
+	// append metrics json to pleMsg
+	if len(metricsMap) > 0 {
+		metricsMsg, err := json.Marshal(metricsMap)
+		if err != nil {
+			return nil, err
+		}
+		metricsMsg[0] = ','
+		pleMsg = append(pleMsg[:len(pleMsg)-1], metricsMsg...)
+	}
+
 	metricCreationTime := cWMetric.timestampMs
 	logEvent := cwlogs.NewEvent(
 		metricCreationTime,

exporter/awsemfexporter/metric_translator_test.go

@@ -405,7 +405,7 @@ func TestTranslateCWMetricToEMF(t *testing.T) {
 				}},
 			}},
 			disableMetricExtraction: false,
-			expectedEMFLogEvent:     "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Version\":\"1\",\"_aws\":{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"Timestamp\":1596151098037},\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanCounter\":0,\"spanName\":\"test\"}",
+			expectedEMFLogEvent:     "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Version\":\"1\",\"_aws\":{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"Timestamp\":1596151098037},\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanName\":\"test\",\"spanCounter\":0}",
 		},
 		"WithMeasurementAndEMFV0": {
 			emfVersion: "0",
@@ -418,7 +418,7 @@ func TestTranslateCWMetricToEMF(t *testing.T) {
 				}},
 			}},
 			disableMetricExtraction: false,
-			expectedEMFLogEvent:     "{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Timestamp\":\"1596151098037\",\"Version\":\"0\",\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanCounter\":0,\"spanName\":\"test\"}",
+			expectedEMFLogEvent:     "{\"CloudWatchMetrics\":[{\"Namespace\":\"test-emf\",\"Dimensions\":[[\"OTelLib\"],[\"OTelLib\",\"spanName\"]],\"Metrics\":[{\"Name\":\"spanCounter\",\"Unit\":\"Count\"}]}],\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"Timestamp\":\"1596151098037\",\"Version\":\"0\",\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanName\":\"test\",\"spanCounter\":0}",
 		},
 		"WithNoMeasurement": {
 			emfVersion:              "1",
@@ -443,7 +443,7 @@ func TestTranslateCWMetricToEMF(t *testing.T) {
 				}},
 			}},
 			disableMetricExtraction: true,
-			expectedEMFLogEvent:     "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanCounter\":0,\"spanName\":\"test\"}",
+			expectedEMFLogEvent:     "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"cadvisor\",\"pod\",\"calculated\"],\"kubernetes\":{\"container_name\":\"cloudwatch-agent\",\"docker\":{\"container_id\":\"fc1b0a4c3faaa1808e187486a3a90cbea883dccaf2e2c46d4069d663b032a1ca\"},\"host\":\"ip-192-168-58-245.ec2.internal\",\"labels\":{\"controller-revision-hash\":\"5bdbf497dc\",\"name\":\"cloudwatch-agent\",\"pod-template-generation\":\"1\"},\"namespace_name\":\"amazon-cloudwatch\",\"pod_id\":\"e23f3413-af2e-4a98-89e0-5df2251e7f05\",\"pod_name\":\"cloudwatch-agent-26bl6\",\"pod_owners\":[{\"owner_kind\":\"DaemonSet\",\"owner_name\":\"cloudwatch-agent\"}]},\"spanName\":\"test\",\"spanCounter\":0}",
 		},
 	}
 
@@ -481,6 +481,67 @@ func TestTranslateCWMetricToEMF(t *testing.T) {
 
 }
 
+func TestTranslateCWMetricToEMFForEnhancedContainerInsights(t *testing.T) {
+	testCases := map[string]struct {
+		EnhancedContainerInsights bool
+		fields                    map[string]interface{}
+		measurements              []cWMeasurement
+		expectedEMFLogEvent       interface{}
+	}{
+		"EnhancedContainerInsightsEnabled": {
+			EnhancedContainerInsights: true,
+			fields: map[string]interface{}{
+				oTellibDimensionKey:                     "cloudwatch-otel",
+				"scrape_samples_post_metric_relabeling": "12",
+				"scrape_samples_scraped":                "34",
+				"scrape_series_added":                   "56",
+				"service.instance.id":                   "1.2.3.4:443",
+				"Sources":                               "[\"apiserver\"]",
+			},
+			measurements:        nil,
+			expectedEMFLogEvent: nil,
+		},
+		"EnhancedContainerInsightsDisabled": {
+			EnhancedContainerInsights: false,
+			fields: map[string]interface{}{
+				oTellibDimensionKey:                     "cloudwatch-otel",
+				"scrape_samples_post_metric_relabeling": "12",
+				"scrape_samples_scraped":                "34",
+				"scrape_series_added":                   "56",
+				"service.instance.id":                   "1.2.3.4:443",
+				"Sources":                               "[\"apiserver\"]",
+			},
+			measurements:        nil,
+			expectedEMFLogEvent: "{\"OTelLib\":\"cloudwatch-otel\",\"Sources\":[\"apiserver\"],\"scrape_samples_post_metric_relabeling\":\"12\",\"scrape_samples_scraped\":\"34\",\"scrape_series_added\":\"56\",\"service.instance.id\":\"1.2.3.4:443\"}",
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(_ *testing.T) {
+			config := &Config{
+				// include valid json string, a non-existing key, and keys whose value are not json/string
+				ParseJSONEncodedAttributeValues: []string{"Sources"},
+				EnhancedContainerInsights:       tc.EnhancedContainerInsights,
+				logger:                          zap.NewNop(),
+			}
+
+			cloudwatchMetric := &cWMetrics{
+				timestampMs:  int64(1596151098037),
+				fields:       tc.fields,
+				measurements: tc.measurements,
+			}
+
+			emfLogEvent, err := translateCWMetricToEMF(cloudwatchMetric, config)
+			require.NoError(t, err)
+
+			if tc.expectedEMFLogEvent != nil {
+				assert.Equal(t, tc.expectedEMFLogEvent, *emfLogEvent.InputLogEvent.Message)
+			}
+		})
+	}
+
+}
+
 func TestTranslateGroupedMetricToCWMetric(t *testing.T) {
 	timestamp := int64(1596151098037)
 	namespace := "Namespace"
@@ -1395,22 +1456,22 @@ func TestGroupedMetricToCWMeasurementsWithFilters(t *testing.T) {
 					MetricNameSelectors: []string{"metric(1|3)"},
 				},
 			}, []cWMeasurement{
-				{
-					Namespace:  namespace,
-					Dimensions: [][]string{{}},
-					Metrics: []map[string]string{
-						{
-							"Name": "metric1",
-							"Unit": "Count",
-						},
-						{
-							"Name": "metric3",
-							"Unit": "Seconds",
-						},
+			{
+				Namespace:  namespace,
+				Dimensions: [][]string{{}},
+				Metrics: []map[string]string{
+					{
+						"Name": "metric1",
+						"Unit": "Count",
+					},
+					{
+						"Name": "metric3",
+						"Unit": "Seconds",
 					},
 				},
 			},
 		},
+		},
 		{
 			"label matchers",
 			[]*MetricDeclaration{

exporter/awsxrayexporter/awsxray.go

@@ -105,17 +105,21 @@ func extractResourceSpans(config component.Config, logger *zap.Logger, td ptrace
 		for j := 0; j < rspans.ScopeSpans().Len(); j++ {
 			spans := rspans.ScopeSpans().At(j).Spans()
 			for k := 0; k < spans.Len(); k++ {
-				document, localErr := translator.MakeSegmentDocumentString(
+				documentsForSpan, localErr := translator.MakeSegmentDocuments(
 					spans.At(k), resource,
 					config.(*Config).IndexedAttributes,
 					config.(*Config).IndexAllAttributes,
 					config.(*Config).LogGroupNames,
 					config.(*Config).skipTimestampValidation)
+
 				if localErr != nil {
 					logger.Debug("Error translating span.", zap.Error(localErr))
 					continue
 				}
-				documents = append(documents, &document)
+
+				for l := range documentsForSpan {
+					documents = append(documents, &documentsForSpan[l])
+				}
 			}
 		}
 	}