diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index 32b1315def..03a15b6c5f 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -54,6 +54,8 @@ jobs: - name: Check out code if: needs.changes.outputs.lint == 'true' uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Check format if: needs.changes.outputs.lint == 'true' @@ -67,7 +69,7 @@ jobs: - name: Check license and imports if: needs.changes.outputs.lint == 'true' - run: make simple-lint + run: make lint build: needs: [lint, changes] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ee89933159..8236ad0fe6 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -563,7 +563,7 @@ jobs: uses: nick-fields/retry@v2 with: max_attempts: 3 - timeout_minutes: 30 + timeout_minutes: 60 retry_wait_seconds: 5 command: | if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then diff --git a/.golangci.yml b/.golangci.yml index 27074e9371..f4fbed2f51 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,9 +29,6 @@ output: # All available settings of specific linters linters-settings: - revive: - # minimal confidence for issues, default is 0.8 - min-confidence: 0.7 gofmt: # Simplify code: gofmt with `-s` option, true by default simplify: true @@ -49,12 +46,17 @@ linters-settings: linters: disable: - errcheck + enable: - gofmt - goimports - enable: - - gosimple - gosec - - unused + - gosimple + - govet + - ineffassign - misspell - revive + - unused - nonamedreturns + +issues: + new-from-rev: 3221f76 \ No newline at end of file diff --git a/Makefile b/Makefile index 21b2fe7d88..b958a0e276 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ install-addlicense: install-golangci-lint: #Install from source for golangci-lint is not recommended based on https://golangci-lint.run/usage/install/#install-from-source so using binary #installation - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN_DIR) v1.50.1 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN_DIR) v1.62.2 fmt: install-goimports addlicense go fmt ./... diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ed001459a0..51bd121123 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,11 @@ +======================================================================== +Amazon CloudWatch Agent 1.300050.0 (2024-11-18) +======================================================================== +Features: +* [Prometheus] Introduce OTel Prometheus Receiver for publishing to AMP +* [Prometheus] Support Target Allocator with Prometheus Receivers +* [ContainerInsights] Introduce Kueue metrics for Container Insights + ======================================================================== Amazon CloudWatch Agent 1.300049.1 (2024-11-06) ======================================================================== diff --git a/go.mod b/go.mod index 3933309b22..3f81a43683 100644 --- a/go.mod +++ b/go.mod @@ -278,7 +278,7 @@ require ( github.com/dennwc/varint v1.0.0 // indirect github.com/digitalocean/godo v1.109.0 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v26.1.4+incompatible // indirect + github.com/docker/docker v26.1.5+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 // indirect diff --git a/go.sum b/go.sum index db8978e873..4cf70280e0 100644 --- a/go.sum +++ b/go.sum @@ -447,8 +447,8 @@ github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU= -github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= +github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= diff --git a/plugins/outputs/cloudwatch/convert_otel.go b/plugins/outputs/cloudwatch/convert_otel.go index a318eaa265..a438bc0890 100644 --- a/plugins/outputs/cloudwatch/convert_otel.go +++ b/plugins/outputs/cloudwatch/convert_otel.go @@ -170,7 +170,7 @@ func ConvertOtelMetric(m pmetric.Metric, entity cloudwatch.Entity) []*aggregatio func ConvertOtelMetrics(m pmetric.Metrics) []*aggregationDatum { datums := make([]*aggregationDatum, 0, m.DataPointCount()) for i := 0; i < m.ResourceMetrics().Len(); i++ { - entity := fetchEntityFields(m.ResourceMetrics().At(i).Resource().Attributes()) + entity := entityattributes.CreateCloudWatchEntityFromAttributes(m.ResourceMetrics().At(i).Resource().Attributes()) scopeMetrics := m.ResourceMetrics().At(i).ScopeMetrics() for j := 0; j < scopeMetrics.Len(); j++ { metrics := scopeMetrics.At(j).Metrics() @@ -184,40 +184,3 @@ func ConvertOtelMetrics(m pmetric.Metrics) []*aggregationDatum { } return datums } - -func fetchEntityFields(resourceAttributes pcommon.Map) cloudwatch.Entity { - keyAttributesMap := map[string]*string{} - attributeMap := map[string]*string{} - platformType := "" - if platformTypeValue, ok := resourceAttributes.Get(entityattributes.AttributeEntityPlatformType); ok { - platformType = platformTypeValue.Str() - } - processEntityAttributes(entityattributes.GetKeyAttributeEntityShortNameMap(), keyAttributesMap, resourceAttributes) - processEntityAttributes(entityattributes.GetAttributeEntityShortNameMap(platformType), attributeMap, resourceAttributes) - removeEntityFields(resourceAttributes) - if _, ok := keyAttributesMap[entityattributes.AwsAccountId]; !ok { - return cloudwatch.Entity{} - } - return cloudwatch.Entity{ - KeyAttributes: keyAttributesMap, - Attributes: attributeMap, - } -} - -// processEntityAttributes fetches the fields with entity prefix and creates an entity to be sent at the PutMetricData call. -func processEntityAttributes(entityMap map[string]string, targetMap map[string]*string, mutableResourceAttributes pcommon.Map) { - for entityField, shortName := range entityMap { - if val, ok := mutableResourceAttributes.Get(entityField); ok { - if strVal := val.Str(); strVal != "" { - targetMap[shortName] = aws.String(strVal) - } - } - } -} - -// removeEntityFields so that it is not tagged as a dimension, and reduces the size of the PMD payload. -func removeEntityFields(mutableResourceAttributes pcommon.Map) { - mutableResourceAttributes.RemoveIf(func(s string, _ pcommon.Value) bool { - return strings.HasPrefix(s, entityattributes.AWSEntityPrefix) - }) -} diff --git a/plugins/outputs/cloudwatch/convert_otel_test.go b/plugins/outputs/cloudwatch/convert_otel_test.go index 458d72de3a..1525fab186 100644 --- a/plugins/outputs/cloudwatch/convert_otel_test.go +++ b/plugins/outputs/cloudwatch/convert_otel_test.go @@ -242,256 +242,6 @@ func TestConvertOtelMetrics_Entity(t *testing.T) { } -func TestProcessAndRemoveEntityAttributes(t *testing.T) { - testCases := []struct { - name string - resourceAttributes map[string]any - wantedAttributes map[string]*string - leftoverAttributes map[string]any - }{ - { - name: "key_attributes", - resourceAttributes: map[string]any{ - entityattributes.AttributeEntityServiceName: "my-service", - entityattributes.AttributeEntityDeploymentEnvironment: "my-environment", - }, - wantedAttributes: map[string]*string{ - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - }, - leftoverAttributes: make(map[string]any), - }, - { - name: "non-key_attributes", - resourceAttributes: map[string]any{ - entityattributes.AttributeEntityCluster: "my-cluster", - entityattributes.AttributeEntityNamespace: "my-namespace", - entityattributes.AttributeEntityNode: "my-node", - entityattributes.AttributeEntityWorkload: "my-workload", - entityattributes.AttributeEntityPlatformType: "AWS::EKS", - }, - wantedAttributes: map[string]*string{ - entityattributes.EksCluster: aws.String("my-cluster"), - entityattributes.NamespaceField: aws.String("my-namespace"), - entityattributes.Node: aws.String("my-node"), - entityattributes.Workload: aws.String("my-workload"), - entityattributes.Platform: aws.String("AWS::EKS"), - }, - leftoverAttributes: make(map[string]any), - }, - { - name: "key_and_non_key_attributes", - resourceAttributes: map[string]any{ - entityattributes.AttributeEntityServiceName: "my-service", - entityattributes.AttributeEntityDeploymentEnvironment: "my-environment", - entityattributes.AttributeEntityCluster: "my-cluster", - entityattributes.AttributeEntityNamespace: "my-namespace", - entityattributes.AttributeEntityNode: "my-node", - entityattributes.AttributeEntityWorkload: "my-workload", - entityattributes.AttributeEntityPlatformType: "K8s", - }, - wantedAttributes: map[string]*string{ - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.K8sCluster: aws.String("my-cluster"), - entityattributes.NamespaceField: aws.String("my-namespace"), - entityattributes.Node: aws.String("my-node"), - entityattributes.Workload: aws.String("my-workload"), - entityattributes.Platform: aws.String("K8s"), - }, - leftoverAttributes: make(map[string]any), - }, - { - name: "key_and_non_key_attributes_plus_extras", - resourceAttributes: map[string]any{ - "extra_attribute": "extra_value", - entityattributes.AttributeEntityServiceName: "my-service", - entityattributes.AttributeEntityDeploymentEnvironment: "my-environment", - entityattributes.AttributeEntityCluster: "my-cluster", - entityattributes.AttributeEntityNamespace: "my-namespace", - entityattributes.AttributeEntityNode: "my-node", - entityattributes.AttributeEntityWorkload: "my-workload", - entityattributes.AttributeEntityPlatformType: "K8s", - }, - wantedAttributes: map[string]*string{ - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.K8sCluster: aws.String("my-cluster"), - entityattributes.NamespaceField: aws.String("my-namespace"), - entityattributes.Node: aws.String("my-node"), - entityattributes.Workload: aws.String("my-workload"), - entityattributes.Platform: aws.String("K8s"), - }, - leftoverAttributes: map[string]any{ - "extra_attribute": "extra_value", - }, - }, - { - name: "key_and_non_key_attributes_plus_unsupported_entity_field", - resourceAttributes: map[string]any{ - entityattributes.AWSEntityPrefix + "not.real.values": "unsupported", - entityattributes.AttributeEntityServiceName: "my-service", - entityattributes.AttributeEntityDeploymentEnvironment: "my-environment", - entityattributes.AttributeEntityCluster: "my-cluster", - entityattributes.AttributeEntityNamespace: "my-namespace", - entityattributes.AttributeEntityNode: "my-node", - entityattributes.AttributeEntityWorkload: "my-workload", - entityattributes.AttributeEntityPlatformType: "AWS::EKS", - }, - wantedAttributes: map[string]*string{ - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.EksCluster: aws.String("my-cluster"), - entityattributes.NamespaceField: aws.String("my-namespace"), - entityattributes.Node: aws.String("my-node"), - entityattributes.Workload: aws.String("my-workload"), - entityattributes.Platform: aws.String("AWS::EKS"), - }, - leftoverAttributes: map[string]any{}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - attrs := pcommon.NewMap() - err := attrs.FromRaw(tc.resourceAttributes) - - // resetting fields for current test case - entityAttrMap := []map[string]string{entityattributes.GetKeyAttributeEntityShortNameMap()} - platformType := "" - if platformTypeValue, ok := attrs.Get(entityattributes.AttributeEntityPlatformType); ok { - platformType = platformTypeValue.Str() - } - if platformType != "" { - delete(entityattributes.GetAttributeEntityShortNameMap(platformType), entityattributes.AttributeEntityCluster) - entityAttrMap = append(entityAttrMap, entityattributes.GetAttributeEntityShortNameMap(platformType)) - } - assert.Nil(t, err) - targetMap := make(map[string]*string) - for _, entityMap := range entityAttrMap { - processEntityAttributes(entityMap, targetMap, attrs) - } - removeEntityFields(attrs) - assert.Equal(t, tc.leftoverAttributes, attrs.AsRaw()) - assert.Equal(t, tc.wantedAttributes, targetMap) - }) - } -} - -func TestFetchEntityFields_WithoutAccountID(t *testing.T) { - resourceMetrics := pmetric.NewResourceMetrics() - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNode, "my-node") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityCluster, "my-cluster") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EKS") - assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) - - expectedEntity := cloudwatch.Entity{ - KeyAttributes: nil, - Attributes: nil, - } - entity := fetchEntityFields(resourceMetrics.Resource().Attributes()) - assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) - assert.Equal(t, expectedEntity, entity) -} - -func TestFetchEntityFields_WithAccountID(t *testing.T) { - resourceMetrics := pmetric.NewResourceMetrics() - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNode, "my-node") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityCluster, "my-cluster") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EKS") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") - assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) - - expectedEntity := cloudwatch.Entity{ - KeyAttributes: map[string]*string{ - entityattributes.EntityType: aws.String(entityattributes.Service), - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.AwsAccountId: aws.String("123456789"), - }, - Attributes: map[string]*string{ - entityattributes.Node: aws.String("my-node"), - entityattributes.EksCluster: aws.String("my-cluster"), - entityattributes.NamespaceField: aws.String("my-namespace"), - entityattributes.Workload: aws.String("my-workload"), - entityattributes.Platform: aws.String("AWS::EKS"), - }, - } - entity := fetchEntityFields(resourceMetrics.Resource().Attributes()) - assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) - assert.Equal(t, expectedEntity, entity) -} - -func TestFetchEntityFieldsOnK8s(t *testing.T) { - entityMap := entityattributes.GetAttributeEntityShortNameMap("") - delete(entityMap, entityattributes.AttributeEntityCluster) - resourceMetrics := pmetric.NewResourceMetrics() - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNode, "my-node") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityCluster, "my-cluster") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "K8s") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") - assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) - - expectedEntity := cloudwatch.Entity{ - KeyAttributes: map[string]*string{ - entityattributes.EntityType: aws.String(entityattributes.Service), - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.AwsAccountId: aws.String("123456789"), - }, - Attributes: map[string]*string{ - entityattributes.Node: aws.String("my-node"), - entityattributes.K8sCluster: aws.String("my-cluster"), - entityattributes.NamespaceField: aws.String("my-namespace"), - entityattributes.Workload: aws.String("my-workload"), - entityattributes.Platform: aws.String("K8s"), - }, - } - entity := fetchEntityFields(resourceMetrics.Resource().Attributes()) - assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) - assert.Equal(t, expectedEntity, entity) -} - -func TestFetchEntityFieldsOnEc2(t *testing.T) { - resourceMetrics := pmetric.NewResourceMetrics() - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EC2") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") - assert.Equal(t, 5, resourceMetrics.Resource().Attributes().Len()) - - expectedEntity := cloudwatch.Entity{ - KeyAttributes: map[string]*string{ - entityattributes.EntityType: aws.String(entityattributes.Service), - entityattributes.ServiceName: aws.String("my-service"), - entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.AwsAccountId: aws.String("123456789"), - }, - Attributes: map[string]*string{ - entityattributes.Platform: aws.String("AWS::EC2"), - }, - } - entity := fetchEntityFields(resourceMetrics.Resource().Attributes()) - assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) - assert.Equal(t, expectedEntity, entity) -} - func TestInvalidMetric(t *testing.T) { m := pmetric.NewMetric() m.SetName("name") diff --git a/plugins/processors/awsentity/entityattributes/entityattributes.go b/plugins/processors/awsentity/entityattributes/entityattributes.go index 9638af83a5..4588695c41 100644 --- a/plugins/processors/awsentity/entityattributes/entityattributes.go +++ b/plugins/processors/awsentity/entityattributes/entityattributes.go @@ -3,6 +3,15 @@ package entityattributes +import ( + "strings" + + "github.com/aws/aws-sdk-go/aws" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/sdk/service/cloudwatch" +) + const ( // The following are the possible values for EntityType config options @@ -80,16 +89,42 @@ var attributeEntityToShortNameMap = map[string]string{ AttributeEntityServiceNameSource: ServiceNameSource, } -func GetKeyAttributeEntityShortNameMap() map[string]string { - return keyAttributeEntityToShortNameMap +func CreateCloudWatchEntityFromAttributes(resourceAttributes pcommon.Map) cloudwatch.Entity { + keyAttributesMap := map[string]*string{} + attributeMap := map[string]*string{} + + // Process KeyAttributes and return empty entity if AwsAccountId is not found + processEntityAttributes(keyAttributeEntityToShortNameMap, keyAttributesMap, resourceAttributes) + if _, ok := keyAttributesMap[AwsAccountId]; !ok { + return cloudwatch.Entity{} + } + + // Process Attributes and add cluster attribute if on EKS/K8s + processEntityAttributes(attributeEntityToShortNameMap, attributeMap, resourceAttributes) + if platformTypeValue, ok := resourceAttributes.Get(AttributeEntityPlatformType); ok { + platformType := clusterType(platformTypeValue.Str()) + if clusterNameValue, ok := resourceAttributes.Get(AttributeEntityCluster); ok { + attributeMap[platformType] = aws.String(clusterNameValue.Str()) + } + } + + // Remove entity fields from attributes and return the entity + removeEntityFields(resourceAttributes) + return cloudwatch.Entity{ + KeyAttributes: keyAttributesMap, + Attributes: attributeMap, + } } -// Cluster attribute prefix could be either EKS or K8s. We set the field once at runtime. -func GetAttributeEntityShortNameMap(platformType string) map[string]string { - if _, ok := attributeEntityToShortNameMap[AttributeEntityCluster]; !ok { - attributeEntityToShortNameMap[AttributeEntityCluster] = clusterType(platformType) +// processEntityAttributes fetches the fields with entity prefix and creates an entity to be sent at the PutMetricData call. +func processEntityAttributes(entityMap map[string]string, targetMap map[string]*string, incomingResourceAttributes pcommon.Map) { + for entityField, shortName := range entityMap { + if val, ok := incomingResourceAttributes.Get(entityField); ok { + if strVal := val.Str(); strVal != "" { + targetMap[shortName] = aws.String(strVal) + } + } } - return attributeEntityToShortNameMap } func clusterType(platformType string) string { @@ -100,3 +135,10 @@ func clusterType(platformType string) string { } return "" } + +// removeEntityFields so that it is not tagged as a dimension, and reduces the size of the PMD payload. +func removeEntityFields(mutableResourceAttributes pcommon.Map) { + mutableResourceAttributes.RemoveIf(func(s string, _ pcommon.Value) bool { + return strings.HasPrefix(s, AWSEntityPrefix) + }) +} diff --git a/plugins/processors/awsentity/entityattributes/entityattributes_test.go b/plugins/processors/awsentity/entityattributes/entityattributes_test.go new file mode 100644 index 0000000000..8e5cca9db0 --- /dev/null +++ b/plugins/processors/awsentity/entityattributes/entityattributes_test.go @@ -0,0 +1,257 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package entityattributes + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/aws/amazon-cloudwatch-agent/sdk/service/cloudwatch" +) + +func TestProcessAndRemoveEntityAttributes(t *testing.T) { + testCases := []struct { + name string + resourceAttributes map[string]any + wantedAttributes map[string]*string + leftoverAttributes map[string]any + }{ + { + name: "key_attributes", + resourceAttributes: map[string]any{ + AttributeEntityServiceName: "my-service", + AttributeEntityDeploymentEnvironment: "my-environment", + }, + wantedAttributes: map[string]*string{ + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + }, + leftoverAttributes: make(map[string]any), + }, + { + name: "non-key_attributes", + resourceAttributes: map[string]any{ + AttributeEntityNamespace: "my-namespace", + AttributeEntityNode: "my-node", + AttributeEntityWorkload: "my-workload", + AttributeEntityPlatformType: "AWS::EKS", + }, + wantedAttributes: map[string]*string{ + NamespaceField: aws.String("my-namespace"), + Node: aws.String("my-node"), + Workload: aws.String("my-workload"), + Platform: aws.String("AWS::EKS"), + }, + leftoverAttributes: make(map[string]any), + }, + { + name: "key_and_non_key_attributes", + resourceAttributes: map[string]any{ + AttributeEntityServiceName: "my-service", + AttributeEntityDeploymentEnvironment: "my-environment", + AttributeEntityNamespace: "my-namespace", + AttributeEntityNode: "my-node", + AttributeEntityWorkload: "my-workload", + AttributeEntityPlatformType: "K8s", + }, + wantedAttributes: map[string]*string{ + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + NamespaceField: aws.String("my-namespace"), + Node: aws.String("my-node"), + Workload: aws.String("my-workload"), + Platform: aws.String("K8s"), + }, + leftoverAttributes: make(map[string]any), + }, + { + name: "key_and_non_key_attributes_plus_extras", + resourceAttributes: map[string]any{ + "extra_attribute": "extra_value", + AttributeEntityServiceName: "my-service", + AttributeEntityDeploymentEnvironment: "my-environment", + AttributeEntityNamespace: "my-namespace", + AttributeEntityNode: "my-node", + AttributeEntityWorkload: "my-workload", + AttributeEntityPlatformType: "K8s", + }, + wantedAttributes: map[string]*string{ + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + NamespaceField: aws.String("my-namespace"), + Node: aws.String("my-node"), + Workload: aws.String("my-workload"), + Platform: aws.String("K8s"), + }, + leftoverAttributes: map[string]any{ + "extra_attribute": "extra_value", + }, + }, + { + name: "key_and_non_key_attributes_plus_unsupported_entity_field", + resourceAttributes: map[string]any{ + AWSEntityPrefix + "not.real.values": "unsupported", + AttributeEntityServiceName: "my-service", + AttributeEntityDeploymentEnvironment: "my-environment", + AttributeEntityNamespace: "my-namespace", + AttributeEntityNode: "my-node", + AttributeEntityWorkload: "my-workload", + AttributeEntityPlatformType: "AWS::EKS", + }, + wantedAttributes: map[string]*string{ + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + NamespaceField: aws.String("my-namespace"), + Node: aws.String("my-node"), + Workload: aws.String("my-workload"), + Platform: aws.String("AWS::EKS"), + }, + leftoverAttributes: map[string]any{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + attrs := pcommon.NewMap() + err := attrs.FromRaw(tc.resourceAttributes) + + // resetting fields for current test case + entityAttrMap := []map[string]string{keyAttributeEntityToShortNameMap} + platformType := "" + if platformTypeValue, ok := attrs.Get(AttributeEntityPlatformType); ok { + platformType = platformTypeValue.Str() + } + if platformType != "" { + delete(attributeEntityToShortNameMap, AttributeEntityCluster) + entityAttrMap = append(entityAttrMap, attributeEntityToShortNameMap) + } + assert.Nil(t, err) + targetMap := make(map[string]*string) + for _, entityMap := range entityAttrMap { + processEntityAttributes(entityMap, targetMap, attrs) + } + removeEntityFields(attrs) + assert.Equal(t, tc.leftoverAttributes, attrs.AsRaw()) + assert.Equal(t, tc.wantedAttributes, targetMap) + }) + } +} + +func TestCreateCloudWatchEntityFromAttributes_WithoutAccountID(t *testing.T) { + resourceMetrics := pmetric.NewResourceMetrics() + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityType, "Service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityDeploymentEnvironment, "my-environment") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityServiceName, "my-service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityNode, "my-node") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityCluster, "my-cluster") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityNamespace, "my-namespace") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityWorkload, "my-workload") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityPlatformType, "AWS::EKS") + assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) + + expectedEntity := cloudwatch.Entity{ + KeyAttributes: nil, + Attributes: nil, + } + entity := CreateCloudWatchEntityFromAttributes(resourceMetrics.Resource().Attributes()) + assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, expectedEntity, entity) +} + +func TestCreateCloudWatchEntityFromAttributes_WithAccountID(t *testing.T) { + resourceMetrics := pmetric.NewResourceMetrics() + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityType, "Service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityDeploymentEnvironment, "my-environment") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityServiceName, "my-service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityNode, "my-node") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityCluster, "my-cluster") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityNamespace, "my-namespace") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityWorkload, "my-workload") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityPlatformType, "AWS::EKS") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityAwsAccountId, "123456789") + assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) + + expectedEntity := cloudwatch.Entity{ + KeyAttributes: map[string]*string{ + EntityType: aws.String(Service), + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + AwsAccountId: aws.String("123456789"), + }, + Attributes: map[string]*string{ + Node: aws.String("my-node"), + EksCluster: aws.String("my-cluster"), + NamespaceField: aws.String("my-namespace"), + Workload: aws.String("my-workload"), + Platform: aws.String("AWS::EKS"), + }, + } + entity := CreateCloudWatchEntityFromAttributes(resourceMetrics.Resource().Attributes()) + assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, expectedEntity, entity) +} + +func TestCreateCloudWatchEntityFromAttributesOnK8s(t *testing.T) { + entityMap := attributeEntityToShortNameMap + delete(entityMap, AttributeEntityCluster) + resourceMetrics := pmetric.NewResourceMetrics() + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityType, "Service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityDeploymentEnvironment, "my-environment") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityServiceName, "my-service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityNode, "my-node") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityCluster, "my-cluster") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityNamespace, "my-namespace") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityWorkload, "my-workload") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityPlatformType, "K8s") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityAwsAccountId, "123456789") + assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) + + expectedEntity := cloudwatch.Entity{ + KeyAttributes: map[string]*string{ + EntityType: aws.String(Service), + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + AwsAccountId: aws.String("123456789"), + }, + Attributes: map[string]*string{ + Node: aws.String("my-node"), + K8sCluster: aws.String("my-cluster"), + NamespaceField: aws.String("my-namespace"), + Workload: aws.String("my-workload"), + Platform: aws.String("K8s"), + }, + } + entity := CreateCloudWatchEntityFromAttributes(resourceMetrics.Resource().Attributes()) + assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, expectedEntity, entity) +} + +func TestCreateCloudWatchEntityFromAttributesOnEc2(t *testing.T) { + resourceMetrics := pmetric.NewResourceMetrics() + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityType, "Service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityDeploymentEnvironment, "my-environment") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityServiceName, "my-service") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityPlatformType, "AWS::EC2") + resourceMetrics.Resource().Attributes().PutStr(AttributeEntityAwsAccountId, "123456789") + assert.Equal(t, 5, resourceMetrics.Resource().Attributes().Len()) + + expectedEntity := cloudwatch.Entity{ + KeyAttributes: map[string]*string{ + EntityType: aws.String(Service), + ServiceName: aws.String("my-service"), + DeploymentEnvironment: aws.String("my-environment"), + AwsAccountId: aws.String("123456789"), + }, + Attributes: map[string]*string{ + Platform: aws.String("AWS::EC2"), + }, + } + entity := CreateCloudWatchEntityFromAttributes(resourceMetrics.Resource().Attributes()) + assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, expectedEntity, entity) +} diff --git a/translator/translate/otel/pipeline/applicationsignals/translator.go b/translator/translate/otel/pipeline/applicationsignals/translator.go index 9540c9844e..239823937c 100644 --- a/translator/translate/otel/pipeline/applicationsignals/translator.go +++ b/translator/translate/otel/pipeline/applicationsignals/translator.go @@ -60,11 +60,12 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } mode := context.CurrentContext().KubernetesMode() + translators.Processors.Set(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType))) + translators.Processors.Set(awsapplicationsignals.NewTranslator(awsapplicationsignals.WithDataType(t.dataType))) + if t.dataType == component.DataTypeMetrics && mode != "" { translators.Processors.Set(awsentity.NewTranslatorWithEntityType(awsentity.Service, common.AppSignals, false)) } - translators.Processors.Set(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType))) - translators.Processors.Set(awsapplicationsignals.NewTranslator(awsapplicationsignals.WithDataType(t.dataType))) if enabled, _ := common.GetBool(conf, common.AgentDebugConfigKey); enabled { translators.Exporters.Set(debug.NewTranslator(common.WithName(common.AppSignals))) diff --git a/translator/translate/otel/pipeline/applicationsignals/translator_test.go b/translator/translate/otel/pipeline/applicationsignals/translator_test.go index 23119032eb..a28c93b19e 100644 --- a/translator/translate/otel/pipeline/applicationsignals/translator_test.go +++ b/translator/translate/otel/pipeline/applicationsignals/translator_test.go @@ -126,7 +126,7 @@ func TestTranslatorMetricsForKubernetes(t *testing.T) { }, want: &want{ receivers: []string{"otlp/application_signals"}, - processors: []string{"metricstransform/application_signals", "awsentity/service/application_signals", "resourcedetection", "awsapplicationsignals"}, + processors: []string{"metricstransform/application_signals", "resourcedetection", "awsapplicationsignals", "awsentity/service/application_signals"}, exporters: []string{"awsemf/application_signals"}, extensions: []string{"agenthealth/logs"}, }, @@ -147,7 +147,7 @@ func TestTranslatorMetricsForKubernetes(t *testing.T) { }, want: &want{ receivers: []string{"otlp/application_signals"}, - processors: []string{"metricstransform/application_signals", "awsentity/service/application_signals", "resourcedetection", "awsapplicationsignals"}, + processors: []string{"metricstransform/application_signals", "resourcedetection", "awsapplicationsignals", "awsentity/service/application_signals"}, exporters: []string{"debug/application_signals", "awsemf/application_signals"}, extensions: []string{"agenthealth/logs"}, }, @@ -165,7 +165,7 @@ func TestTranslatorMetricsForKubernetes(t *testing.T) { }, want: &want{ receivers: []string{"otlp/application_signals"}, - processors: []string{"metricstransform/application_signals", "awsentity/service/application_signals", "resourcedetection", "awsapplicationsignals"}, + processors: []string{"metricstransform/application_signals", "resourcedetection", "awsapplicationsignals", "awsentity/service/application_signals"}, exporters: []string{"awsemf/application_signals"}, extensions: []string{"agenthealth/logs"}, },