From 07cd92aa46a33849100738a2c3ae881c34c51a7d Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 30 Oct 2024 12:41:20 -0400 Subject: [PATCH 1/2] Revert "Drop entity if account ID is not present" This reverts commit 5c6298d46df11d8e15ec4a270c67417bed3ea8fd. --- extension/entitystore/extension.go | 18 +++--- extension/entitystore/extension_test.go | 38 ++++++++---- plugins/outputs/cloudwatch/convert_otel.go | 4 +- .../outputs/cloudwatch/convert_otel_test.go | 33 +--------- plugins/outputs/cloudwatchlogs/pusher.go | 9 ++- plugins/outputs/cloudwatchlogs/pusher_test.go | 61 ++----------------- 6 files changed, 51 insertions(+), 112 deletions(-) diff --git a/extension/entitystore/extension.go b/extension/entitystore/extension.go index bd27012a3c..d9058f9f78 100644 --- a/extension/entitystore/extension.go +++ b/extension/entitystore/extension.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/sts/stsiface" "github.com/jellydator/ttlcache/v3" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/extension" @@ -19,7 +20,6 @@ import ( configaws "github.com/aws/amazon-cloudwatch-agent/cfg/aws" "github.com/aws/amazon-cloudwatch-agent/internal/ec2metadataprovider" "github.com/aws/amazon-cloudwatch-agent/internal/retryer" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/entityattributes" "github.com/aws/amazon-cloudwatch-agent/sdk/service/cloudwatchlogs" "github.com/aws/amazon-cloudwatch-agent/translator/config" ) @@ -31,6 +31,9 @@ const ( ServiceNameSourceKey = "AWS.ServiceNameSource" PlatformType = "PlatformType" EC2PlatForm = "AWS::EC2" + Type = "Type" + Name = "Name" + Environment = "Environment" podTerminationCheckInterval = 5 * time.Minute ) @@ -71,6 +74,8 @@ type EntityStore struct { metadataprovider ec2metadataprovider.MetadataProvider + stsClient stsiface.STSAPI + podTerminationCheckInterval time.Duration } @@ -144,9 +149,7 @@ func (e *EntityStore) CreateLogFileEntity(logFileGlob LogFileGlob, logGroupName keyAttributes := e.createServiceKeyAttributes(serviceAttr) attributeMap := e.createAttributeMap() addNonEmptyToMap(attributeMap, ServiceNameSourceKey, serviceAttr.ServiceNameSource) - if _, ok := keyAttributes[entityattributes.AwsAccountId]; !ok { - return nil - } + return &cloudwatchlogs.Entity{ KeyAttributes: keyAttributes, Attributes: attributeMap, @@ -222,11 +225,10 @@ func (e *EntityStore) createAttributeMap() map[string]*string { // createServiceKeyAttribute creates KeyAttributes for Service entities func (e *EntityStore) createServiceKeyAttributes(serviceAttr ServiceAttribute) map[string]*string { serviceKeyAttr := map[string]*string{ - entityattributes.EntityType: aws.String(Service), + Type: aws.String(Service), } - addNonEmptyToMap(serviceKeyAttr, entityattributes.ServiceName, serviceAttr.ServiceName) - addNonEmptyToMap(serviceKeyAttr, entityattributes.DeploymentEnvironment, serviceAttr.Environment) - addNonEmptyToMap(serviceKeyAttr, entityattributes.AwsAccountId, e.ec2Info.GetAccountID()) + addNonEmptyToMap(serviceKeyAttr, Name, serviceAttr.ServiceName) + addNonEmptyToMap(serviceKeyAttr, Environment, serviceAttr.Environment) return serviceKeyAttr } diff --git a/extension/entitystore/extension_test.go b/extension/entitystore/extension_test.go index fea148847a..668c8bd1a4 100644 --- a/extension/entitystore/extension_test.go +++ b/extension/entitystore/extension_test.go @@ -15,6 +15,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go/service/sts/stsiface" "github.com/jellydator/ttlcache/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -22,7 +24,6 @@ import ( "go.uber.org/zap/zapcore" "github.com/aws/amazon-cloudwatch-agent/internal/ec2metadataprovider" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/entityattributes" "github.com/aws/amazon-cloudwatch-agent/sdk/service/cloudwatchlogs" "github.com/aws/amazon-cloudwatch-agent/translator/config" ) @@ -63,6 +64,15 @@ func (s *mockServiceProvider) getServiceNameAndSource() (string, string) { return "test-service-name", "UserConfiguration" } +type mockSTSClient struct { + stsiface.STSAPI + accountId string +} + +func (ms *mockSTSClient) GetCallerIdentity(*sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) { + return &sts.GetCallerIdentityOutput{Account: aws.String(ms.accountId)}, nil +} + type mockMetadataProvider struct { InstanceIdentityDocument *ec2metadata.EC2InstanceIdentityDocument Tags map[string]string @@ -278,25 +288,25 @@ func TestEntityStore_createServiceKeyAttributes(t *testing.T) { name: "NameAndEnvironmentSet", serviceAttr: ServiceAttribute{ServiceName: "test-service", Environment: "test-environment"}, want: map[string]*string{ - entityattributes.DeploymentEnvironment: aws.String("test-environment"), - entityattributes.ServiceName: aws.String("test-service"), - entityattributes.EntityType: aws.String(Service), + Environment: aws.String("test-environment"), + Name: aws.String("test-service"), + Type: aws.String(Service), }, }, { name: "OnlyNameSet", serviceAttr: ServiceAttribute{ServiceName: "test-service"}, want: map[string]*string{ - entityattributes.ServiceName: aws.String("test-service"), - entityattributes.EntityType: aws.String(Service), + Name: aws.String("test-service"), + Type: aws.String(Service), }, }, { name: "OnlyEnvironmentSet", serviceAttr: ServiceAttribute{Environment: "test-environment"}, want: map[string]*string{ - entityattributes.DeploymentEnvironment: aws.String("test-environment"), - entityattributes.EntityType: aws.String(Service), + Environment: aws.String("test-environment"), + Type: aws.String(Service), }, }, } @@ -322,8 +332,10 @@ func TestEntityStore_createLogFileRID(t *testing.T) { sp.On("logFileServiceAttribute", glob, group).Return(serviceAttr) e := EntityStore{ mode: config.ModeEC2, - ec2Info: EC2Info{InstanceID: instanceId, AccountID: accountId}, + ec2Info: EC2Info{InstanceID: instanceId}, serviceprovider: sp, + metadataprovider: mockMetadataProviderWithAccountId(accountId), + stsClient: &mockSTSClient{accountId: accountId}, nativeCredential: &session.Session{}, } @@ -331,10 +343,9 @@ func TestEntityStore_createLogFileRID(t *testing.T) { expectedEntity := cloudwatchlogs.Entity{ KeyAttributes: map[string]*string{ - entityattributes.DeploymentEnvironment: aws.String("test-environment"), - entityattributes.ServiceName: aws.String("test-service"), - entityattributes.EntityType: aws.String(Service), - entityattributes.AwsAccountId: aws.String(accountId), + Environment: aws.String("test-environment"), + Name: aws.String("test-service"), + Type: aws.String(Service), }, Attributes: map[string]*string{ InstanceIDKey: aws.String(instanceId), @@ -532,6 +543,7 @@ func TestEntityStore_GetMetricServiceNameSource(t *testing.T) { ec2Info: EC2Info{InstanceID: instanceId}, serviceprovider: sp, metadataprovider: mockMetadataProviderWithAccountId(accountId), + stsClient: &mockSTSClient{accountId: accountId}, nativeCredential: &session.Session{}, } diff --git a/plugins/outputs/cloudwatch/convert_otel.go b/plugins/outputs/cloudwatch/convert_otel.go index a318eaa265..d91ce92c61 100644 --- a/plugins/outputs/cloudwatch/convert_otel.go +++ b/plugins/outputs/cloudwatch/convert_otel.go @@ -195,9 +195,7 @@ func fetchEntityFields(resourceAttributes pcommon.Map) cloudwatch.Entity { processEntityAttributes(entityattributes.GetKeyAttributeEntityShortNameMap(), keyAttributesMap, resourceAttributes) processEntityAttributes(entityattributes.GetAttributeEntityShortNameMap(platformType), attributeMap, resourceAttributes) removeEntityFields(resourceAttributes) - if _, ok := keyAttributesMap[entityattributes.AwsAccountId]; !ok { - return cloudwatch.Entity{} - } + return cloudwatch.Entity{ KeyAttributes: keyAttributesMap, Attributes: attributeMap, diff --git a/plugins/outputs/cloudwatch/convert_otel_test.go b/plugins/outputs/cloudwatch/convert_otel_test.go index 458d72de3a..ae4a825f5b 100644 --- a/plugins/outputs/cloudwatch/convert_otel_test.go +++ b/plugins/outputs/cloudwatch/convert_otel_test.go @@ -378,7 +378,7 @@ func TestProcessAndRemoveEntityAttributes(t *testing.T) { } } -func TestFetchEntityFields_WithoutAccountID(t *testing.T) { +func TestFetchEntityFields(t *testing.T) { resourceMetrics := pmetric.NewResourceMetrics() resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") @@ -390,34 +390,11 @@ func TestFetchEntityFields_WithoutAccountID(t *testing.T) { resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EKS") assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) - expectedEntity := cloudwatch.Entity{ - KeyAttributes: nil, - Attributes: nil, - } - entity := fetchEntityFields(resourceMetrics.Resource().Attributes()) - assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) - assert.Equal(t, expectedEntity, entity) -} - -func TestFetchEntityFields_WithAccountID(t *testing.T) { - resourceMetrics := pmetric.NewResourceMetrics() - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNode, "my-node") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityCluster, "my-cluster") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EKS") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") - assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) - expectedEntity := cloudwatch.Entity{ KeyAttributes: map[string]*string{ entityattributes.EntityType: aws.String(entityattributes.Service), entityattributes.ServiceName: aws.String("my-service"), entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.AwsAccountId: aws.String("123456789"), }, Attributes: map[string]*string{ entityattributes.Node: aws.String("my-node"), @@ -444,15 +421,13 @@ func TestFetchEntityFieldsOnK8s(t *testing.T) { resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "K8s") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") - assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) expectedEntity := cloudwatch.Entity{ KeyAttributes: map[string]*string{ entityattributes.EntityType: aws.String(entityattributes.Service), entityattributes.ServiceName: aws.String("my-service"), entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.AwsAccountId: aws.String("123456789"), }, Attributes: map[string]*string{ entityattributes.Node: aws.String("my-node"), @@ -473,15 +448,13 @@ func TestFetchEntityFieldsOnEc2(t *testing.T) { resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EC2") - resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") - assert.Equal(t, 5, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, 4, resourceMetrics.Resource().Attributes().Len()) expectedEntity := cloudwatch.Entity{ KeyAttributes: map[string]*string{ entityattributes.EntityType: aws.String(entityattributes.Service), entityattributes.ServiceName: aws.String("my-service"), entityattributes.DeploymentEnvironment: aws.String("my-environment"), - entityattributes.AwsAccountId: aws.String("123456789"), }, Attributes: map[string]*string{ entityattributes.Platform: aws.String("AWS::EC2"), diff --git a/plugins/outputs/cloudwatchlogs/pusher.go b/plugins/outputs/cloudwatchlogs/pusher.go index ff0bb1dd0e..2c1036f1e8 100644 --- a/plugins/outputs/cloudwatchlogs/pusher.go +++ b/plugins/outputs/cloudwatchlogs/pusher.go @@ -222,15 +222,18 @@ func (p *pusher) send() { if p.needSort { sort.Stable(ByTimestamp(p.events)) } + var entity *cloudwatchlogs.Entity + if p.logSrc != nil { + entity = p.logSrc.Entity() + } + input := &cloudwatchlogs.PutLogEventsInput{ LogEvents: p.events, LogGroupName: &p.Group, LogStreamName: &p.Stream, SequenceToken: p.sequenceToken, } - if p.logSrc != nil { - input.Entity = p.logSrc.Entity() - } + input.Entity = entity startTime := time.Now() diff --git a/plugins/outputs/cloudwatchlogs/pusher_test.go b/plugins/outputs/cloudwatchlogs/pusher_test.go index fc3957e5cc..f97b9f22ec 100644 --- a/plugins/outputs/cloudwatchlogs/pusher_test.go +++ b/plugins/outputs/cloudwatchlogs/pusher_test.go @@ -27,26 +27,20 @@ import ( type mockLogSrc struct { logs.LogSrc - returnEmpty bool } func (m *mockLogSrc) Entity() *cloudwatchlogs.Entity { - entity := &cloudwatchlogs.Entity{ + return &cloudwatchlogs.Entity{ Attributes: map[string]*string{ "PlatformType": aws.String("AWS::EC2"), "EC2.InstanceId": aws.String("i-123456789"), "EC2.AutoScalingGroup": aws.String("test-group"), }, KeyAttributes: map[string]*string{ - "Name": aws.String("myService"), - "Environment": aws.String("myEnvironment"), - "AwsAccountId": aws.String("123456789"), + "Name": aws.String("myService"), + "Environment": aws.String("myEnvironment"), }, } - if m.returnEmpty { - return nil - } - return entity } var wg sync.WaitGroup @@ -109,7 +103,7 @@ func (e evtMock) Done() { } } -func TestAddSingleEvent_WithAccountId(t *testing.T) { +func TestAddSingleEvent(t *testing.T) { var s svcMock called := false nst := "NEXT_SEQ_TOKEN" @@ -120,9 +114,8 @@ func TestAddSingleEvent_WithAccountId(t *testing.T) { "EC2.AutoScalingGroup": aws.String("test-group"), }, KeyAttributes: map[string]*string{ - "Name": aws.String("myService"), - "Environment": aws.String("myEnvironment"), - "AwsAccountId": aws.String("123456789"), + "Name": aws.String("myService"), + "Environment": aws.String("myEnvironment"), }, } @@ -162,48 +155,6 @@ func TestAddSingleEvent_WithAccountId(t *testing.T) { wg.Wait() } -func TestAddSingleEvent_WithoutAccountId(t *testing.T) { - var s svcMock - called := false - nst := "NEXT_SEQ_TOKEN" - - s.ple = func(in *cloudwatchlogs.PutLogEventsInput) (*cloudwatchlogs.PutLogEventsOutput, error) { - called = true - - if in.SequenceToken != nil { - t.Errorf("PutLogEvents called with wrong sequenceToken, first call should not provide any token") - } - - if *in.LogGroupName != "G" || *in.LogStreamName != "S" { - t.Errorf("PutLogEvents called with wrong group and stream: %v/%v", *in.LogGroupName, *in.LogStreamName) - } - - if len(in.LogEvents) != 1 || *in.LogEvents[0].Message != "MSG" { - t.Errorf("PutLogEvents called with incorrect message, got: '%v'", *in.LogEvents[0].Message) - } - require.Nil(t, in.Entity) - return &cloudwatchlogs.PutLogEventsOutput{ - NextSequenceToken: &nst, - }, nil - } - - stop, p := testPreparation(-1, &s, 1*time.Hour, maxRetryTimeout) - p.logSrc = &mockLogSrc{returnEmpty: true} - - p.AddEvent(evtMock{"MSG", time.Now(), nil}) - require.False(t, called, "PutLogEvents has been called too fast, it should wait until FlushTimeout.") - - p.FlushTimeout = 10 * time.Millisecond - p.resetFlushTimer() - - time.Sleep(3 * time.Second) - require.True(t, called, "PutLogEvents has not been called after FlushTimeout has been reached.") - require.NotNil(t, nst, *p.sequenceToken, "Pusher did not capture the NextSequenceToken") - - close(stop) - wg.Wait() -} - func TestStopPusherWouldDoFinalSend(t *testing.T) { var s svcMock called := false From 32ed243ca7d94eaa3412df1de700b7a252f43cfb Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 30 Oct 2024 13:12:12 -0400 Subject: [PATCH 2/2] Drop entity if account ID is not present in cloudwatch exporter --- extension/entitystore/extension.go | 13 +++---- extension/entitystore/extension_test.go | 36 ++++++------------- plugins/outputs/cloudwatch/convert_otel.go | 4 ++- .../outputs/cloudwatch/convert_otel_test.go | 33 +++++++++++++++-- plugins/outputs/cloudwatchlogs/pusher.go | 9 ++--- 5 files changed, 51 insertions(+), 44 deletions(-) diff --git a/extension/entitystore/extension.go b/extension/entitystore/extension.go index d9058f9f78..ec3a3e4d1d 100644 --- a/extension/entitystore/extension.go +++ b/extension/entitystore/extension.go @@ -11,7 +11,6 @@ import ( "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/aws/aws-sdk-go/service/sts/stsiface" "github.com/jellydator/ttlcache/v3" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/extension" @@ -20,6 +19,7 @@ import ( configaws "github.com/aws/amazon-cloudwatch-agent/cfg/aws" "github.com/aws/amazon-cloudwatch-agent/internal/ec2metadataprovider" "github.com/aws/amazon-cloudwatch-agent/internal/retryer" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/entityattributes" "github.com/aws/amazon-cloudwatch-agent/sdk/service/cloudwatchlogs" "github.com/aws/amazon-cloudwatch-agent/translator/config" ) @@ -31,9 +31,6 @@ const ( ServiceNameSourceKey = "AWS.ServiceNameSource" PlatformType = "PlatformType" EC2PlatForm = "AWS::EC2" - Type = "Type" - Name = "Name" - Environment = "Environment" podTerminationCheckInterval = 5 * time.Minute ) @@ -74,8 +71,6 @@ type EntityStore struct { metadataprovider ec2metadataprovider.MetadataProvider - stsClient stsiface.STSAPI - podTerminationCheckInterval time.Duration } @@ -225,10 +220,10 @@ func (e *EntityStore) createAttributeMap() map[string]*string { // createServiceKeyAttribute creates KeyAttributes for Service entities func (e *EntityStore) createServiceKeyAttributes(serviceAttr ServiceAttribute) map[string]*string { serviceKeyAttr := map[string]*string{ - Type: aws.String(Service), + entityattributes.EntityType: aws.String(Service), } - addNonEmptyToMap(serviceKeyAttr, Name, serviceAttr.ServiceName) - addNonEmptyToMap(serviceKeyAttr, Environment, serviceAttr.Environment) + addNonEmptyToMap(serviceKeyAttr, entityattributes.ServiceName, serviceAttr.ServiceName) + addNonEmptyToMap(serviceKeyAttr, entityattributes.DeploymentEnvironment, serviceAttr.Environment) return serviceKeyAttr } diff --git a/extension/entitystore/extension_test.go b/extension/entitystore/extension_test.go index 668c8bd1a4..c4d44c006d 100644 --- a/extension/entitystore/extension_test.go +++ b/extension/entitystore/extension_test.go @@ -15,8 +15,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/sts" - "github.com/aws/aws-sdk-go/service/sts/stsiface" "github.com/jellydator/ttlcache/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -24,6 +22,7 @@ import ( "go.uber.org/zap/zapcore" "github.com/aws/amazon-cloudwatch-agent/internal/ec2metadataprovider" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/entityattributes" "github.com/aws/amazon-cloudwatch-agent/sdk/service/cloudwatchlogs" "github.com/aws/amazon-cloudwatch-agent/translator/config" ) @@ -64,15 +63,6 @@ func (s *mockServiceProvider) getServiceNameAndSource() (string, string) { return "test-service-name", "UserConfiguration" } -type mockSTSClient struct { - stsiface.STSAPI - accountId string -} - -func (ms *mockSTSClient) GetCallerIdentity(*sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) { - return &sts.GetCallerIdentityOutput{Account: aws.String(ms.accountId)}, nil -} - type mockMetadataProvider struct { InstanceIdentityDocument *ec2metadata.EC2InstanceIdentityDocument Tags map[string]string @@ -288,25 +278,25 @@ func TestEntityStore_createServiceKeyAttributes(t *testing.T) { name: "NameAndEnvironmentSet", serviceAttr: ServiceAttribute{ServiceName: "test-service", Environment: "test-environment"}, want: map[string]*string{ - Environment: aws.String("test-environment"), - Name: aws.String("test-service"), - Type: aws.String(Service), + entityattributes.DeploymentEnvironment: aws.String("test-environment"), + entityattributes.ServiceName: aws.String("test-service"), + entityattributes.EntityType: aws.String(Service), }, }, { name: "OnlyNameSet", serviceAttr: ServiceAttribute{ServiceName: "test-service"}, want: map[string]*string{ - Name: aws.String("test-service"), - Type: aws.String(Service), + entityattributes.ServiceName: aws.String("test-service"), + entityattributes.EntityType: aws.String(Service), }, }, { name: "OnlyEnvironmentSet", serviceAttr: ServiceAttribute{Environment: "test-environment"}, want: map[string]*string{ - Environment: aws.String("test-environment"), - Type: aws.String(Service), + entityattributes.DeploymentEnvironment: aws.String("test-environment"), + entityattributes.EntityType: aws.String(Service), }, }, } @@ -320,7 +310,6 @@ func TestEntityStore_createServiceKeyAttributes(t *testing.T) { func TestEntityStore_createLogFileRID(t *testing.T) { instanceId := "i-abcd1234" - accountId := "123456789012" glob := LogFileGlob("glob") group := LogGroupName("group") serviceAttr := ServiceAttribute{ @@ -334,8 +323,6 @@ func TestEntityStore_createLogFileRID(t *testing.T) { mode: config.ModeEC2, ec2Info: EC2Info{InstanceID: instanceId}, serviceprovider: sp, - metadataprovider: mockMetadataProviderWithAccountId(accountId), - stsClient: &mockSTSClient{accountId: accountId}, nativeCredential: &session.Session{}, } @@ -343,9 +330,9 @@ func TestEntityStore_createLogFileRID(t *testing.T) { expectedEntity := cloudwatchlogs.Entity{ KeyAttributes: map[string]*string{ - Environment: aws.String("test-environment"), - Name: aws.String("test-service"), - Type: aws.String(Service), + entityattributes.DeploymentEnvironment: aws.String("test-environment"), + entityattributes.ServiceName: aws.String("test-service"), + entityattributes.EntityType: aws.String(Service), }, Attributes: map[string]*string{ InstanceIDKey: aws.String(instanceId), @@ -543,7 +530,6 @@ func TestEntityStore_GetMetricServiceNameSource(t *testing.T) { ec2Info: EC2Info{InstanceID: instanceId}, serviceprovider: sp, metadataprovider: mockMetadataProviderWithAccountId(accountId), - stsClient: &mockSTSClient{accountId: accountId}, nativeCredential: &session.Session{}, } diff --git a/plugins/outputs/cloudwatch/convert_otel.go b/plugins/outputs/cloudwatch/convert_otel.go index d91ce92c61..a318eaa265 100644 --- a/plugins/outputs/cloudwatch/convert_otel.go +++ b/plugins/outputs/cloudwatch/convert_otel.go @@ -195,7 +195,9 @@ func fetchEntityFields(resourceAttributes pcommon.Map) cloudwatch.Entity { processEntityAttributes(entityattributes.GetKeyAttributeEntityShortNameMap(), keyAttributesMap, resourceAttributes) processEntityAttributes(entityattributes.GetAttributeEntityShortNameMap(platformType), attributeMap, resourceAttributes) removeEntityFields(resourceAttributes) - + if _, ok := keyAttributesMap[entityattributes.AwsAccountId]; !ok { + return cloudwatch.Entity{} + } return cloudwatch.Entity{ KeyAttributes: keyAttributesMap, Attributes: attributeMap, diff --git a/plugins/outputs/cloudwatch/convert_otel_test.go b/plugins/outputs/cloudwatch/convert_otel_test.go index ae4a825f5b..458d72de3a 100644 --- a/plugins/outputs/cloudwatch/convert_otel_test.go +++ b/plugins/outputs/cloudwatch/convert_otel_test.go @@ -378,7 +378,7 @@ func TestProcessAndRemoveEntityAttributes(t *testing.T) { } } -func TestFetchEntityFields(t *testing.T) { +func TestFetchEntityFields_WithoutAccountID(t *testing.T) { resourceMetrics := pmetric.NewResourceMetrics() resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") @@ -390,11 +390,34 @@ func TestFetchEntityFields(t *testing.T) { resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EKS") assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) + expectedEntity := cloudwatch.Entity{ + KeyAttributes: nil, + Attributes: nil, + } + entity := fetchEntityFields(resourceMetrics.Resource().Attributes()) + assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len()) + assert.Equal(t, expectedEntity, entity) +} + +func TestFetchEntityFields_WithAccountID(t *testing.T) { + resourceMetrics := pmetric.NewResourceMetrics() + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityType, "Service") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNode, "my-node") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityCluster, "my-cluster") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EKS") + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") + assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) + expectedEntity := cloudwatch.Entity{ KeyAttributes: map[string]*string{ entityattributes.EntityType: aws.String(entityattributes.Service), entityattributes.ServiceName: aws.String("my-service"), entityattributes.DeploymentEnvironment: aws.String("my-environment"), + entityattributes.AwsAccountId: aws.String("123456789"), }, Attributes: map[string]*string{ entityattributes.Node: aws.String("my-node"), @@ -421,13 +444,15 @@ func TestFetchEntityFieldsOnK8s(t *testing.T) { resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityNamespace, "my-namespace") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityWorkload, "my-workload") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "K8s") - assert.Equal(t, 8, resourceMetrics.Resource().Attributes().Len()) + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") + assert.Equal(t, 9, resourceMetrics.Resource().Attributes().Len()) expectedEntity := cloudwatch.Entity{ KeyAttributes: map[string]*string{ entityattributes.EntityType: aws.String(entityattributes.Service), entityattributes.ServiceName: aws.String("my-service"), entityattributes.DeploymentEnvironment: aws.String("my-environment"), + entityattributes.AwsAccountId: aws.String("123456789"), }, Attributes: map[string]*string{ entityattributes.Node: aws.String("my-node"), @@ -448,13 +473,15 @@ func TestFetchEntityFieldsOnEc2(t *testing.T) { resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityDeploymentEnvironment, "my-environment") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityServiceName, "my-service") resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityPlatformType, "AWS::EC2") - assert.Equal(t, 4, resourceMetrics.Resource().Attributes().Len()) + resourceMetrics.Resource().Attributes().PutStr(entityattributes.AttributeEntityAwsAccountId, "123456789") + assert.Equal(t, 5, resourceMetrics.Resource().Attributes().Len()) expectedEntity := cloudwatch.Entity{ KeyAttributes: map[string]*string{ entityattributes.EntityType: aws.String(entityattributes.Service), entityattributes.ServiceName: aws.String("my-service"), entityattributes.DeploymentEnvironment: aws.String("my-environment"), + entityattributes.AwsAccountId: aws.String("123456789"), }, Attributes: map[string]*string{ entityattributes.Platform: aws.String("AWS::EC2"), diff --git a/plugins/outputs/cloudwatchlogs/pusher.go b/plugins/outputs/cloudwatchlogs/pusher.go index 2c1036f1e8..ff0bb1dd0e 100644 --- a/plugins/outputs/cloudwatchlogs/pusher.go +++ b/plugins/outputs/cloudwatchlogs/pusher.go @@ -222,18 +222,15 @@ func (p *pusher) send() { if p.needSort { sort.Stable(ByTimestamp(p.events)) } - var entity *cloudwatchlogs.Entity - if p.logSrc != nil { - entity = p.logSrc.Entity() - } - input := &cloudwatchlogs.PutLogEventsInput{ LogEvents: p.events, LogGroupName: &p.Group, LogStreamName: &p.Stream, SequenceToken: p.sequenceToken, } - input.Entity = entity + if p.logSrc != nil { + input.Entity = p.logSrc.Entity() + } startTime := time.Now()