From 739edef722882367082ac6ff2a4b8f062f06da3f Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Wed, 20 Mar 2024 11:34:29 -0700 Subject: [PATCH 01/11] Revert "[AppSignals] Revert EC2 to use generic configs (#1069)" This reverts commit 622dc67257624706332034609f69d070448a017a. --- .../otel/exporter/awsemf/translator.go | 2 ++ .../otel/exporter/awsemf/translator_test.go | 2 +- .../otel/exporter/awsxray/translator.go | 2 ++ .../otel/exporter/awsxray/translator_test.go | 2 +- .../processor/awsappsignals/translator.go | 20 ++++++++++++++----- .../awsappsignals/translator_test.go | 4 ++-- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 3830742569..7b82a3d463 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -148,6 +148,8 @@ func getAppSignalsConfig() string { return appSignalsConfigEks } else if kubernetesMode == config.ModeK8sEC2 || kubernetesMode == config.ModeK8sOnPrem { return appSignalsConfigK8s + } else if ctx.Mode() == config.ModeEC2 { + return appSignalsConfigEC2 } return appSignalsConfigGeneric } diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 73ed9d41d6..3ddde1f50a 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -949,7 +949,7 @@ func TestTranslateAppSignals(t *testing.T) { "app_signals": map[string]any{}, }, }}, - want: testutil.GetConfWithOverrides(t, filepath.Join("appsignals_config_generic.yaml"), map[string]any{ + want: testutil.GetConfWithOverrides(t, filepath.Join("appsignals_config_ec2.yaml"), map[string]any{ "local_mode": "false", "region": "us-east-1", "role_arn": "global_arn", diff --git a/translator/translate/otel/exporter/awsxray/translator.go b/translator/translate/otel/exporter/awsxray/translator.go index 825b300b3f..5708558d25 100644 --- a/translator/translate/otel/exporter/awsxray/translator.go +++ b/translator/translate/otel/exporter/awsxray/translator.go @@ -85,6 +85,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg.IndexedAttributes = indexedAttributesEKS } else if ctx.KubernetesMode() == config.ModeK8sEC2 || ctx.KubernetesMode() == config.ModeK8sOnPrem { cfg.IndexedAttributes = indexedAttributesK8s + } else if ctx.Mode() == config.ModeEC2 { + cfg.IndexedAttributes = indexedAttributesEC2 } else { cfg.IndexedAttributes = indexedAttributesGeneric } diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index dd6ff49227..44fab012db 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -140,8 +140,8 @@ func TestTranslator(t *testing.T) { "aws.local.operation", "aws.remote.service", "aws.remote.operation", + "HostedIn.EC2.Environment", "aws.remote.target", - "HostedIn.Environment", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsappsignals/translator.go index 689a807aab..19f86dbfd8 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsappsignals/translator.go @@ -76,17 +76,27 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } kubernetesMode := context.CurrentContext().KubernetesMode() - if kubernetesMode == config.ModeEKS { + switch kubernetesMode { + case config.ModeEKS: cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewEKSResolver(hostedIn), } - } else if kubernetesMode == config.ModeK8sEC2 || kubernetesMode == config.ModeK8sOnPrem { + case config.ModeK8sEC2, config.ModeK8sOnPrem: cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewK8sResolver(hostedIn), } - } else { - cfg.Resolvers = []appsignalsconfig.Resolver{ - appsignalsconfig.NewGenericResolver(hostedIn), + } + + if kubernetesMode == "" { + switch context.CurrentContext().Mode() { + case config.ModeEC2: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewEC2Resolver(hostedIn), + } + default: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewGenericResolver(hostedIn), + } } } diff --git a/translator/translate/otel/processor/awsappsignals/translator_test.go b/translator/translate/otel/processor/awsappsignals/translator_test.go index cca81d4f1b..c1897b8601 100644 --- a/translator/translate/otel/processor/awsappsignals/translator_test.go +++ b/translator/translate/otel/processor/awsappsignals/translator_test.go @@ -26,7 +26,7 @@ var ( validAppSignalsYamlEKS string //go:embed testdata/config_k8s.yaml validAppSignalsYamlK8s string - //go:embed testdata/config_generic.yaml + //go:embed testdata/config_ec2.yaml validAppSignalsYamlEC2 string //go:embed testdata/config_generic.yaml validAppSignalsYamlGeneric string @@ -169,7 +169,7 @@ func TestTranslate(t *testing.T) { "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ "app_signals": map[string]interface{}{ - "hosted_in": "", + "hosted_in": "test", }, }, }}, From 51d7b926d4c5e5b88581df02a06551ea5f87ea01 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Thu, 11 Apr 2024 09:10:37 -0700 Subject: [PATCH 02/11] Update AppSignals metric schema and log group name --- .github/workflows/appsignals-e2e-ec2-test.yml | 2 +- .github/workflows/appsignals-e2e-eks-test.yml | 2 +- .../processors/awsappsignals/common/types.go | 31 +-- .../processors/awsappsignals/config/config.go | 2 +- .../awsappsignals/config/resolvers.go | 11 + plugins/processors/awsappsignals/factory.go | 2 +- .../internal/attributes/attributes.go | 38 +-- .../cardinalitycontrol/metrics_limiter.go | 16 +- .../metrics_limiter_test.go | 3 +- .../normalizer/attributesnormalizer.go | 64 +++-- .../normalizer/attributesnormalizer_test.go | 12 +- .../internal/resolver/attributesresolver.go | 96 +++++-- .../resolver/attributesresolver_test.go | 175 ++++++++++-- .../awsappsignals/internal/resolver/ec2.go | 59 ---- .../internal/resolver/ec2_test.go | 148 ---------- .../internal/resolver/kubernetes.go | 69 +++-- .../internal/resolver/kubernetes_test.go | 260 +++++++++++++----- .../processors/awsappsignals/rules/common.go | 31 ++- .../awsappsignals/rules/common_test.go | 41 ++- .../awsappsignals/rules/replacer.go | 15 +- .../awsappsignals/rules/replacer_test.go | 71 +++++ translator/config/mode.go | 4 + .../appsignals_and_eks_config.yaml | 80 +++--- .../appsignals_and_k8s_config.yaml | 84 +++--- .../appsignals_fallback_and_eks_config.yaml | 80 +++--- .../appsignals_over_fallback_config.yaml | 80 +++--- .../sampleConfig/base_appsignals_config.yaml | 38 +-- .../base_appsignals_fallback_config.yaml | 38 +-- .../awsemf/appsignals_config_ec2.yaml | 33 --- .../awsemf/appsignals_config_eks.yaml | 30 +- .../awsemf/appsignals_config_generic.yaml | 20 +- .../awsemf/appsignals_config_k8s.yaml | 30 +- .../otel/exporter/awsemf/translator.go | 27 +- .../otel/exporter/awsemf/translator_test.go | 2 +- .../otel/exporter/awsxray/translator.go | 35 +-- .../otel/exporter/awsxray/translator_test.go | 24 +- .../processor/awsappsignals/translator.go | 36 ++- .../awsappsignals/translator_test.go | 2 +- 38 files changed, 985 insertions(+), 806 deletions(-) delete mode 100644 plugins/processors/awsappsignals/internal/resolver/ec2.go delete mode 100644 plugins/processors/awsappsignals/internal/resolver/ec2_test.go delete mode 100644 translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml diff --git a/.github/workflows/appsignals-e2e-ec2-test.yml b/.github/workflows/appsignals-e2e-ec2-test.yml index da1bd70139..95464b7918 100644 --- a/.github/workflows/appsignals-e2e-ec2-test.yml +++ b/.github/workflows/appsignals-e2e-ec2-test.yml @@ -21,7 +21,7 @@ env: GET_ADOT_JAR_COMMAND: "wget -O adot.jar https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest/download/aws-opentelemetry-agent.jar" GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" METRIC_NAMESPACE: AppSignals - LOG_GROUP_NAME: /aws/appsignals/generic + LOG_GROUP_NAME: /aws/application-signals/data jobs: e2e-ec2-test: diff --git a/.github/workflows/appsignals-e2e-eks-test.yml b/.github/workflows/appsignals-e2e-eks-test.yml index 0d905deb10..011f4df596 100644 --- a/.github/workflows/appsignals-e2e-eks-test.yml +++ b/.github/workflows/appsignals-e2e-eks-test.yml @@ -25,7 +25,7 @@ env: SAMPLE_APP_FRONTEND_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_FRONTEND_SVC_IMG }} SAMPLE_APP_REMOTE_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_REMOTE_SVC_IMG }} METRIC_NAMESPACE: AppSignals - LOG_GROUP_NAME: /aws/appsignals/eks + LOG_GROUP_NAME: /aws/application-signals/data ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" jobs: diff --git a/plugins/processors/awsappsignals/common/types.go b/plugins/processors/awsappsignals/common/types.go index 993e83b633..86615c1f2d 100644 --- a/plugins/processors/awsappsignals/common/types.go +++ b/plugins/processors/awsappsignals/common/types.go @@ -4,24 +4,23 @@ package common const ( - AttributeRemoteService = "aws.remote.service" - AttributeHostedInEnvironment = "aws.hostedin.environment" -) - -const ( - MetricAttributeRemoteNamespace = "K8s.RemoteNamespace" - MetricAttributeLocalService = "Service" - MetricAttributeLocalOperation = "Operation" - MetricAttributeRemoteService = "RemoteService" - MetricAttributeRemoteOperation = "RemoteOperation" - MetricAttributeRemoteTarget = "RemoteTarget" + MetricAttributeLocalService = "Service" + MetricAttributeLocalOperation = "Operation" + MetricAttributeEnvironment = "Environment" + MetricAttributeRemoteService = "RemoteService" + MetricAttributeRemoteEnvironment = "RemoteEnvironment" + MetricAttributeRemoteOperation = "RemoteOperation" + MetricAttributeRemoteResourceIdentifier = "RemoteResourceIdentifier" + MetricAttributeRemoteResourceType = "RemoteResourceType" ) const ( - HostedInAttributeClusterName = "HostedIn.EKS.Cluster" - HostedInAttributeK8SNamespace = "HostedIn.K8s.Namespace" - HostedInAttributeEnvironment = "HostedIn.Environment" - HostedInAttributeK8SClusterName = "HostedIn.K8s.Cluster" - HostedInAttributeEC2Environment = "HostedIn.EC2.Environment" + AttributeEKSClusterName = "EKS.Cluster" + AttributeK8SClusterName = "K8s.Cluster" + AttributeK8SNamespace = "K8s.Namespace" + AttributeEC2AutoScalingGroupName = "EC2.AutoScalingGroupName" + AttributeEC2InstanceId = "EC2.InstanceId" + AttributePlatformType = "PlatformType" + AttributeSDK = "SDK" ) const ( diff --git a/plugins/processors/awsappsignals/config/config.go b/plugins/processors/awsappsignals/config/config.go index f8ba4e16ed..facdefbf51 100644 --- a/plugins/processors/awsappsignals/config/config.go +++ b/plugins/processors/awsappsignals/config/config.go @@ -62,7 +62,7 @@ func (cfg *Config) Validate() error { if resolver.Name == "" { return errors.New("name must not be empty for k8s resolver") } - case PlatformEC2, PlatformGeneric: + case PlatformEC2, PlatformGeneric, PlatformECS, PlatformLambda: default: return errors.New("unknown resolver") } diff --git a/plugins/processors/awsappsignals/config/resolvers.go b/plugins/processors/awsappsignals/config/resolvers.go index 59fe3f69c2..7f6291112f 100644 --- a/plugins/processors/awsappsignals/config/resolvers.go +++ b/plugins/processors/awsappsignals/config/resolvers.go @@ -12,6 +12,10 @@ const ( PlatformK8s = "k8s" // PlatformEC2 Amazon EC2 platform PlatformEC2 = "ec2" + // PlatformECS Amazon ECS + PlatformECS = "ecs" + // PlatformLambda Amazon Lambda + PlatformLambda = "lambda" ) type Resolver struct { @@ -40,6 +44,13 @@ func NewEC2Resolver(name string) Resolver { } } +func NewECSResolver(name string) Resolver { + return Resolver{ + Name: name, + Platform: PlatformECS, + } +} + func NewGenericResolver(name string) Resolver { return Resolver{ Name: name, diff --git a/plugins/processors/awsappsignals/factory.go b/plugins/processors/awsappsignals/factory.go index d652c04a6e..8a8b0a45a5 100644 --- a/plugins/processors/awsappsignals/factory.go +++ b/plugins/processors/awsappsignals/factory.go @@ -92,7 +92,7 @@ func createProcessor( ) (*awsappsignalsprocessor, error) { pCfg, ok := cfg.(*appsignalsconfig.Config) if !ok { - return nil, errors.New("could not initialize awsappsignalsprocessor") + return nil, errors.New("could not initialize awsapplicationsignalsprocessor") } ap := &awsappsignalsprocessor{logger: params.Logger, config: pCfg} diff --git a/plugins/processors/awsappsignals/internal/attributes/attributes.go b/plugins/processors/awsappsignals/internal/attributes/attributes.go index dc3a1a5194..fd9877cc3e 100644 --- a/plugins/processors/awsappsignals/internal/attributes/attributes.go +++ b/plugins/processors/awsappsignals/internal/attributes/attributes.go @@ -5,38 +5,20 @@ package attributes const ( // aws attributes - AWSLocalService = "aws.local.service" - AWSLocalOperation = "aws.local.operation" - AWSRemoteService = "aws.remote.service" - AWSRemoteOperation = "aws.remote.operation" - AWSRemoteTarget = "aws.remote.target" - AWSHostedInEnvironment = "aws.hostedin.environment" + AWSLocalService = "aws.local.service" + AWSLocalOperation = "aws.local.operation" + AWSRemoteService = "aws.remote.service" + AWSRemoteOperation = "aws.remote.operation" + AWSRemoteTarget = "aws.remote.target" + AWSRemoteResourceIdentifier = "aws.remote.resource.identifier" + AWSRemoteResourceType = "aws.remote.resource.type" + AWSHostedInEnvironment = "aws.hostedin.environment" // resource detection processor attributes ResourceDetectionHostId = "host.id" ResourceDetectionHostName = "host.name" ResourceDetectionASG = "ec2.tag.aws:autoscaling:groupName" - // kubernetes resource attributes - K8SDeploymentName = "k8s.deployment.name" - K8SStatefulSetName = "k8s.statefulset.name" - K8SDaemonSetName = "k8s.daemonset.name" - K8SJobName = "k8s.job.name" - K8SCronJobName = "k8s.cronjob.name" - K8SPodName = "k8s.pod.name" - K8SRemoteNamespace = "K8s.RemoteNamespace" - - // ec2 resource attributes - EC2AutoScalingGroupName = "EC2.AutoScalingGroupName" - EC2InstanceId = "EC2.InstanceId" - - // hosted in attribute names - HostedInClusterNameEKS = "HostedIn.EKS.Cluster" - HostedInClusterNameK8s = "HostedIn.K8s.Cluster" - HostedInK8SNamespace = "HostedIn.K8s.Namespace" - HostedInEC2Environment = "HostedIn.EC2.Environment" - HostedInEnvironment = "HostedIn.Environment" - - // sdk attributes - MetricAttributeSDKMetadata = "SDK" + AWSLocalEnvironment = "aws.local.environment" + AWSRemoteEnvironment = "aws.remote.environment" ) diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go b/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go index 4fad107807..b1309b8bb5 100644 --- a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go +++ b/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "sort" - "strings" "sync" "time" @@ -29,17 +28,16 @@ const ( ) var awsDeclaredMetricAttributes = []string{ - common.HostedInAttributeClusterName, - common.HostedInAttributeK8SNamespace, - common.HostedInAttributeEnvironment, + common.AttributeEKSClusterName, + common.AttributeK8SNamespace, + common.MetricAttributeEnvironment, common.MetricAttributeLocalService, common.MetricAttributeLocalOperation, common.MetricAttributeRemoteService, common.MetricAttributeRemoteOperation, - common.MetricAttributeRemoteTarget, - common.MetricAttributeRemoteNamespace, - common.HostedInAttributeK8SClusterName, - common.HostedInAttributeEC2Environment, + common.MetricAttributeRemoteResourceIdentifier, + common.MetricAttributeRemoteEnvironment, + common.AttributeK8SClusterName, } type Limiter interface { @@ -338,7 +336,7 @@ func (s *service) admitMetricData(metric *MetricData) bool { func (s *service) rollupMetricData(attributes pcommon.Map) { for _, indexAttr := range awsDeclaredMetricAttributes { - if strings.HasPrefix(indexAttr, "HostedIn.") || (indexAttr == common.MetricAttributeLocalService) || (indexAttr == common.MetricAttributeRemoteService) { + if (indexAttr == common.MetricAttributeEnvironment) || (indexAttr == common.MetricAttributeLocalService) || (indexAttr == common.MetricAttributeRemoteService) { continue } if indexAttr == common.MetricAttributeLocalOperation { diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go b/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go index ca502cc005..9ce7112b20 100644 --- a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go +++ b/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go @@ -8,7 +8,6 @@ import ( "fmt" "math/rand" "strconv" - "strings" "testing" "time" @@ -42,7 +41,7 @@ func TestAdmitAndRollup(t *testing.T) { admittedAttributes[uniqKey.AsString()] = attr } else { for _, indexedAttrKey := range awsDeclaredMetricAttributes { - if strings.HasPrefix(indexedAttrKey, "HostedIn.") || indexedAttrKey == "Service" || indexedAttrKey == "RemoteService" { + if indexedAttrKey == "Environment" || indexedAttrKey == "Service" || indexedAttrKey == "RemoteService" { continue } attrValue, _ := attr.Get(indexedAttrKey) diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go index 22559f91a0..52fb8fbd59 100644 --- a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go @@ -9,8 +9,10 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" conventions "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) @@ -18,25 +20,33 @@ type attributesNormalizer struct { logger *zap.Logger } -var renameMapForMetric = map[string]string{ - attr.AWSLocalService: "Service", - attr.AWSLocalOperation: "Operation", - attr.AWSRemoteService: "RemoteService", - attr.AWSRemoteOperation: "RemoteOperation", - attr.AWSRemoteTarget: "RemoteTarget", +var attributesRenamingForMetric = map[string]string{ + attr.AWSLocalService: common.MetricAttributeLocalService, + attr.AWSLocalOperation: common.MetricAttributeLocalOperation, + attr.AWSRemoteService: common.MetricAttributeRemoteService, + attr.AWSRemoteOperation: common.MetricAttributeRemoteOperation, + attr.AWSRemoteEnvironment: common.MetricAttributeRemoteEnvironment, + attr.AWSRemoteTarget: common.MetricAttributeRemoteResourceIdentifier, + attr.AWSRemoteResourceIdentifier: common.MetricAttributeRemoteResourceIdentifier, + attr.AWSRemoteResourceType: common.MetricAttributeRemoteResourceType, } -var renameMapForTrace = map[string]string{ +var resourceAttributesRenamingForTrace = map[string]string{ // these kubernetes resource attributes are set by the openTelemetry operator // see the code references from upstream: // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 - attr.K8SDeploymentName: "K8s.Workload", - attr.K8SStatefulSetName: "K8s.Workload", - attr.K8SDaemonSetName: "K8s.Workload", - attr.K8SJobName: "K8s.Workload", - attr.K8SCronJobName: "K8s.Workload", - attr.K8SPodName: "K8s.Pod", + semconv.AttributeK8SDeploymentName: "K8s.Workload", + semconv.AttributeK8SStatefulSetName: "K8s.Workload", + semconv.AttributeK8SDaemonSetName: "K8s.Workload", + semconv.AttributeK8SJobName: "K8s.Workload", + semconv.AttributeK8SCronJobName: "K8s.Workload", + semconv.AttributeK8SPodName: "K8s.Pod", +} + +var attributesRenamingForTrace = map[string]string{ + common.MetricAttributeEnvironment: attr.AWSLocalEnvironment, + attr.AWSRemoteTarget: attr.AWSRemoteResourceIdentifier, } var copyMapForMetric = map[string]string{ @@ -44,12 +54,12 @@ var copyMapForMetric = map[string]string{ // see the code referecnes from upstream: // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 - attr.K8SDeploymentName: "K8s.Workload", - attr.K8SStatefulSetName: "K8s.Workload", - attr.K8SDaemonSetName: "K8s.Workload", - attr.K8SJobName: "K8s.Workload", - attr.K8SCronJobName: "K8s.Workload", - attr.K8SPodName: "K8s.Pod", + semconv.AttributeK8SDeploymentName: "K8s.Workload", + semconv.AttributeK8SStatefulSetName: "K8s.Workload", + semconv.AttributeK8SDaemonSetName: "K8s.Workload", + semconv.AttributeK8SJobName: "K8s.Workload", + semconv.AttributeK8SCronJobName: "K8s.Workload", + semconv.AttributeK8SPodName: "K8s.Pod", } const ( @@ -71,14 +81,12 @@ func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Ma } func (n *attributesNormalizer) renameAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { - attrs := attributes - renameMap := renameMapForMetric if isTrace { - attrs = resourceAttributes - renameMap = renameMapForTrace + rename(resourceAttributes, resourceAttributesRenamingForTrace) + rename(attributes, attributesRenamingForTrace) + } else { + rename(attributes, attributesRenamingForMetric) } - - rename(attrs, renameMap) } func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { @@ -92,7 +100,7 @@ func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, re n.logger.Debug("attribute value is overwritten", zap.String("attribute", k), zap.String("original", originalAttrValue.AsString()), zap.String("new", resourceAttrValue.AsString())) } attributes.PutStr(v, resourceAttrValue.AsString()) - if k == attr.K8SPodName { + if k == semconv.AttributeK8SPodName { // only copy "host.id" from resource attributes to "K8s.Node" in attributesif the pod name is set if host, ok := resourceAttributes.Get("host.id"); ok { attributes.PutStr("K8s.Node", host.AsString()) @@ -134,7 +142,7 @@ func (n *attributesNormalizer) appendNewAttributes(attributes, resourceAttribute sdkVersion = sdkAutoVersion mode = instrumentationModeAuto } - attributes.PutStr(attr.MetricAttributeSDKMetadata, fmt.Sprintf("%s,%s,%s,%s", sdkName, sdkVersion, sdkLang, mode)) + attributes.PutStr(common.AttributeSDK, fmt.Sprintf("%s,%s,%s,%s", sdkName, sdkVersion, sdkLang, mode)) } func rename(attrs pcommon.Map, renameMap map[string]string) { @@ -142,7 +150,7 @@ func rename(attrs pcommon.Map, renameMap map[string]string) { if value, ok := attrs.Get(original); ok { attrs.PutStr(replacement, value.AsString()) attrs.Remove(original) - if original == attr.K8SPodName { + if original == semconv.AttributeK8SPodName { // only rename host.id if the pod name is set if host, ok := attrs.Get("host.id"); ok { attrs.PutStr("K8s.Node", host.AsString()) diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go index ac36c77803..a9e1eff137 100644 --- a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go @@ -19,7 +19,7 @@ func TestRenameAttributes_for_metric(t *testing.T) { // test for metric // Create a pcommon.Map with some attributes attributes := pcommon.NewMap() - for originalKey, replacementKey := range renameMapForMetric { + for originalKey, replacementKey := range attributesRenamingForMetric { attributes.PutStr(originalKey, replacementKey+"-value") } @@ -28,14 +28,14 @@ func TestRenameAttributes_for_metric(t *testing.T) { normalizer.renameAttributes(attributes, resourceAttributes, false) // Check that the original key has been removed - for originalKey := range renameMapForMetric { + for originalKey := range attributesRenamingForMetric { if _, ok := attributes.Get(originalKey); ok { t.Errorf("originalKey was not removed") } } // Check that the new key has the correct value - for _, replacementKey := range renameMapForMetric { + for _, replacementKey := range attributesRenamingForMetric { if value, ok := attributes.Get(replacementKey); !ok || value.AsString() != replacementKey+"-value" { t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), replacementKey+"-value") } @@ -49,7 +49,7 @@ func TestRenameAttributes_for_trace(t *testing.T) { // test for trace // Create a pcommon.Map with some attributes resourceAttributes := pcommon.NewMap() - for originalKey, replacementKey := range renameMapForTrace { + for originalKey, replacementKey := range resourceAttributesRenamingForTrace { resourceAttributes.PutStr(originalKey, replacementKey+"-value") } resourceAttributes.PutStr("host.id", "i-01ef7d37f42caa168") @@ -59,14 +59,14 @@ func TestRenameAttributes_for_trace(t *testing.T) { normalizer.renameAttributes(attributes, resourceAttributes, true) // Check that the original key has been removed - for originalKey := range renameMapForTrace { + for originalKey := range resourceAttributesRenamingForTrace { if _, ok := resourceAttributes.Get(originalKey); ok { t.Errorf("originalKey was not removed") } } // Check that the new key has the correct value - for _, replacementKey := range renameMapForTrace { + for _, replacementKey := range resourceAttributesRenamingForTrace { if value, ok := resourceAttributes.Get(replacementKey); !ok || value.AsString() != replacementKey+"-value" { t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), replacementKey+"-value") } diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go index 3531545ec1..1dd617aeb3 100644 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go @@ -6,19 +6,40 @@ package resolver import ( "context" "errors" + "fmt" + "strings" "go.opentelemetry.io/collector/pdata/pcommon" + semconv1 "go.opentelemetry.io/collector/semconv/v1.17.0" "go.uber.org/zap" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) -const AttributePlatformGeneric = "Generic" +const ( + AttributeEnvironmentDefault = "default" + AttributePlatformGeneric = "Generic" + AttributePlatformEC2 = "AWS::EC2" + AttributePlatformEKS = "AWS::EKS" + AttributePlatformLambda = "AWS::Lambda" + AttributePlatformK8S = "K8s" +) + +var GenericInheritedAttributes = map[string]string{ + attr.AWSHostedInEnvironment: common.MetricAttributeEnvironment, + semconv1.AttributeDeploymentEnvironment: common.MetricAttributeEnvironment, + attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, +} -var DefaultHostedInAttributes = map[string]string{ - attr.AWSHostedInEnvironment: attr.HostedInEnvironment, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, +// DefaultInheritedAttributes is an allow-list that also renames attributes from the resource detection processor +var DefaultInheritedAttributes = map[string]string{ + attr.AWSHostedInEnvironment: common.MetricAttributeEnvironment, + semconv1.AttributeDeploymentEnvironment: common.MetricAttributeEnvironment, + attr.ResourceDetectionASG: common.AttributeEC2AutoScalingGroupName, + attr.ResourceDetectionHostId: common.AttributeEC2InstanceId, + attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, } type subResolver interface { @@ -37,11 +58,13 @@ func NewAttributesResolver(resolvers []appsignalsconfig.Resolver, logger *zap.Lo for _, resolver := range resolvers { switch resolver.Platform { case appsignalsconfig.PlatformEKS, appsignalsconfig.PlatformK8s: - subResolvers = append(subResolvers, getKubernetesResolver(logger), newKubernetesHostedInAttributeResolver(resolver.Name)) + subResolvers = append(subResolvers, getKubernetesResolver(resolver.Platform, resolver.Name, logger), newKubernetesResourceAttributesResolver(resolver.Platform, resolver.Name)) case appsignalsconfig.PlatformEC2: - subResolvers = append(subResolvers, newEC2HostedInAttributeResolver(resolver.Name)) + subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformEC2, DefaultInheritedAttributes)) + case appsignalsconfig.PlatformECS: + subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformGeneric, DefaultInheritedAttributes)) default: - subResolvers = append(subResolvers, newHostedInAttributeResolver(resolver.Name, DefaultHostedInAttributes)) + subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformGeneric, GenericInheritedAttributes)) } } return &attributesResolver{ @@ -69,34 +92,67 @@ func (r *attributesResolver) Stop(ctx context.Context) error { return errs } -type hostedInAttributeResolver struct { - name string +type resourceAttributesResolver struct { + platformCode string + platformType string attributeMap map[string]string } -func newHostedInAttributeResolver(name string, attributeMap map[string]string) *hostedInAttributeResolver { - if name == "" { - name = AttributePlatformGeneric - } - return &hostedInAttributeResolver{ - name: name, +func newResourceAttributesResolver(platformCode, platformType string, attributeMap map[string]string) *resourceAttributesResolver { + return &resourceAttributesResolver{ + platformCode: platformCode, + platformType: platformType, attributeMap: attributeMap, } } -func (h *hostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { +func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcommon.Map) error { for attrKey, mappingKey := range h.attributeMap { if val, ok := resourceAttributes.Get(attrKey); ok { attributes.PutStr(mappingKey, val.AsString()) } } - - if _, ok := resourceAttributes.Get(attr.AWSHostedInEnvironment); !ok { - attributes.PutStr(attr.HostedInEnvironment, h.name) + if _, ok := attributes.Get(common.MetricAttributeEnvironment); !ok { + if h.platformCode == appsignalsconfig.PlatformECS { + if clusterName, ok := getECSClusterName(resourceAttributes); ok { + attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.platformCode, clusterName)) + } + } + if h.platformCode == appsignalsconfig.PlatformEC2 { + if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { + attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.platformCode, asgAttr.Str())) + } + } } + if _, ok := attributes.Get(common.MetricAttributeEnvironment); !ok { + attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.platformCode, AttributeEnvironmentDefault)) + } + attributes.PutStr(common.AttributePlatformType, h.platformType) return nil } -func (h *hostedInAttributeResolver) Stop(ctx context.Context) error { +func getECSClusterName(resourceAttributes pcommon.Map) (string, bool) { + if clusterAttr, ok := resourceAttributes.Get(semconv1.AttributeAWSECSClusterARN); ok { + parts := strings.Split(clusterAttr.Str(), "/") + clusterName := parts[len(parts)-1] + return clusterName, true + } else if taskAttr, ok := resourceAttributes.Get(semconv1.AttributeAWSECSTaskARN); ok { + parts := strings.SplitAfterN(taskAttr.Str(), ":task/", 2) + if len(parts) == 2 { + taskParts := strings.Split(parts[1], "/") + // cluster name in ARN + if len(taskParts) == 2 { + return taskParts[0], true + } + } + } + return "", false +} + +func GetDefaultEnvironment(platformCode, val string) string { + return fmt.Sprintf("%s:%s", platformCode, val) +} + +func (h *resourceAttributesResolver) Stop(ctx context.Context) error { return nil } diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go index 336ce57ad7..d1eba59b9f 100644 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go +++ b/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go @@ -11,7 +11,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "go.opentelemetry.io/collector/pdata/pcommon" + semconv "go.opentelemetry.io/collector/semconv/v1.17.0" + "go.uber.org/zap" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) @@ -29,45 +33,96 @@ func (m *MockSubResolver) Stop(ctx context.Context) error { return args.Error(0) } -func TestHostedInAttributeResolverWithNoConfiguredName(t *testing.T) { - resolver := newHostedInAttributeResolver("", DefaultHostedInAttributes) +func TestResourceAttributesResolverWithNoConfiguredName(t *testing.T) { + tests := []struct { + name string + platformCode string + platformType string + resolver config.Resolver + }{ + { + "testOnGeneric", + config.PlatformGeneric, + AttributePlatformGeneric, + config.NewGenericResolver(""), + }, + { + "testOnEC2", + config.PlatformEC2, + AttributePlatformEC2, + config.NewEC2Resolver(""), + }, + { + "testOnECS", + config.PlatformECS, + AttributePlatformGeneric, + config.NewECSResolver(""), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{tt.resolver}, logger) + resolver := attributesResolver.subResolvers[0] - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEnvironment) - assert.True(t, ok) - assert.Equal(t, "Generic", envAttr.AsString()) + resolver.Process(attributes, resourceAttributes) + + attr, ok := attributes.Get(common.AttributePlatformType) + assert.True(t, ok) + assert.Equal(t, tt.platformType, attr.Str()) + + attr, ok = attributes.Get(common.MetricAttributeEnvironment) + assert.True(t, ok) + assert.Equal(t, tt.platformCode+":default", attr.Str()) + }) + } } -func TestHostedInAttributeResolverWithConfiguredName(t *testing.T) { - resolver := newHostedInAttributeResolver("test", DefaultHostedInAttributes) +func TestResourceAttributesResolverWithECSClusterName(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{config.NewECSResolver("")}, logger) + resolver := attributesResolver.subResolvers[0] attributes := pcommon.NewMap() resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/10838bed-421f-43ef-870a-f43feacbbb5b") resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEnvironment) + + attr, ok := attributes.Get(common.AttributePlatformType) + assert.True(t, ok) + assert.Equal(t, "Generic", attr.Str()) + + attr, ok = attributes.Get(common.MetricAttributeEnvironment) assert.True(t, ok) - assert.Equal(t, "test", envAttr.AsString()) + assert.Equal(t, "ecs:my-cluster", attr.Str()) } -func TestHostedInAttributeResolverWithConflictedName(t *testing.T) { - resolver := newHostedInAttributeResolver("test", DefaultHostedInAttributes) +func TestResourceAttributesResolverWithOnEC2WithASG(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{config.NewEC2Resolver("")}, logger) + resolver := attributesResolver.subResolvers[0] attributes := pcommon.NewMap() resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "self-defined") + resourceAttributes.PutStr(attr.ResourceDetectionASG, "my-asg") resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEnvironment) + platformAttr, ok := attributes.Get(common.AttributePlatformType) + assert.True(t, ok) + assert.Equal(t, "AWS::EC2", platformAttr.Str()) + envAttr, ok := attributes.Get(common.MetricAttributeEnvironment) assert.True(t, ok) - assert.Equal(t, "self-defined", envAttr.AsString()) + assert.Equal(t, "ec2:my-asg", envAttr.Str()) } -func TestHostedInAttributeResolverWithHostname(t *testing.T) { - resolver := newHostedInAttributeResolver("test", DefaultHostedInAttributes) +func TestResourceAttributesResolverWithHostname(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{config.NewGenericResolver("")}, logger) + resolver := attributesResolver.subResolvers[0] attributes := pcommon.NewMap() resourceAttributes := pcommon.NewMap() @@ -79,6 +134,69 @@ func TestHostedInAttributeResolverWithHostname(t *testing.T) { assert.Equal(t, "hostname", envAttr.AsString()) } +func TestResourceAttributesResolverWithCustomEnvironment(t *testing.T) { + tests := []struct { + name string + platformCode string + resolver config.Resolver + }{ + { + "testOnGeneric", + config.PlatformGeneric, + config.NewGenericResolver(""), + }, + { + "testOnEC2", + config.PlatformEC2, + config.NewEC2Resolver(""), + }, + { + "testOnECS", + config.PlatformECS, + config.NewEC2Resolver(""), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{tt.resolver}, logger) + resolver := attributesResolver.subResolvers[0] + + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + // insert default env + resourceAttributes.PutStr(attr.ResourceDetectionASG, "my-asg") + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/10838bed-421f-43ef-870a-f43feacbbb5b") + + // insert custom env + resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "env1") + resolver.Process(attributes, resourceAttributes) + envAttr, ok := attributes.Get(common.MetricAttributeEnvironment) + assert.True(t, ok) + assert.Equal(t, "env1", envAttr.Str()) + + attributes = pcommon.NewMap() + resourceAttributes = pcommon.NewMap() + + resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "error") + resourceAttributes.PutStr(semconv.AttributeDeploymentEnvironment, "env2") + resolver.Process(attributes, resourceAttributes) + envAttr, ok = attributes.Get(common.MetricAttributeEnvironment) + assert.True(t, ok) + assert.Equal(t, "env2", envAttr.Str()) + + attributes = pcommon.NewMap() + resourceAttributes = pcommon.NewMap() + + resourceAttributes.PutStr(semconv.AttributeDeploymentEnvironment, "env3") + resolver.Process(attributes, resourceAttributes) + envAttr, ok = attributes.Get(common.MetricAttributeEnvironment) + assert.True(t, ok) + assert.Equal(t, "env3", envAttr.Str()) + }) + } +} + func TestAttributesResolver_Process(t *testing.T) { attributes := pcommon.NewMap() resourceAttributes := pcommon.NewMap() @@ -117,3 +235,22 @@ func TestAttributesResolver_Stop(t *testing.T) { mockSubResolver1.AssertExpectations(t) mockSubResolver2.AssertExpectations(t) } + +func TestGetClusterName(t *testing.T) { + resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSClusterARN, "arn:aws:ecs:us-west-2:123456789123:cluster/my-cluster") + clusterName, ok := getECSClusterName(resourceAttributes) + assert.True(t, ok) + assert.Equal(t, "my-cluster", clusterName) + + resourceAttributes = pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/10838bed-421f-43ef-870a-f43feacbbb5b") + _, ok = getECSClusterName(resourceAttributes) + assert.False(t, ok) + + resourceAttributes = pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/10838bed-421f-43ef-870a-f43feacbbb5b") + clusterName, ok = getECSClusterName(resourceAttributes) + assert.True(t, ok) + assert.Equal(t, "my-cluster", clusterName) +} diff --git a/plugins/processors/awsappsignals/internal/resolver/ec2.go b/plugins/processors/awsappsignals/internal/resolver/ec2.go deleted file mode 100644 index f856f4b5d4..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/ec2.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -import ( - "context" - - "go.opentelemetry.io/collector/pdata/pcommon" - - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -const AttributePlatformEC2 = "EC2" - -// EC2HostedInAttributes is an allow-list that also renames attributes from the resource detection processor -var EC2HostedInAttributes = map[string]string{ - attr.ResourceDetectionHostId: attr.EC2InstanceId, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, - attr.ResourceDetectionASG: attr.EC2AutoScalingGroupName, -} - -type ec2HostedInAttributeResolver struct { - name string - attributeMap map[string]string -} - -func newEC2HostedInAttributeResolver(name string) *ec2HostedInAttributeResolver { - if name == "" { - name = AttributePlatformEC2 - } - return &ec2HostedInAttributeResolver{ - name: name, - attributeMap: EC2HostedInAttributes, - } -} -func (h *ec2HostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { - for attrKey, mappingKey := range h.attributeMap { - if val, ok := resourceAttributes.Get(attrKey); ok { - attributes.PutStr(mappingKey, val.AsString()) - } - } - - // If aws.hostedin.environment is populated, override HostedIn.EC2.Environment value - // Otherwise, keep ASG name if it exists - if val, ok := resourceAttributes.Get(attr.AWSHostedInEnvironment); ok { - attributes.PutStr(attr.HostedInEC2Environment, val.AsString()) - } else if val, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(attr.HostedInEC2Environment, val.AsString()) - } else { - attributes.PutStr(attr.HostedInEC2Environment, h.name) - } - - return nil -} - -func (h *ec2HostedInAttributeResolver) Stop(ctx context.Context) error { - return nil -} diff --git a/plugins/processors/awsappsignals/internal/resolver/ec2_test.go b/plugins/processors/awsappsignals/internal/resolver/ec2_test.go deleted file mode 100644 index d65c80c897..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/ec2_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "go.opentelemetry.io/collector/pdata/pcommon" - - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_NoASG_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, AttributePlatformEC2, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_ASGExists_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - asgName := "ASG" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionASG, asgName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, asgName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_NoASG_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, "test", envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_ASGExists_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - asgName := "ASG" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionASG, asgName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, asgName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_NoASG_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_NoASG_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_ASGExists_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - asgName := "ASG" - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.EC2AutoScalingGroupName, asgName) - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_ASGExists_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - asgName := "ASG" - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.EC2AutoScalingGroupName, asgName) - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithResourceDetectionAttributes(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionHostId, "hostid") - resourceAttributes.PutStr(attr.ResourceDetectionHostName, "hostname") - resourceAttributes.PutStr(attr.ResourceDetectionASG, "asg") - - resolver.Process(attributes, resourceAttributes) - expectedInstanceId, ok := attributes.Get(attr.EC2InstanceId) - assert.True(t, ok) - assert.Equal(t, "hostid", expectedInstanceId.AsString()) - - expectedHostName, ok := attributes.Get(attr.ResourceDetectionHostName) - assert.True(t, ok) - assert.Equal(t, "hostname", expectedHostName.AsString()) - - expectedASG, ok := attributes.Get(attr.EC2AutoScalingGroupName) - assert.True(t, ok) - assert.Equal(t, "asg", expectedASG.AsString()) -} diff --git a/plugins/processors/awsappsignals/internal/resolver/kubernetes.go b/plugins/processors/awsappsignals/internal/resolver/kubernetes.go index a87d8fdc0a..b0e41f7ed3 100644 --- a/plugins/processors/awsappsignals/internal/resolver/kubernetes.go +++ b/plugins/processors/awsappsignals/internal/resolver/kubernetes.go @@ -25,8 +25,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" - "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" ) const ( @@ -48,13 +49,6 @@ const ( jitterKubernetesAPISeconds = 10 ) -var kubernetesHostedInAttributeMap = map[string]string{ - semconv.AttributeK8SNamespaceName: attr.HostedInK8SNamespace, - attr.ResourceDetectionHostId: attr.EC2InstanceId, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, - attr.ResourceDetectionASG: attr.EC2AutoScalingGroupName, -} - var ( // ReplicaSet name = Deployment name + "-" + up to 10 alphanumeric characters string, if the ReplicaSet was created through a deployment // The suffix string of the ReplicaSet name is an int32 number (0 to 4,294,967,295) that is cast to a string and then @@ -74,6 +68,8 @@ var ( type kubernetesResolver struct { logger *zap.Logger clientset kubernetes.Interface + clusterName string + platformCode string ipToPod *sync.Map podToWorkloadAndNamespace *sync.Map ipToServiceAndNamespace *sync.Map @@ -512,7 +508,7 @@ func (m *ServiceToWorkloadMapper) Start(stopCh chan struct{}) { }() } -func getKubernetesResolver(logger *zap.Logger) subResolver { +func getKubernetesResolver(platformCode, clusterName string, logger *zap.Logger) subResolver { once.Do(func() { config, err := clientcmd.BuildConfigFromFlags("", "") if err != nil { @@ -550,6 +546,8 @@ func getKubernetesResolver(logger *zap.Logger) subResolver { instance = &kubernetesResolver{ logger: logger, clientset: clientset, + clusterName: clusterName, + platformCode: platformCode, ipToServiceAndNamespace: serviceWatcher.ipToServiceAndNamespace, serviceAndNamespaceToSelectors: serviceWatcher.serviceAndNamespaceToSelectors, ipToPod: podWatcher.ipToPod, @@ -592,13 +590,14 @@ func (e *kubernetesResolver) GetWorkloadAndNamespaceByIP(ip string) (string, str } func (e *kubernetesResolver) Process(attributes, resourceAttributes pcommon.Map) error { + var namespace string if value, ok := attributes.Get(attr.AWSRemoteService); ok { valueStr := value.AsString() ipStr := "" if ip, _, ok := extractIPPort(valueStr); ok { - if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { + if workload, ns, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { attributes.PutStr(attr.AWSRemoteService, workload) - attributes.PutStr(attr.K8SRemoteNamespace, namespace) + namespace = ns } else { ipStr = ip } @@ -607,9 +606,9 @@ func (e *kubernetesResolver) Process(attributes, resourceAttributes pcommon.Map) } if ipStr != "" { - if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { + if workload, ns, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { attributes.PutStr(attr.AWSRemoteService, workload) - attributes.PutStr(attr.K8SRemoteNamespace, namespace) + namespace = ns } else { e.logger.Debug("failed to Process ip", zap.String("ip", ipStr), zap.Error(err)) attributes.PutStr(attr.AWSRemoteService, "UnknownRemoteService") @@ -617,6 +616,12 @@ func (e *kubernetesResolver) Process(attributes, resourceAttributes pcommon.Map) } } + if _, ok := attributes.Get(attr.AWSRemoteEnvironment); !ok { + if namespace != "" { + attributes.PutStr(attr.AWSRemoteEnvironment, fmt.Sprintf("%s:%s/%s", e.platformCode, e.clusterName, namespace)) + } + } + return nil } @@ -662,30 +667,50 @@ func getHostNetworkPorts(pod *corev1.Pod) []string { return ports } -type kubernetesHostedInAttributeResolver struct { +type kubernetesResourceAttributesResolver struct { + platformCode string clusterName string attributeMap map[string]string } -func newKubernetesHostedInAttributeResolver(clusterName string) *kubernetesHostedInAttributeResolver { - return &kubernetesHostedInAttributeResolver{ +func newKubernetesResourceAttributesResolver(platformCode, clusterName string) *kubernetesResourceAttributesResolver { + return &kubernetesResourceAttributesResolver{ + platformCode: platformCode, clusterName: clusterName, - attributeMap: kubernetesHostedInAttributeMap, + attributeMap: DefaultInheritedAttributes, } } -func (h *kubernetesHostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { +func (h *kubernetesResourceAttributesResolver) Process(attributes, resourceAttributes pcommon.Map) error { for attrKey, mappingKey := range h.attributeMap { if val, ok := resourceAttributes.Get(attrKey); ok { attributes.PutStr(mappingKey, val.AsString()) } } + if h.platformCode == config.PlatformEKS { + attributes.PutStr(common.AttributePlatformType, AttributePlatformEKS) + attributes.PutStr(common.AttributeEKSClusterName, h.clusterName) + } else { + attributes.PutStr(common.AttributePlatformType, AttributePlatformK8S) + attributes.PutStr(common.AttributeK8SClusterName, h.clusterName) + } + var namespace string + if nsAttr, ok := resourceAttributes.Get(semconv.AttributeK8SNamespaceName); ok { + namespace = nsAttr.Str() + } else { + namespace = "UnknownNamespace" + } - if isEks := eksdetector.IsEKS(); isEks.Value { - attributes.PutStr(attr.HostedInClusterNameEKS, h.clusterName) + if val, ok := resourceAttributes.Get(semconv.AttributeDeploymentEnvironment); !ok { + env := GetDefaultEnvironment(h.platformCode, h.clusterName+"/"+namespace) + if len(env) > 255 { + env = env[:255] + } + attributes.PutStr(common.MetricAttributeEnvironment, env) } else { - attributes.PutStr(attr.HostedInClusterNameK8s, h.clusterName) + attributes.PutStr(common.MetricAttributeEnvironment, val.Str()) } + attributes.PutStr(common.AttributeK8SNamespace, namespace) //The application log group in Container Insights is a fixed pattern: // "/aws/containerinsights/{Cluster_Name}/application" // See https://github.com/aws/amazon-cloudwatch-agent-operator/blob/fe144bb02d7b1930715aa3ea32e57a5ff13406aa/helm/templates/fluent-bit-configmap.yaml#L82 @@ -695,6 +720,6 @@ func (h *kubernetesHostedInAttributeResolver) Process(attributes, resourceAttrib return nil } -func (h *kubernetesHostedInAttributeResolver) Stop(ctx context.Context) error { +func (h *kubernetesResourceAttributesResolver) Stop(ctx context.Context) error { return nil } diff --git a/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go b/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go index c21b8ee2ab..c7855b4598 100644 --- a/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go +++ b/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go @@ -5,6 +5,7 @@ package resolver import ( "context" + "fmt" "strings" "sync" "testing" @@ -18,6 +19,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" ) @@ -697,6 +700,7 @@ func TestEksResolver(t *testing.T) { t.Run("Test GetWorkloadAndNamespaceByIP", func(t *testing.T) { resolver := &kubernetesResolver{ logger: logger, + clusterName: "test", ipToPod: &sync.Map{}, podToWorkloadAndNamespace: &sync.Map{}, ipToServiceAndNamespace: &sync.Map{}, @@ -770,6 +774,8 @@ func TestEksResolver(t *testing.T) { logger, _ := zap.NewProduction() resolver := &kubernetesResolver{ logger: logger, + clusterName: "test", + platformCode: config.PlatformEKS, ipToPod: &sync.Map{}, podToWorkloadAndNamespace: &sync.Map{}, ipToServiceAndNamespace: &sync.Map{}, @@ -785,7 +791,7 @@ func TestEksResolver(t *testing.T) { err := resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) assert.Equal(t, "test-deployment", getStrAttr(attributes, attr.AWSRemoteService, t)) - assert.Equal(t, "test-namespace", getStrAttr(attributes, attr.K8SRemoteNamespace, t)) + assert.Equal(t, "eks:test/test-namespace", getStrAttr(attributes, attr.AWSRemoteEnvironment, t)) // Test case 2: "aws.remote.service" contains only IP attributes = pcommon.NewMap() @@ -796,7 +802,7 @@ func TestEksResolver(t *testing.T) { err = resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) assert.Equal(t, "test-deployment-2", getStrAttr(attributes, attr.AWSRemoteService, t)) - assert.Equal(t, "test-namespace-2", getStrAttr(attributes, attr.K8SRemoteNamespace, t)) + assert.Equal(t, "eks:test/test-namespace-2", getStrAttr(attributes, attr.AWSRemoteEnvironment, t)) // Test case 3: "aws.remote.service" contains non-ip string attributes = pcommon.NewMap() @@ -816,7 +822,7 @@ func TestEksResolver(t *testing.T) { }) } -func TestHostedInEksResolver(t *testing.T) { +func TestK8sResourceAttributesResolverOnEKS(t *testing.T) { eksdetector.NewDetector = eksdetector.TestEKSDetector eksdetector.IsEKS = eksdetector.TestIsEKSCacheEKS // helper function to get string values from the attributes @@ -829,27 +835,71 @@ func TestHostedInEksResolver(t *testing.T) { } } - resolver := newKubernetesHostedInAttributeResolver("test-cluster") - - // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr("cloud.provider", "aws") - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("host.id", "instance-id") - resourceAttributes.PutStr("host.name", "hostname") - resourceAttributes.PutStr("ec2.tag.aws:autoscaling:groupName", "asg") - err := resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterNameEKS, t)) - assert.Equal(t, "instance-id", getStrAttr(attributes, attr.EC2InstanceId, t)) - assert.Equal(t, "hostname", getStrAttr(attributes, attr.ResourceDetectionHostName, t)) - assert.Equal(t, "asg", getStrAttr(attributes, attr.EC2AutoScalingGroupName, t)) - assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + resolver := newKubernetesResourceAttributesResolver(config.PlatformEKS, "test-cluster") + + resourceAttributesBase := map[string]string{ + "cloud.provider": "aws", + "k8s.namespace.name": "test-namespace-3", + "host.id": "instance-id", + "host.name": "hostname", + "ec2.tag.aws:autoscaling:groupName": "asg", + } + + tests := []struct { + name string + resourceAttributesOverwrite map[string]string + expectedAttributes map[string]string + }{ + { + "testDefault", + map[string]string{}, + + map[string]string{ + common.MetricAttributeEnvironment: "eks:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeEKSClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + attr.ResourceDetectionHostName: "hostname", + common.AttributeEC2AutoScalingGroupName: "asg", + }, + }, + { + "testOverwrite", + map[string]string{ + semconv.AttributeDeploymentEnvironment: "custom-env", + }, + map[string]string{ + common.MetricAttributeEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeEKSClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + attr.ResourceDetectionHostName: "hostname", + common.AttributeEC2AutoScalingGroupName: "asg", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + for key, val := range resourceAttributesBase { + resourceAttributes.PutStr(key, val) + } + for key, val := range tt.resourceAttributesOverwrite { + resourceAttributes.PutStr(key, val) + } + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + + for key, val := range tt.expectedAttributes { + assert.Equal(t, val, getStrAttr(attributes, key, t), fmt.Sprintf("expected %s for key %s", val, key)) + } + assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + }) + } } -func TestHostedInNativeK8sEC2Resolver(t *testing.T) { +func TestK8sResourceAttributesResolverOnK8S(t *testing.T) { eksdetector.NewDetector = eksdetector.TestK8sDetector eksdetector.IsEKS = eksdetector.TestIsEKSCacheK8s // helper function to get string values from the attributes @@ -862,27 +912,71 @@ func TestHostedInNativeK8sEC2Resolver(t *testing.T) { } } - resolver := newKubernetesHostedInAttributeResolver("test-cluster") - - // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr("cloud.provider", "aws") - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("host.id", "instance-id") - resourceAttributes.PutStr("host.name", "hostname") - resourceAttributes.PutStr("ec2.tag.aws:autoscaling:groupName", "asg") - err := resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterNameK8s, t)) - assert.Equal(t, "instance-id", getStrAttr(attributes, attr.EC2InstanceId, t)) - assert.Equal(t, "hostname", getStrAttr(attributes, attr.ResourceDetectionHostName, t)) - assert.Equal(t, "asg", getStrAttr(attributes, attr.EC2AutoScalingGroupName, t)) - assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + resolver := newKubernetesResourceAttributesResolver(config.PlatformK8s, "test-cluster") + + resourceAttributesBase := map[string]string{ + "cloud.provider": "aws", + "k8s.namespace.name": "test-namespace-3", + "host.id": "instance-id", + "host.name": "hostname", + "ec2.tag.aws:autoscaling:groupName": "asg", + } + + tests := []struct { + name string + resourceAttributesOverwrite map[string]string + expectedAttributes map[string]string + }{ + { + "testDefaultOnK8s", + map[string]string{}, + + map[string]string{ + common.MetricAttributeEnvironment: "k8s:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + attr.ResourceDetectionHostName: "hostname", + common.AttributeEC2AutoScalingGroupName: "asg", + }, + }, + { + "testOverwriteOnK8s", + map[string]string{ + semconv.AttributeDeploymentEnvironment: "custom-env", + }, + map[string]string{ + common.MetricAttributeEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + attr.ResourceDetectionHostName: "hostname", + common.AttributeEC2AutoScalingGroupName: "asg", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + for key, val := range resourceAttributesBase { + resourceAttributes.PutStr(key, val) + } + for key, val := range tt.resourceAttributesOverwrite { + resourceAttributes.PutStr(key, val) + } + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + + for key, val := range tt.expectedAttributes { + assert.Equal(t, val, getStrAttr(attributes, key, t), fmt.Sprintf("expected %s for key %s", val, key)) + } + assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + }) + } } -func TestHostedInNativeK8sOnPremResolver(t *testing.T) { +func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { eksdetector.NewDetector = eksdetector.TestK8sDetector // helper function to get string values from the attributes getStrAttr := func(attributes pcommon.Map, key string, t *testing.T) string { @@ -894,27 +988,69 @@ func TestHostedInNativeK8sOnPremResolver(t *testing.T) { } } - resolver := newKubernetesHostedInAttributeResolver("test-cluster") - - // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr("cloud.provider", "aws") - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("host.name", "hostname") - err := resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterNameK8s, t)) - assert.Equal(t, "hostname", getStrAttr(attributes, attr.ResourceDetectionHostName, t)) - assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) - - // EC2 related fields that should not exist for on-prem - _, exists := attributes.Get(attr.EC2AutoScalingGroupName) - assert.False(t, exists) - - _, exists = attributes.Get(attr.EC2InstanceId) - assert.False(t, exists) + resolver := newKubernetesResourceAttributesResolver(config.PlatformK8s, "test-cluster") + + resourceAttributesBase := map[string]string{ + "cloud.provider": "aws", + "k8s.namespace.name": "test-namespace-3", + "host.name": "hostname", + } + + tests := []struct { + name string + resourceAttributesOverwrite map[string]string + expectedAttributes map[string]string + }{ + { + "testDefault", + map[string]string{}, + + map[string]string{ + common.MetricAttributeEnvironment: "k8s:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + attr.ResourceDetectionHostName: "hostname", + }, + }, + { + "testOverwrite", + map[string]string{ + semconv.AttributeDeploymentEnvironment: "custom-env", + }, + map[string]string{ + common.MetricAttributeEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + attr.ResourceDetectionHostName: "hostname", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + for key, val := range resourceAttributesBase { + resourceAttributes.PutStr(key, val) + } + for key, val := range tt.resourceAttributesOverwrite { + resourceAttributes.PutStr(key, val) + } + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + + for key, val := range tt.expectedAttributes { + assert.Equal(t, val, getStrAttr(attributes, key, t), fmt.Sprintf("expected %s for key %s", val, key)) + } + assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + + // EC2 related fields that should not exist for on-prem + _, exists := attributes.Get(common.AttributeEC2AutoScalingGroupName) + assert.False(t, exists) + + _, exists = attributes.Get(common.AttributeEC2InstanceId) + assert.False(t, exists) + }) + } } func TestExtractIPPort(t *testing.T) { diff --git a/plugins/processors/awsappsignals/rules/common.go b/plugins/processors/awsappsignals/rules/common.go index 4ac6ca8c14..4c4c319de8 100644 --- a/plugins/processors/awsappsignals/rules/common.go +++ b/plugins/processors/awsappsignals/rules/common.go @@ -8,6 +8,9 @@ import ( "github.com/gobwas/glob" "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) type AllowListAction string @@ -46,10 +49,14 @@ type ActionItem struct { } var traceKeyMap = map[string]string{ - "Service": "aws.local.service", - "Operation": "aws.local.operation", - "RemoteService": "aws.remote.service", - "RemoteOperation": "aws.remote.operation", + common.MetricAttributeLocalService: attributes.AWSLocalService, + common.MetricAttributeEnvironment: attributes.AWSLocalEnvironment, + common.MetricAttributeLocalOperation: attributes.AWSLocalOperation, + common.MetricAttributeRemoteService: attributes.AWSRemoteService, + common.MetricAttributeRemoteEnvironment: attributes.AWSRemoteEnvironment, + common.MetricAttributeRemoteOperation: attributes.AWSRemoteOperation, + common.MetricAttributeRemoteResourceIdentifier: attributes.AWSRemoteResourceIdentifier, + common.MetricAttributeRemoteResourceType: attributes.AWSRemoteResourceType, } func GetAllowListAction(action string) (AllowListAction, error) { @@ -64,21 +71,17 @@ func GetAllowListAction(action string) (AllowListAction, error) { return "", errors.New("invalid action in rule") } -func getExactKey(metricDimensionKey string, isTrace bool) string { - if !isTrace { - return metricDimensionKey - } - traceDimensionKey, ok := traceKeyMap[metricDimensionKey] - if !ok { - // return original key if there is no matches - return metricDimensionKey +func convertToManagedAttributeKey(attributeKey string, isTrace bool) string { + val, ok := traceKeyMap[attributeKey] + if ok && isTrace { + return val } - return traceDimensionKey + return attributeKey } func matchesSelectors(attributes pcommon.Map, selectorMatchers []SelectorMatcherItem, isTrace bool) bool { for _, item := range selectorMatchers { - exactKey := getExactKey(item.Key, isTrace) + exactKey := convertToManagedAttributeKey(item.Key, isTrace) value, ok := attributes.Get(exactKey) if !ok { return false diff --git a/plugins/processors/awsappsignals/rules/common_test.go b/plugins/processors/awsappsignals/rules/common_test.go index e2ab0fe9a1..1433023a1f 100644 --- a/plugins/processors/awsappsignals/rules/common_test.go +++ b/plugins/processors/awsappsignals/rules/common_test.go @@ -3,21 +3,44 @@ package rules -import "go.opentelemetry.io/collector/pdata/pcommon" +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" +) func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, + isTrace bool) pcommon.Map { + return generateAttributesWithEnv(service, operation, "", remoteService, remoteOperation, "", isTrace) +} + +func generateAttributesWithEnv(service string, operation string, environment string, + remoteService string, remoteOperation string, remoteEnvironment string, isTrace bool) pcommon.Map { attributes := pcommon.NewMap() if isTrace { - attributes.PutStr("aws.local.service", service) - attributes.PutStr("aws.local.operation", operation) - attributes.PutStr("aws.remote.service", remoteService) - attributes.PutStr("aws.remote.operation", remoteOperation) + attributes.PutStr(attr.AWSLocalService, service) + attributes.PutStr(attr.AWSLocalOperation, operation) + if environment != "" { + attributes.PutStr(attr.AWSLocalEnvironment, environment) + } + attributes.PutStr(attr.AWSRemoteService, remoteService) + attributes.PutStr(attr.AWSRemoteOperation, remoteOperation) + if remoteEnvironment != "" { + attributes.PutStr(attr.AWSRemoteEnvironment, remoteEnvironment) + } } else { - attributes.PutStr("Service", service) - attributes.PutStr("Operation", operation) - attributes.PutStr("RemoteService", remoteService) - attributes.PutStr("RemoteOperation", remoteOperation) + attributes.PutStr(common.MetricAttributeLocalService, service) + attributes.PutStr(common.MetricAttributeLocalOperation, operation) + if environment != "" { + attributes.PutStr(common.MetricAttributeEnvironment, environment) + } + attributes.PutStr(common.MetricAttributeRemoteService, remoteService) + attributes.PutStr(common.MetricAttributeRemoteOperation, remoteOperation) + if remoteEnvironment != "" { + attributes.PutStr(common.MetricAttributeRemoteEnvironment, remoteEnvironment) + } } return attributes } diff --git a/plugins/processors/awsappsignals/rules/replacer.go b/plugins/processors/awsappsignals/rules/replacer.go index de3b392c6b..fbca29d5d1 100644 --- a/plugins/processors/awsappsignals/rules/replacer.go +++ b/plugins/processors/awsappsignals/rules/replacer.go @@ -36,18 +36,13 @@ func (r *ReplaceActions) Process(attributes, _ pcommon.Map, isTrace bool) error continue } for _, replacement := range element.Replacements { - targetDimensionKey := getExactKey(replacement.TargetDimension, isTrace) - // don't allow customer add new dimension key - _, isExist := attributes.Get(targetDimensionKey) - if !isExist { - continue - } + targetDimension := replacement.TargetDimension + + attr := convertToManagedAttributeKey(targetDimension, isTrace) // every replacement in one specific dimension only will be performed once - _, ok := finalRules[targetDimensionKey] - if ok { - continue + if _, visited := finalRules[attr]; !visited { + finalRules[attr] = replacement.Value } - finalRules[targetDimensionKey] = replacement.Value } } diff --git a/plugins/processors/awsappsignals/rules/replacer_test.go b/plugins/processors/awsappsignals/rules/replacer_test.go index d86bc2ab96..d777194550 100644 --- a/plugins/processors/awsappsignals/rules/replacer_test.go +++ b/plugins/processors/awsappsignals/rules/replacer_test.go @@ -119,6 +119,77 @@ func TestReplacerProcess(t *testing.T) { } } +func TestAddManagedDimensionKey(t *testing.T) { + config := []Rule{ + { + Selectors: []Selector{ + { + Dimension: "Service", + Match: "app", + }, + { + Dimension: "RemoteService", + Match: "remote-app", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteEnvironment", + Value: "test", + }, + }, + Action: "replace", + }, + } + + testReplacer := NewReplacer(config, false) + assert.Equal(t, 1, len(testReplacer.Actions)) + + testCases := []TestCaseForReplacer{ + { + name: "testAddMissingRemoteEnvironmentInMetric", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "", false), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", false), + isTrace: false, + }, + { + name: "testAddMissingRemoteEnvironmentInTrace", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "", true), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", true), + isTrace: true, + }, + { + name: "testReplaceRemoteEnvironmentInMetric", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "error", false), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", false), + isTrace: false, + }, + { + name: "testReplaceRemoteEnvironmentInTrace", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "error", true), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", true), + isTrace: true, + }, + } + + testMapPlaceHolder := pcommon.NewMap() + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, testReplacer.Process(tt.input, testMapPlaceHolder, tt.isTrace)) + assert.Equal(t, tt.output, tt.input) + }) + } +} + func TestReplacerProcessWithPriority(t *testing.T) { config := []Rule{ diff --git a/translator/config/mode.go b/translator/config/mode.go index a5cb44f492..5f8c8f93ed 100644 --- a/translator/config/mode.go +++ b/translator/config/mode.go @@ -10,6 +10,10 @@ const ( ModeWithIRSA = "withIRSA" ) +const ( + ModeECS = "ECS" +) + const ( ModeEKS = "EKS" ModeK8sEC2 = "K8sEC2" diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 3e9061946c..e88c901e39 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -9,18 +9,16 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/eks + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: @@ -32,74 +30,69 @@ exporters: - Fault - Error - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.EKS.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index 23342166e1..59470c97af 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -9,18 +9,16 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/k8s + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: @@ -32,74 +30,69 @@ exporters: - Fault - Error - dimensions: - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.K8s.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces @@ -302,8 +294,8 @@ processors: log_dropped_metrics: true rotation_interval: 10m0s resolvers: - - name: TestCluster - platform: k8s + - name: TestCluster + platform: k8s batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index cf66cc4d5e..88a432f7bd 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -9,18 +9,16 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/eks + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: @@ -32,74 +30,69 @@ exporters: - Fault - Error - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.EKS.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 45822ef84e..5ae1913f53 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -9,18 +9,16 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/eks + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: @@ -32,74 +30,69 @@ exporters: - Fault - Error - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.EKS.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index f065a5b739..9ef457bb94 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -9,16 +9,16 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: true - log_group_name: /aws/appsignals/generic + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.Environment + - - Environment - Operation - Service - - - HostedIn.Environment + - - Environment - Service label_matchers: - label_names: @@ -30,35 +30,39 @@ exporters: - Fault - Error - dimensions: - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: @@ -94,10 +98,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - aws.remote.target - - HostedIn.Environment + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: true max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml index 3901d23a85..c7a93c9f27 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml @@ -9,16 +9,16 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: true - log_group_name: /aws/appsignals/generic + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.Environment + - - Environment - Operation - Service - - - HostedIn.Environment + - - Environment - Service label_matchers: - label_names: @@ -30,35 +30,39 @@ exporters: - Fault - Error - dimensions: - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: @@ -94,10 +98,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - aws.remote.target - - HostedIn.Environment + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: true max_retries: 2 middleware: agenthealth/traces diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml deleted file mode 100644 index a128be9a5a..0000000000 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml +++ /dev/null @@ -1,33 +0,0 @@ -log_group_name: "/aws/appsignals/ec2" -namespace: "AppSignals" -middleware: agenthealth/logs -dimension_rollup_option: "NoDimensionRollup" -metric_declarations: - - dimensions: - - [HostedIn.EC2.Environment, Service, Operation] - - [HostedIn.EC2.Environment, Service] - label_matchers: - - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' - metric_name_selectors: - - Latency - - Fault - - Error - - dimensions: - - [HostedIn.EC2.Environment, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EC2.Environment, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.EC2.Environment, Service, RemoteService] - - [HostedIn.EC2.Environment, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EC2.Environment, Service, RemoteService, RemoteOperation] - - [HostedIn.EC2.Environment, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] - - [RemoteService] - label_matchers: - - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' - metric_name_selectors: - - Latency - - Fault - - Error \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml index 7171c91dac..7e5122e6b5 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml @@ -1,11 +1,11 @@ -log_group_name: "/aws/appsignals/eks" +log_group_name: "/aws/application-signals/data" namespace: "AppSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service] + - [Environment, Service, Operation] + - [Environment, Service] label_matchers: - label_names: - aws.span.kind @@ -15,18 +15,18 @@ metric_declarations: - Fault - Error - dimensions: - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteEnvironment] + - [Environment, Service, RemoteService] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteResourceIdentifier, RemoteResourceType] + - [RemoteService, RemoteResourceIdentifier, RemoteResourceType] - [RemoteService] label_matchers: - label_names: diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml index 942faf6d81..ff4af39f81 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml @@ -1,11 +1,11 @@ -log_group_name: "/aws/appsignals/generic" +log_group_name: "/aws/application-signals/data" namespace: "AppSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: - - [HostedIn.Environment, Service, Operation] - - [HostedIn.Environment, Service] + - [Environment, Service, Operation] + - [Environment, Service] label_matchers: - label_names: - aws.span.kind @@ -15,13 +15,13 @@ metric_declarations: - Fault - Error - dimensions: - - [HostedIn.Environment, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.Environment, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.Environment, Service, RemoteService] - - [HostedIn.Environment, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.Environment, Service, RemoteService, RemoteOperation] - - [HostedIn.Environment, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService] + - [Environment, Service, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteResourceIdentifier, RemoteResourceType] + - [RemoteService, RemoteResourceIdentifier, RemoteResourceType] - [RemoteService] label_matchers: - label_names: diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml index d4bf0794bd..7e5122e6b5 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml @@ -1,11 +1,11 @@ -log_group_name: "/aws/appsignals/k8s" +log_group_name: "/aws/application-signals/data" namespace: "AppSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service] + - [Environment, Service, Operation] + - [Environment, Service] label_matchers: - label_names: - aws.span.kind @@ -15,18 +15,18 @@ metric_declarations: - Fault - Error - dimensions: - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteEnvironment] + - [Environment, Service, RemoteService] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteResourceIdentifier, RemoteResourceType] + - [RemoteService, RemoteResourceIdentifier, RemoteResourceType] - [RemoteService] label_matchers: - label_names: diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 7b82a3d463..bc130f3d54 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -22,6 +22,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) //go:embed awsemf_default_ecs.yaml @@ -42,9 +43,6 @@ var appSignalsConfigK8s string //go:embed appsignals_config_generic.yaml var appSignalsConfigGeneric string -//go:embed appsignals_config_ec2.yaml -var appSignalsConfigEC2 string - var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) @@ -142,16 +140,27 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { func getAppSignalsConfig() string { ctx := context.CurrentContext() - kubernetesMode := ctx.KubernetesMode() - if kubernetesMode == config.ModeEKS { + mode := ctx.KubernetesMode() + if mode == "" { + mode = ctx.Mode() + } + if mode == config.ModeEC2 { + if ecsutil.GetECSUtilSingleton().IsECS() { + mode = config.ModeECS + } + } + + switch mode { + case config.ModeEKS: return appSignalsConfigEks - } else if kubernetesMode == config.ModeK8sEC2 || kubernetesMode == config.ModeK8sOnPrem { + case config.ModeK8sEC2, config.ModeK8sOnPrem: return appSignalsConfigK8s - } else if ctx.Mode() == config.ModeEC2 { - return appSignalsConfigEC2 + case config.ModeEC2, config.ModeECS: + return appSignalsConfigGeneric + default: + return appSignalsConfigGeneric } - return appSignalsConfigGeneric } func (t *translator) isAppSignals(conf *confmap.Conf) bool { diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 3ddde1f50a..73ed9d41d6 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -949,7 +949,7 @@ func TestTranslateAppSignals(t *testing.T) { "app_signals": map[string]any{}, }, }}, - want: testutil.GetConfWithOverrides(t, filepath.Join("appsignals_config_ec2.yaml"), map[string]any{ + want: testutil.GetConfWithOverrides(t, filepath.Join("appsignals_config_generic.yaml"), map[string]any{ "local_mode": "false", "region": "us-east-1", "role_arn": "global_arn", diff --git a/translator/translate/otel/exporter/awsxray/translator.go b/translator/translate/otel/exporter/awsxray/translator.go index 5708558d25..3754080f86 100644 --- a/translator/translate/otel/exporter/awsxray/translator.go +++ b/translator/translate/otel/exporter/awsxray/translator.go @@ -35,26 +35,10 @@ type translator struct { var _ common.Translator[component.Config] = (*translator)(nil) var ( - indexedAttributesEKS = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", - "HostedIn.Environment", "HostedIn.EKS.Cluster", - } - - indexedAttributesK8s = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", - "HostedIn.Environment", "HostedIn.K8s.Cluster", - } - - indexedAttributesEC2 = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "HostedIn.EC2.Environment", "aws.remote.target", - } - - indexedAttributesGeneric = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", "aws.remote.target", - "HostedIn.Environment", + indexedAttributes = []string{ + "aws.local.service", "aws.local.operation", "aws.local.environment", + "aws.remote.service", "aws.remote.operation", "aws.remote.environment", + "aws.remote.resource.identifier", "aws.remote.resource.type", } ) @@ -80,16 +64,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*awsxrayexporter.Config) if isAppSignals(conf) { - ctx := context.CurrentContext() - if ctx.KubernetesMode() == config.ModeEKS { - cfg.IndexedAttributes = indexedAttributesEKS - } else if ctx.KubernetesMode() == config.ModeK8sEC2 || ctx.KubernetesMode() == config.ModeK8sOnPrem { - cfg.IndexedAttributes = indexedAttributesK8s - } else if ctx.Mode() == config.ModeEC2 { - cfg.IndexedAttributes = indexedAttributesEC2 - } else { - cfg.IndexedAttributes = indexedAttributesGeneric - } + cfg.IndexedAttributes = indexedAttributes } c := confmap.NewFromStringMap(map[string]interface{}{ diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index 44fab012db..e59f1c5672 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -74,13 +74,12 @@ func TestTranslator(t *testing.T) { "indexed_attributes": []string{ "aws.local.service", "aws.local.operation", + "aws.local.environment", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", - "K8s.RemoteNamespace", - "aws.remote.target", - "HostedIn.Environment", - "HostedIn.EKS.Cluster", + "aws.remote.environment", + "aws.remote.resource.identifier", + "aws.remote.resource.type", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", @@ -106,13 +105,12 @@ func TestTranslator(t *testing.T) { "indexed_attributes": []string{ "aws.local.service", "aws.local.operation", + "aws.local.environment", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", - "K8s.RemoteNamespace", - "aws.remote.target", - "HostedIn.Environment", - "HostedIn.K8s.Cluster", + "aws.remote.environment", + "aws.remote.resource.identifier", + "aws.remote.resource.type", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", @@ -138,10 +136,12 @@ func TestTranslator(t *testing.T) { "indexed_attributes": []string{ "aws.local.service", "aws.local.operation", + "aws.local.environment", "aws.remote.service", "aws.remote.operation", - "HostedIn.EC2.Environment", - "aws.remote.target", + "aws.remote.environment", + "aws.remote.resource.identifier", + "aws.remote.resource.type", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsappsignals/translator.go index 19f86dbfd8..bb1721060b 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsappsignals/translator.go @@ -19,6 +19,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) type translator struct { @@ -75,8 +76,16 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } } - kubernetesMode := context.CurrentContext().KubernetesMode() - switch kubernetesMode { + mode := context.CurrentContext().KubernetesMode() + if mode == "" { + mode = context.CurrentContext().Mode() + } + if mode == config.ModeEC2 { + if ecsutil.GetECSUtilSingleton().IsECS() { + mode = config.ModeECS + } + } + switch mode { case config.ModeEKS: cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewEKSResolver(hostedIn), @@ -85,18 +94,17 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewK8sResolver(hostedIn), } - } - - if kubernetesMode == "" { - switch context.CurrentContext().Mode() { - case config.ModeEC2: - cfg.Resolvers = []appsignalsconfig.Resolver{ - appsignalsconfig.NewEC2Resolver(hostedIn), - } - default: - cfg.Resolvers = []appsignalsconfig.Resolver{ - appsignalsconfig.NewGenericResolver(hostedIn), - } + case config.ModeEC2: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewEC2Resolver(hostedIn), + } + case config.ModeECS: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewECSResolver(hostedIn), + } + default: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewGenericResolver(hostedIn), } } diff --git a/translator/translate/otel/processor/awsappsignals/translator_test.go b/translator/translate/otel/processor/awsappsignals/translator_test.go index c1897b8601..a44fef61df 100644 --- a/translator/translate/otel/processor/awsappsignals/translator_test.go +++ b/translator/translate/otel/processor/awsappsignals/translator_test.go @@ -118,7 +118,7 @@ func TestTranslate(t *testing.T) { "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ "application_signals": map[string]interface{}{ - "hosted_in": "", + "hosted_in": "test", }, }, }}, From dc0e2a9cd8b930cf0018d05b928b0f4969212dcd Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Thu, 11 Apr 2024 14:43:28 -0700 Subject: [PATCH 03/11] 1. update metric namespace 2. code refactoring --- .../README.md | 36 ++++++------- .../common/types.go | 0 .../config/config.go | 6 ++- .../config/config_test.go | 0 .../config/resolvers.go | 9 ---- .../config/resolvers_test.go | 0 .../factory.go | 8 +-- .../factory_test.go | 6 +-- .../internal/attributes/attributes.go | 5 +- .../cardinalitycontrol/count_min_sketch.go | 0 .../count_min_sketch_test.go | 0 .../cardinalitycontrol/metrics_limiter.go | 4 +- .../metrics_limiter_test.go | 24 ++++----- .../normalizer/attributesnormalizer.go | 4 +- .../normalizer/attributesnormalizer_test.go | 0 .../internal/resolver/attributesresolver.go | 50 ++++++++++--------- .../resolver/attributesresolver_test.go | 25 +++------- .../internal/resolver/kubernetes.go | 6 +-- .../internal/resolver/kubernetes_test.go | 6 +-- .../processor.go | 26 +++++----- .../processor_test.go | 12 ++--- .../rules/common.go | 4 +- .../rules/common_test.go | 4 +- .../rules/dropper.go | 0 .../rules/dropper_test.go | 0 .../rules/keeper.go | 2 +- .../rules/keeper_test.go | 0 .../rules/replacer.go | 2 +- .../rules/replacer_test.go | 0 .../testdata/config_eks.yaml | 0 .../testdata/config_generic.yaml | 0 service/defaultcomponents/components.go | 4 +- .../appsignals_and_eks_config.yaml | 2 +- .../appsignals_and_k8s_config.yaml | 2 +- .../appsignals_fallback_and_eks_config.yaml | 2 +- .../appsignals_over_fallback_config.yaml | 2 +- .../sampleConfig/base_appsignals_config.yaml | 2 +- .../base_appsignals_fallback_config.yaml | 2 +- .../awsemf/appsignals_config_eks.yaml | 2 +- .../awsemf/appsignals_config_generic.yaml | 2 +- .../awsemf/appsignals_config_k8s.yaml | 2 +- .../translator.go | 6 +-- .../translator_test.go | 2 +- .../testdata/config_ec2.yaml | 0 .../testdata/config_eks.yaml | 0 .../testdata/config_generic.yaml | 0 .../testdata/config_k8s.yaml | 0 .../testdata/invalidRulesConfig.json | 0 .../testdata/validRulesConfig.json | 0 .../testdata/validRulesConfigEKS.yaml | 0 .../testdata/validRulesConfigGeneric.yaml | 0 .../translator.go | 12 ++--- .../translator_test.go | 12 ++--- translator/translate/otel/translate_otel.go | 6 +-- 54 files changed, 142 insertions(+), 157 deletions(-) rename plugins/processors/{awsappsignals => awsapplicationsignals}/README.md (92%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/common/types.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/config.go (93%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/config_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/resolvers.go (85%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/resolvers_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/factory.go (92%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/factory_test.go (97%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/attributes/attributes.go (87%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/count_min_sketch.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/count_min_sketch_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/metrics_limiter.go (99%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/metrics_limiter_test.go (90%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/normalizer/attributesnormalizer.go (98%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/normalizer/attributesnormalizer_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/resolver/attributesresolver.go (77%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/resolver/attributesresolver_test.go (94%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/resolver/kubernetes.go (99%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/resolver/kubernetes_test.go (99%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/processor.go (91%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/processor_test.go (97%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/common.go (98%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/common_test.go (96%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/dropper.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/dropper_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/keeper.go (97%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/keeper_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/replacer.go (98%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/replacer_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/testdata/config_eks.yaml (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/testdata/config_generic.yaml (100%) rename translator/translate/otel/pipeline/{appsignals => applicationsignals}/translator.go (93%) rename translator/translate/otel/pipeline/{appsignals => applicationsignals}/translator_test.go (99%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_ec2.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_eks.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_generic.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_k8s.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/invalidRulesConfig.json (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/validRulesConfig.json (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/validRulesConfigEKS.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/validRulesConfigGeneric.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/translator.go (97%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/translator_test.go (93%) diff --git a/plugins/processors/awsappsignals/README.md b/plugins/processors/awsapplicationsignals/README.md similarity index 92% rename from plugins/processors/awsappsignals/README.md rename to plugins/processors/awsapplicationsignals/README.md index 4906b69a46..36893c2528 100644 --- a/plugins/processors/awsappsignals/README.md +++ b/plugins/processors/awsapplicationsignals/README.md @@ -50,36 +50,36 @@ A replacements section defines a matching against the dimensions of incoming met ## AWS AppSignals Processor Configuration Example ```yaml -awsappsignals: +awsapplicationsignals: resolvers: ["eks"] rules: - selectors: - dimension: Operation - match: "POST *" + match: "POST *" - dimension: RemoteService - match: "*" - action: keep - rule_name: "keep01" + match: "*" + action: keep + rule_name: "keep01" - selectors: - dimension: Operation - match: "GET *" + match: "GET *" - dimension: RemoteService - match: "*" - action: keep - rule_name: "keep02" + match: "*" + action: keep + rule_name: "keep02" - selectors: - dimension: Operation - match: "POST *" - action: drop - rule_name: "drop01" + match: "POST *" + action: drop + rule_name: "drop01" - selectors: - dimension: Operation - match: "*" - replacements: - - target_dimension: RemoteOperation - value: "This is a test string" - action: replace - rule_name: "replace01" + match: "*" + replacements: + - target_dimension: RemoteOperation + value: "This is a test string" + action: replace + rule_name: "replace01" ``` ## Amazon CloudWatch Agent Configuration Example diff --git a/plugins/processors/awsappsignals/common/types.go b/plugins/processors/awsapplicationsignals/common/types.go similarity index 100% rename from plugins/processors/awsappsignals/common/types.go rename to plugins/processors/awsapplicationsignals/common/types.go diff --git a/plugins/processors/awsappsignals/config/config.go b/plugins/processors/awsapplicationsignals/config/config.go similarity index 93% rename from plugins/processors/awsappsignals/config/config.go rename to plugins/processors/awsapplicationsignals/config/config.go index facdefbf51..19d5faeddf 100644 --- a/plugins/processors/awsappsignals/config/config.go +++ b/plugins/processors/awsapplicationsignals/config/config.go @@ -8,7 +8,7 @@ import ( "errors" "time" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) type Config struct { @@ -62,7 +62,9 @@ func (cfg *Config) Validate() error { if resolver.Name == "" { return errors.New("name must not be empty for k8s resolver") } - case PlatformEC2, PlatformGeneric, PlatformECS, PlatformLambda: + case PlatformEC2, PlatformGeneric: + case PlatformECS: + return errors.New("ecs resolver is not supported") default: return errors.New("unknown resolver") } diff --git a/plugins/processors/awsappsignals/config/config_test.go b/plugins/processors/awsapplicationsignals/config/config_test.go similarity index 100% rename from plugins/processors/awsappsignals/config/config_test.go rename to plugins/processors/awsapplicationsignals/config/config_test.go diff --git a/plugins/processors/awsappsignals/config/resolvers.go b/plugins/processors/awsapplicationsignals/config/resolvers.go similarity index 85% rename from plugins/processors/awsappsignals/config/resolvers.go rename to plugins/processors/awsapplicationsignals/config/resolvers.go index 7f6291112f..2ad6e25bf4 100644 --- a/plugins/processors/awsappsignals/config/resolvers.go +++ b/plugins/processors/awsapplicationsignals/config/resolvers.go @@ -14,8 +14,6 @@ const ( PlatformEC2 = "ec2" // PlatformECS Amazon ECS PlatformECS = "ecs" - // PlatformLambda Amazon Lambda - PlatformLambda = "lambda" ) type Resolver struct { @@ -44,13 +42,6 @@ func NewEC2Resolver(name string) Resolver { } } -func NewECSResolver(name string) Resolver { - return Resolver{ - Name: name, - Platform: PlatformECS, - } -} - func NewGenericResolver(name string) Resolver { return Resolver{ Name: name, diff --git a/plugins/processors/awsappsignals/config/resolvers_test.go b/plugins/processors/awsapplicationsignals/config/resolvers_test.go similarity index 100% rename from plugins/processors/awsappsignals/config/resolvers_test.go rename to plugins/processors/awsapplicationsignals/config/resolvers_test.go diff --git a/plugins/processors/awsappsignals/factory.go b/plugins/processors/awsapplicationsignals/factory.go similarity index 92% rename from plugins/processors/awsappsignals/factory.go rename to plugins/processors/awsapplicationsignals/factory.go index 8a8b0a45a5..18b5cb5a54 100644 --- a/plugins/processors/awsappsignals/factory.go +++ b/plugins/processors/awsapplicationsignals/factory.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "context" @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/collector/processor/processorhelper" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" ) const ( @@ -89,12 +89,12 @@ func createMetricsProcessor( func createProcessor( params processor.CreateSettings, cfg component.Config, -) (*awsappsignalsprocessor, error) { +) (*awsapplicationsignalsprocessor, error) { pCfg, ok := cfg.(*appsignalsconfig.Config) if !ok { return nil, errors.New("could not initialize awsapplicationsignalsprocessor") } - ap := &awsappsignalsprocessor{logger: params.Logger, config: pCfg} + ap := &awsapplicationsignalsprocessor{logger: params.Logger, config: pCfg} return ap, nil } diff --git a/plugins/processors/awsappsignals/factory_test.go b/plugins/processors/awsapplicationsignals/factory_test.go similarity index 97% rename from plugins/processors/awsappsignals/factory_test.go rename to plugins/processors/awsapplicationsignals/factory_test.go index 25118acb35..29aa9472d3 100644 --- a/plugins/processors/awsappsignals/factory_test.go +++ b/plugins/processors/awsapplicationsignals/factory_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "path/filepath" @@ -12,8 +12,8 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) var expectedRules = []rules.Rule{ diff --git a/plugins/processors/awsappsignals/internal/attributes/attributes.go b/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go similarity index 87% rename from plugins/processors/awsappsignals/internal/attributes/attributes.go rename to plugins/processors/awsapplicationsignals/internal/attributes/attributes.go index fd9877cc3e..0740511ed6 100644 --- a/plugins/processors/awsappsignals/internal/attributes/attributes.go +++ b/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go @@ -6,9 +6,11 @@ package attributes const ( // aws attributes AWSLocalService = "aws.local.service" + AWSLocalEnvironment = "aws.local.environment" AWSLocalOperation = "aws.local.operation" AWSRemoteService = "aws.remote.service" AWSRemoteOperation = "aws.remote.operation" + AWSRemoteEnvironment = "aws.remote.environment" AWSRemoteTarget = "aws.remote.target" AWSRemoteResourceIdentifier = "aws.remote.resource.identifier" AWSRemoteResourceType = "aws.remote.resource.type" @@ -18,7 +20,4 @@ const ( ResourceDetectionHostId = "host.id" ResourceDetectionHostName = "host.name" ResourceDetectionASG = "ec2.tag.aws:autoscaling:groupName" - - AWSLocalEnvironment = "aws.local.environment" - AWSRemoteEnvironment = "aws.remote.environment" ) diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch.go similarity index 100% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch.go diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch_test.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch_test.go similarity index 100% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch_test.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch_test.go diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter.go similarity index 99% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter.go index b1309b8bb5..3b2efda288 100644 --- a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go +++ b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter.go @@ -13,8 +13,8 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" ) const ( diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go similarity index 90% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go index 9ce7112b20..aa9cb426cb 100644 --- a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go +++ b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go @@ -15,19 +15,19 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - awsappsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + awsapplicationsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" ) var emptyResourceAttributes = pcommon.NewMap() var logger, _ = zap.NewDevelopment() func TestAdmitAndRollup(t *testing.T) { - config := &awsappsignalsconfig.LimiterConfig{ + config := &awsapplicationsignalsconfig.LimiterConfig{ Threshold: 2, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -59,11 +59,11 @@ func TestAdmitAndRollup(t *testing.T) { } func TestAdmitByTopK(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 100, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -89,11 +89,11 @@ func TestAdmitByTopK(t *testing.T) { } func TestAdmitLowCardinalityAttributes(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -109,11 +109,11 @@ func TestAdmitLowCardinalityAttributes(t *testing.T) { } func TestAdmitReservedMetrics(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -140,7 +140,7 @@ func TestAdmitReservedMetrics(t *testing.T) { func TestClearStaleService(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: false, @@ -166,7 +166,7 @@ func TestClearStaleService(t *testing.T) { } func TestInheritanceAfterRotation(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: true, diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go similarity index 98% rename from plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go rename to plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go index 52fb8fbd59..9ba7a22e0a 100644 --- a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -12,8 +12,8 @@ import ( semconv "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) type attributesNormalizer struct { diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go similarity index 100% rename from plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go rename to plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go similarity index 77% rename from plugins/processors/awsappsignals/internal/resolver/attributesresolver.go rename to plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go index 1dd617aeb3..3a2941621e 100644 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -13,18 +13,19 @@ import ( semconv1 "go.opentelemetry.io/collector/semconv/v1.17.0" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) const ( AttributeEnvironmentDefault = "default" - AttributePlatformGeneric = "Generic" - AttributePlatformEC2 = "AWS::EC2" - AttributePlatformEKS = "AWS::EKS" - AttributePlatformLambda = "AWS::Lambda" - AttributePlatformK8S = "K8s" + + AttributePlatformGeneric = "Generic" + AttributePlatformEC2 = "AWS::EC2" + AttributePlatformEKS = "AWS::EKS" + AttributePlatformK8S = "K8s" ) var GenericInheritedAttributes = map[string]string{ @@ -53,7 +54,6 @@ type attributesResolver struct { // create a new attributes resolver func NewAttributesResolver(resolvers []appsignalsconfig.Resolver, logger *zap.Logger) *attributesResolver { - //TODO: Logic for native k8s needs to be implemented subResolvers := []subResolver{} for _, resolver := range resolvers { switch resolver.Platform { @@ -61,10 +61,12 @@ func NewAttributesResolver(resolvers []appsignalsconfig.Resolver, logger *zap.Lo subResolvers = append(subResolvers, getKubernetesResolver(resolver.Platform, resolver.Name, logger), newKubernetesResourceAttributesResolver(resolver.Platform, resolver.Name)) case appsignalsconfig.PlatformEC2: subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformEC2, DefaultInheritedAttributes)) - case appsignalsconfig.PlatformECS: - subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformGeneric, DefaultInheritedAttributes)) default: - subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformGeneric, GenericInheritedAttributes)) + if ecsutil.GetECSUtilSingleton().IsECS() { + subResolvers = append(subResolvers, newResourceAttributesResolver(appsignalsconfig.PlatformECS, AttributePlatformGeneric, DefaultInheritedAttributes)) + } else { + subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformGeneric, GenericInheritedAttributes)) + } } } return &attributesResolver{ @@ -93,16 +95,16 @@ func (r *attributesResolver) Stop(ctx context.Context) error { } type resourceAttributesResolver struct { - platformCode string - platformType string - attributeMap map[string]string + defaultEnvPrefix string + platformType string + attributeMap map[string]string } -func newResourceAttributesResolver(platformCode, platformType string, attributeMap map[string]string) *resourceAttributesResolver { +func newResourceAttributesResolver(defaultEnvPrefix, platformType string, attributeMap map[string]string) *resourceAttributesResolver { return &resourceAttributesResolver{ - platformCode: platformCode, - platformType: platformType, - attributeMap: attributeMap, + defaultEnvPrefix: defaultEnvPrefix, + platformType: platformType, + attributeMap: attributeMap, } } func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcommon.Map) error { @@ -112,19 +114,19 @@ func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcom } } if _, ok := attributes.Get(common.MetricAttributeEnvironment); !ok { - if h.platformCode == appsignalsconfig.PlatformECS { + if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { if clusterName, ok := getECSClusterName(resourceAttributes); ok { - attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.platformCode, clusterName)) + attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, clusterName)) } } - if h.platformCode == appsignalsconfig.PlatformEC2 { + if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.platformCode, asgAttr.Str())) + attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) } } } if _, ok := attributes.Get(common.MetricAttributeEnvironment); !ok { - attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.platformCode, AttributeEnvironmentDefault)) + attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, AttributeEnvironmentDefault)) } attributes.PutStr(common.AttributePlatformType, h.platformType) diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go similarity index 94% rename from plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go rename to plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go index d1eba59b9f..3b6c9994cf 100644 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go @@ -14,9 +14,9 @@ import ( semconv "go.opentelemetry.io/collector/semconv/v1.17.0" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) type MockSubResolver struct { @@ -52,12 +52,6 @@ func TestResourceAttributesResolverWithNoConfiguredName(t *testing.T) { AttributePlatformEC2, config.NewEC2Resolver(""), }, - { - "testOnECS", - config.PlatformECS, - AttributePlatformGeneric, - config.NewECSResolver(""), - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -82,9 +76,11 @@ func TestResourceAttributesResolverWithNoConfiguredName(t *testing.T) { } func TestResourceAttributesResolverWithECSClusterName(t *testing.T) { - logger, _ := zap.NewDevelopment() - attributesResolver := NewAttributesResolver([]config.Resolver{config.NewECSResolver("")}, logger) - resolver := attributesResolver.subResolvers[0] + resolver := resourceAttributesResolver{ + defaultEnvPrefix: "ecs", + platformType: "Generic", + attributeMap: DefaultInheritedAttributes, + } attributes := pcommon.NewMap() resourceAttributes := pcommon.NewMap() @@ -150,11 +146,6 @@ func TestResourceAttributesResolverWithCustomEnvironment(t *testing.T) { config.PlatformEC2, config.NewEC2Resolver(""), }, - { - "testOnECS", - config.PlatformECS, - config.NewEC2Resolver(""), - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/plugins/processors/awsappsignals/internal/resolver/kubernetes.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go similarity index 99% rename from plugins/processors/awsappsignals/internal/resolver/kubernetes.go rename to plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go index b0e41f7ed3..2aa50ecc21 100644 --- a/plugins/processors/awsappsignals/internal/resolver/kubernetes.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go @@ -25,9 +25,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) const ( diff --git a/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go similarity index 99% rename from plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go rename to plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go index c7855b4598..b3a81634c2 100644 --- a/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go @@ -19,9 +19,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" ) diff --git a/plugins/processors/awsappsignals/processor.go b/plugins/processors/awsapplicationsignals/processor.go similarity index 91% rename from plugins/processors/awsappsignals/processor.go rename to plugins/processors/awsapplicationsignals/processor.go index 6d0fcc3108..d83a5783b7 100644 --- a/plugins/processors/awsappsignals/processor.go +++ b/plugins/processors/awsapplicationsignals/processor.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "context" @@ -14,11 +14,11 @@ import ( "golang.org/x/text/cases" "golang.org/x/text/language" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/cardinalitycontrol" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/normalizer" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/resolver" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/normalizer" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/resolver" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) const ( @@ -42,7 +42,7 @@ type stopper interface { Stop(context.Context) error } -type awsappsignalsprocessor struct { +type awsapplicationsignalsprocessor struct { logger *zap.Logger config *appsignalsconfig.Config replaceActions *rules.ReplaceActions @@ -53,7 +53,7 @@ type awsappsignalsprocessor struct { stoppers []stopper } -func (ap *awsappsignalsprocessor) StartMetrics(ctx context.Context, _ component.Host) error { +func (ap *awsapplicationsignalsprocessor) StartMetrics(ctx context.Context, _ component.Host) error { attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) ap.stoppers = []stopper{attributesResolver} attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) @@ -82,7 +82,7 @@ func (ap *awsappsignalsprocessor) StartMetrics(ctx context.Context, _ component. return nil } -func (ap *awsappsignalsprocessor) StartTraces(_ context.Context, _ component.Host) error { +func (ap *awsapplicationsignalsprocessor) StartTraces(_ context.Context, _ component.Host) error { attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) customReplacer := rules.NewReplacer(ap.config.Rules, false) @@ -92,7 +92,7 @@ func (ap *awsappsignalsprocessor) StartTraces(_ context.Context, _ component.Hos return nil } -func (ap *awsappsignalsprocessor) Shutdown(ctx context.Context) error { +func (ap *awsapplicationsignalsprocessor) Shutdown(ctx context.Context) error { for _, stopper := range ap.stoppers { err := stopper.Stop(ctx) if err != nil { @@ -102,7 +102,7 @@ func (ap *awsappsignalsprocessor) Shutdown(ctx context.Context) error { return nil } -func (ap *awsappsignalsprocessor) processTraces(_ context.Context, td ptrace.Traces) (ptrace.Traces, error) { +func (ap *awsapplicationsignalsprocessor) processTraces(_ context.Context, td ptrace.Traces) (ptrace.Traces, error) { rss := td.ResourceSpans() for i := 0; i < rss.Len(); i++ { rs := rss.At(i) @@ -125,7 +125,7 @@ func (ap *awsappsignalsprocessor) processTraces(_ context.Context, td ptrace.Tra return td, nil } -func (ap *awsappsignalsprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { +func (ap *awsapplicationsignalsprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { rms := md.ResourceMetrics() for i := 0; i < rms.Len(); i++ { rs := rms.At(i) @@ -146,7 +146,7 @@ func (ap *awsappsignalsprocessor) processMetrics(ctx context.Context, md pmetric // Attributes are provided for each log and trace, but not at the metric level // Need to process attributes for every data point within a metric. -func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { +func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { // This is a lot of repeated code, but since there is no single parent superclass // between metric data types, we can't use polymorphism. switch m.Type() { diff --git a/plugins/processors/awsappsignals/processor_test.go b/plugins/processors/awsapplicationsignals/processor_test.go similarity index 97% rename from plugins/processors/awsappsignals/processor_test.go rename to plugins/processors/awsapplicationsignals/processor_test.go index 397dae63a2..630272cca4 100644 --- a/plugins/processors/awsappsignals/processor_test.go +++ b/plugins/processors/awsapplicationsignals/processor_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "context" @@ -12,8 +12,8 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) var testRules = []rules.Rule{ @@ -58,7 +58,7 @@ var testRules = []rules.Rule{ func TestProcessMetrics(t *testing.T) { logger, _ := zap.NewDevelopment() - ap := &awsappsignalsprocessor{ + ap := &awsapplicationsignalsprocessor{ logger: logger, config: &config.Config{ Resolvers: []config.Resolver{config.NewGenericResolver("")}, @@ -102,7 +102,7 @@ func TestProcessMetrics(t *testing.T) { func TestProcessMetricsLowercase(t *testing.T) { logger, _ := zap.NewDevelopment() - ap := &awsappsignalsprocessor{ + ap := &awsapplicationsignalsprocessor{ logger: logger, config: &config.Config{ Resolvers: []config.Resolver{config.NewGenericResolver("")}, @@ -129,7 +129,7 @@ func TestProcessMetricsLowercase(t *testing.T) { func TestProcessTraces(t *testing.T) { logger, _ := zap.NewDevelopment() - ap := &awsappsignalsprocessor{ + ap := &awsapplicationsignalsprocessor{ logger: logger, config: &config.Config{ Resolvers: []config.Resolver{config.NewGenericResolver("")}, diff --git a/plugins/processors/awsappsignals/rules/common.go b/plugins/processors/awsapplicationsignals/rules/common.go similarity index 98% rename from plugins/processors/awsappsignals/rules/common.go rename to plugins/processors/awsapplicationsignals/rules/common.go index 4c4c319de8..8e432f20d1 100644 --- a/plugins/processors/awsappsignals/rules/common.go +++ b/plugins/processors/awsapplicationsignals/rules/common.go @@ -9,8 +9,8 @@ import ( "github.com/gobwas/glob" "go.opentelemetry.io/collector/pdata/pcommon" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) type AllowListAction string diff --git a/plugins/processors/awsappsignals/rules/common_test.go b/plugins/processors/awsapplicationsignals/rules/common_test.go similarity index 96% rename from plugins/processors/awsappsignals/rules/common_test.go rename to plugins/processors/awsapplicationsignals/rules/common_test.go index 1433023a1f..40dade4386 100644 --- a/plugins/processors/awsappsignals/rules/common_test.go +++ b/plugins/processors/awsapplicationsignals/rules/common_test.go @@ -6,8 +6,8 @@ package rules import ( "go.opentelemetry.io/collector/pdata/pcommon" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, diff --git a/plugins/processors/awsappsignals/rules/dropper.go b/plugins/processors/awsapplicationsignals/rules/dropper.go similarity index 100% rename from plugins/processors/awsappsignals/rules/dropper.go rename to plugins/processors/awsapplicationsignals/rules/dropper.go diff --git a/plugins/processors/awsappsignals/rules/dropper_test.go b/plugins/processors/awsapplicationsignals/rules/dropper_test.go similarity index 100% rename from plugins/processors/awsappsignals/rules/dropper_test.go rename to plugins/processors/awsapplicationsignals/rules/dropper_test.go diff --git a/plugins/processors/awsappsignals/rules/keeper.go b/plugins/processors/awsapplicationsignals/rules/keeper.go similarity index 97% rename from plugins/processors/awsappsignals/rules/keeper.go rename to plugins/processors/awsapplicationsignals/rules/keeper.go index 50ed315fea..c4b65e999c 100644 --- a/plugins/processors/awsappsignals/rules/keeper.go +++ b/plugins/processors/awsapplicationsignals/rules/keeper.go @@ -6,7 +6,7 @@ package rules import ( "go.opentelemetry.io/collector/pdata/pcommon" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" ) type KeepActions struct { diff --git a/plugins/processors/awsappsignals/rules/keeper_test.go b/plugins/processors/awsapplicationsignals/rules/keeper_test.go similarity index 100% rename from plugins/processors/awsappsignals/rules/keeper_test.go rename to plugins/processors/awsapplicationsignals/rules/keeper_test.go diff --git a/plugins/processors/awsappsignals/rules/replacer.go b/plugins/processors/awsapplicationsignals/rules/replacer.go similarity index 98% rename from plugins/processors/awsappsignals/rules/replacer.go rename to plugins/processors/awsapplicationsignals/rules/replacer.go index fbca29d5d1..fdae4ef3e1 100644 --- a/plugins/processors/awsappsignals/rules/replacer.go +++ b/plugins/processors/awsapplicationsignals/rules/replacer.go @@ -6,7 +6,7 @@ package rules import ( "go.opentelemetry.io/collector/pdata/pcommon" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" ) type ReplaceActions struct { diff --git a/plugins/processors/awsappsignals/rules/replacer_test.go b/plugins/processors/awsapplicationsignals/rules/replacer_test.go similarity index 100% rename from plugins/processors/awsappsignals/rules/replacer_test.go rename to plugins/processors/awsapplicationsignals/rules/replacer_test.go diff --git a/plugins/processors/awsappsignals/testdata/config_eks.yaml b/plugins/processors/awsapplicationsignals/testdata/config_eks.yaml similarity index 100% rename from plugins/processors/awsappsignals/testdata/config_eks.yaml rename to plugins/processors/awsapplicationsignals/testdata/config_eks.yaml diff --git a/plugins/processors/awsappsignals/testdata/config_generic.yaml b/plugins/processors/awsapplicationsignals/testdata/config_generic.yaml similarity index 100% rename from plugins/processors/awsappsignals/testdata/config_generic.yaml rename to plugins/processors/awsapplicationsignals/testdata/config_generic.yaml diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 2d575aed4e..d2c314d5a6 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -27,7 +27,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/plugins/outputs/cloudwatch" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/gpuattributes" ) @@ -47,7 +47,7 @@ func Factories() (otelcol.Factories, error) { } if factories.Processors, err = processor.MakeFactoryMap( - awsappsignals.NewFactory(), + awsapplicationsignals.NewFactory(), batchprocessor.NewFactory(), cumulativetodeltaprocessor.NewFactory(), ec2tagger.NewFactory(), diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index e88c901e39..7d36e78acf 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -104,7 +104,7 @@ exporters: - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index 59470c97af..4f8cdd56e7 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -104,7 +104,7 @@ exporters: - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index 88a432f7bd..7321e14f4c 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -104,7 +104,7 @@ exporters: - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 5ae1913f53..d74924d8f4 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -104,7 +104,7 @@ exporters: - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index 9ef457bb94..ebde6d94d8 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -74,7 +74,7 @@ exporters: - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml index c7a93c9f27..511de4d458 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml @@ -74,7 +74,7 @@ exporters: - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml index 7e5122e6b5..6ece7a381c 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml @@ -1,5 +1,5 @@ log_group_name: "/aws/application-signals/data" -namespace: "AppSignals" +namespace: "ApplicationSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml index ff4af39f81..f48a575ba0 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml @@ -1,5 +1,5 @@ log_group_name: "/aws/application-signals/data" -namespace: "AppSignals" +namespace: "ApplicationSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml index 7e5122e6b5..6ece7a381c 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml @@ -1,5 +1,5 @@ log_group_name: "/aws/application-signals/data" -namespace: "AppSignals" +namespace: "ApplicationSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: diff --git a/translator/translate/otel/pipeline/appsignals/translator.go b/translator/translate/otel/pipeline/applicationsignals/translator.go similarity index 93% rename from translator/translate/otel/pipeline/appsignals/translator.go rename to translator/translate/otel/pipeline/applicationsignals/translator.go index db2b2ac25d..803afbd002 100644 --- a/translator/translate/otel/pipeline/appsignals/translator.go +++ b/translator/translate/otel/pipeline/applicationsignals/translator.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package appsignals +package applicationsignals import ( "fmt" @@ -14,7 +14,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsxray" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/awsproxy" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsappsignals" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsapplicationsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" ) @@ -52,7 +52,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators.Processors.Set(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType))) - translators.Processors.Set(awsappsignals.NewTranslator(awsappsignals.WithDataType(t.dataType))) + translators.Processors.Set(awsapplicationsignals.NewTranslator(awsapplicationsignals.WithDataType(t.dataType))) if t.dataType == component.DataTypeTraces { translators.Exporters.Set(awsxray.NewTranslatorWithName(common.AppSignals)) diff --git a/translator/translate/otel/pipeline/appsignals/translator_test.go b/translator/translate/otel/pipeline/applicationsignals/translator_test.go similarity index 99% rename from translator/translate/otel/pipeline/appsignals/translator_test.go rename to translator/translate/otel/pipeline/applicationsignals/translator_test.go index dc215e406e..575f726c95 100644 --- a/translator/translate/otel/pipeline/appsignals/translator_test.go +++ b/translator/translate/otel/pipeline/applicationsignals/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package appsignals +package applicationsignals import ( "fmt" diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_ec2.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_ec2.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_ec2.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_ec2.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_eks.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_eks.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_eks.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_eks.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_generic.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_generic.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_generic.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_generic.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_k8s.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_k8s.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_k8s.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_k8s.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json b/translator/translate/otel/processor/awsapplicationsignals/testdata/invalidRulesConfig.json similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json rename to translator/translate/otel/processor/awsapplicationsignals/testdata/invalidRulesConfig.json diff --git a/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json b/translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfig.json similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json rename to translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfig.json diff --git a/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigEKS.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigEKS.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigEKS.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigEKS.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigGeneric.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigGeneric.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigGeneric.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigGeneric.yaml diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsapplicationsignals/translator.go similarity index 97% rename from translator/translate/otel/processor/awsappsignals/translator.go rename to translator/translate/otel/processor/awsapplicationsignals/translator.go index bb1721060b..49c669efc9 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsapplicationsignals/translator.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( _ "embed" @@ -12,9 +12,9 @@ import ( "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/processor" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" @@ -49,7 +49,7 @@ func WithDataType(dataType component.DataType) Option { var _ common.Translator[component.Config] = (*translator)(nil) func NewTranslator(opts ...Option) common.Translator[component.Config] { - t := &translator{factory: awsappsignals.NewFactory()} + t := &translator{factory: awsapplicationsignals.NewFactory()} for _, opt := range opts { opt.apply(t) } @@ -100,7 +100,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } case config.ModeECS: cfg.Resolvers = []appsignalsconfig.Resolver{ - appsignalsconfig.NewECSResolver(hostedIn), + appsignalsconfig.NewGenericResolver(hostedIn), } default: cfg.Resolvers = []appsignalsconfig.Resolver{ diff --git a/translator/translate/otel/processor/awsappsignals/translator_test.go b/translator/translate/otel/processor/awsapplicationsignals/translator_test.go similarity index 93% rename from translator/translate/otel/processor/awsappsignals/translator_test.go rename to translator/translate/otel/processor/awsapplicationsignals/translator_test.go index a44fef61df..9f222d0116 100644 --- a/translator/translate/otel/processor/awsappsignals/translator_test.go +++ b/translator/translate/otel/processor/awsapplicationsignals/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( _ "embed" @@ -14,8 +14,8 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" translatorConfig "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" @@ -54,8 +54,8 @@ func TestTranslate(t *testing.T) { kubernetesMode string mode string }{ - //The config for the awsappsignals processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/appsignals/auto-instrumentation-new.yaml#L20 - //The awsappsignals processor config does not have a platform field, instead it gets added to resolvers when marshalled + //The config for the awsapplicationsignals processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/appsignals/auto-instrumentation-new.yaml#L20 + //The awsapplicationsignals processor config does not have a platform field, instead it gets added to resolvers when marshalled "WithAppSignalsEnabledEKS": { input: map[string]interface{}{ "logs": map[string]interface{}{ @@ -177,7 +177,7 @@ func TestTranslate(t *testing.T) { mode: translatorConfig.ModeEC2, }, } - factory := awsappsignals.NewFactory() + factory := awsapplicationsignals.NewFactory() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { if testCase.isKubernetes { diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 4cdc8540c1..79649dfa45 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -22,7 +22,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/appsignals" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/applicationsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsights" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/emf_logs" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/host" @@ -67,8 +67,8 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { } }) translators := common.NewTranslatorMap( - appsignals.NewTranslator(component.DataTypeTraces), - appsignals.NewTranslator(component.DataTypeMetrics), + applicationsignals.NewTranslator(component.DataTypeTraces), + applicationsignals.NewTranslator(component.DataTypeMetrics), host.NewTranslator(common.PipelineNameHost, hostReceivers), host.NewTranslator(common.PipelineNameHostDeltaMetrics, deltaMetricsReceivers), containerinsights.NewTranslator(), From 60b8a4fcb020ba623e93ec9f67a8bfe18ced5841 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Fri, 12 Apr 2024 20:31:02 -0700 Subject: [PATCH 04/11] Enforcing limits on Application Signals metric dimension value --- .../normalizer/attributesnormalizer.go | 43 ++++++++++++++++++- .../normalizer/attributesnormalizer_test.go | 28 ++++++++++++ .../internal/resolver/attributesresolver.go | 18 ++++---- .../resolver/attributesresolver_test.go | 24 +++++------ 4 files changed, 90 insertions(+), 23 deletions(-) diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go index 9ba7a22e0a..8955bdccd2 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -16,6 +16,15 @@ import ( attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) +const ( + // Length limits from Application Signals SLOs + MaxEnvironmentLength = 259 + MaxServiceNameLength = 255 + + // Length limits from CloudWatch Metrics + DefaultMetricAttributeLength = 1024 +) + type attributesNormalizer struct { logger *zap.Logger } @@ -23,6 +32,7 @@ type attributesNormalizer struct { var attributesRenamingForMetric = map[string]string{ attr.AWSLocalService: common.MetricAttributeLocalService, attr.AWSLocalOperation: common.MetricAttributeLocalOperation, + attr.AWSLocalEnvironment: common.MetricAttributeEnvironment, attr.AWSRemoteService: common.MetricAttributeRemoteService, attr.AWSRemoteOperation: common.MetricAttributeRemoteOperation, attr.AWSRemoteEnvironment: common.MetricAttributeRemoteEnvironment, @@ -45,8 +55,7 @@ var resourceAttributesRenamingForTrace = map[string]string{ } var attributesRenamingForTrace = map[string]string{ - common.MetricAttributeEnvironment: attr.AWSLocalEnvironment, - attr.AWSRemoteTarget: attr.AWSRemoteResourceIdentifier, + attr.AWSRemoteTarget: attr.AWSRemoteResourceIdentifier, } var copyMapForMetric = map[string]string{ @@ -75,6 +84,10 @@ func NewAttributesNormalizer(logger *zap.Logger) *attributesNormalizer { func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error { n.copyResourceAttributesToAttributes(attributes, resourceAttributes, isTrace) + // It's assumed that all attributes are initially inserted as trace attribute, and attributesRenamingForMetric + // contains all attributes that will be used for CloudWatch metric dimension. Therefore, we iterate the keys + // for enforcing the limits on length. + truncateAttributesByLength(attributes) n.renameAttributes(attributes, resourceAttributes, isTrace) n.appendNewAttributes(attributes, resourceAttributes, isTrace) return nil @@ -160,3 +173,29 @@ func rename(attrs pcommon.Map, renameMap map[string]string) { } } } + +func truncateAttributesByLength(attributes pcommon.Map) { + for attrKey, _ := range attributesRenamingForMetric { + switch attrKey { + case attr.AWSLocalEnvironment, attr.AWSRemoteEnvironment: + if val, ok := attributes.Get(attrKey); ok { + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), MaxEnvironmentLength)) + } + case attr.AWSLocalService, attr.AWSRemoteService: + if val, ok := attributes.Get(attrKey); ok { + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), MaxServiceNameLength)) + } + default: + if val, ok := attributes.Get(attrKey); ok { + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), DefaultMetricAttributeLength)) + } + } + } +} + +func truncateStringByLength(val string, length int) string { + if len(val) > length { + return val[:length] + } + return val +} diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go index a9e1eff137..c96fce4558 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go @@ -10,6 +10,8 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" conventions "go.opentelemetry.io/collector/semconv/v1.6.1" "go.uber.org/zap" + + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) func TestRenameAttributes_for_metric(t *testing.T) { @@ -106,6 +108,32 @@ func TestCopyResourceAttributesToAttributes(t *testing.T) { } } +func TestTruncateAttributes(t *testing.T) { + attributes := pcommon.NewMap() + + longValue := make([]byte, 300) + for i := 0; i < 300; i++ { + longValue[i] = 'a' + } + longStringValue := string(longValue) + for key, _ := range attributesRenamingForMetric { + attributes.PutStr(key, longStringValue) + } + + truncateAttributesByLength(attributes) + + val, _ := attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, len(val.Str()) == MaxEnvironmentLength) + val, _ = attributes.Get(attr.AWSRemoteEnvironment) + assert.True(t, len(val.Str()) == MaxEnvironmentLength) + val, _ = attributes.Get(attr.AWSLocalService) + assert.True(t, len(val.Str()) == MaxServiceNameLength) + val, _ = attributes.Get(attr.AWSRemoteService) + assert.True(t, len(val.Str()) == MaxServiceNameLength) + val, _ = attributes.Get(attr.AWSRemoteResourceIdentifier) + assert.True(t, len(val.Str()) == 300) +} + func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { logger, _ := zap.NewDevelopment() diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go index 3a2941621e..bbe94ea921 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -29,15 +29,15 @@ const ( ) var GenericInheritedAttributes = map[string]string{ - attr.AWSHostedInEnvironment: common.MetricAttributeEnvironment, - semconv1.AttributeDeploymentEnvironment: common.MetricAttributeEnvironment, + attr.AWSHostedInEnvironment: attr.AWSLocalEnvironment, + semconv1.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, } // DefaultInheritedAttributes is an allow-list that also renames attributes from the resource detection processor var DefaultInheritedAttributes = map[string]string{ - attr.AWSHostedInEnvironment: common.MetricAttributeEnvironment, - semconv1.AttributeDeploymentEnvironment: common.MetricAttributeEnvironment, + attr.AWSHostedInEnvironment: attr.AWSLocalEnvironment, + semconv1.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, attr.ResourceDetectionASG: common.AttributeEC2AutoScalingGroupName, attr.ResourceDetectionHostId: common.AttributeEC2InstanceId, attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, @@ -113,20 +113,20 @@ func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcom attributes.PutStr(mappingKey, val.AsString()) } } - if _, ok := attributes.Get(common.MetricAttributeEnvironment); !ok { + if _, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { if clusterName, ok := getECSClusterName(resourceAttributes); ok { - attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, clusterName)) + attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, clusterName)) } } if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) + attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) } } } - if _, ok := attributes.Get(common.MetricAttributeEnvironment); !ok { - attributes.PutStr(common.MetricAttributeEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, AttributeEnvironmentDefault)) + if _, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { + attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, AttributeEnvironmentDefault)) } attributes.PutStr(common.AttributePlatformType, h.platformType) diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go index 3b6c9994cf..e30bb1ec64 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go @@ -64,13 +64,13 @@ func TestResourceAttributesResolverWithNoConfiguredName(t *testing.T) { resolver.Process(attributes, resourceAttributes) - attr, ok := attributes.Get(common.AttributePlatformType) + attribute, ok := attributes.Get(common.AttributePlatformType) assert.True(t, ok) - assert.Equal(t, tt.platformType, attr.Str()) + assert.Equal(t, tt.platformType, attribute.Str()) - attr, ok = attributes.Get(common.MetricAttributeEnvironment) + attribute, ok = attributes.Get(attr.AWSLocalEnvironment) assert.True(t, ok) - assert.Equal(t, tt.platformCode+":default", attr.Str()) + assert.Equal(t, tt.platformCode+":default", attribute.Str()) }) } } @@ -88,13 +88,13 @@ func TestResourceAttributesResolverWithECSClusterName(t *testing.T) { resolver.Process(attributes, resourceAttributes) - attr, ok := attributes.Get(common.AttributePlatformType) + attribute, ok := attributes.Get(common.AttributePlatformType) assert.True(t, ok) - assert.Equal(t, "Generic", attr.Str()) + assert.Equal(t, "Generic", attribute.Str()) - attr, ok = attributes.Get(common.MetricAttributeEnvironment) + attribute, ok = attributes.Get(attr.AWSLocalEnvironment) assert.True(t, ok) - assert.Equal(t, "ecs:my-cluster", attr.Str()) + assert.Equal(t, "ecs:my-cluster", attribute.Str()) } func TestResourceAttributesResolverWithOnEC2WithASG(t *testing.T) { @@ -110,7 +110,7 @@ func TestResourceAttributesResolverWithOnEC2WithASG(t *testing.T) { platformAttr, ok := attributes.Get(common.AttributePlatformType) assert.True(t, ok) assert.Equal(t, "AWS::EC2", platformAttr.Str()) - envAttr, ok := attributes.Get(common.MetricAttributeEnvironment) + envAttr, ok := attributes.Get(attr.AWSLocalEnvironment) assert.True(t, ok) assert.Equal(t, "ec2:my-asg", envAttr.Str()) } @@ -162,7 +162,7 @@ func TestResourceAttributesResolverWithCustomEnvironment(t *testing.T) { // insert custom env resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "env1") resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(common.MetricAttributeEnvironment) + envAttr, ok := attributes.Get(attr.AWSLocalEnvironment) assert.True(t, ok) assert.Equal(t, "env1", envAttr.Str()) @@ -172,7 +172,7 @@ func TestResourceAttributesResolverWithCustomEnvironment(t *testing.T) { resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "error") resourceAttributes.PutStr(semconv.AttributeDeploymentEnvironment, "env2") resolver.Process(attributes, resourceAttributes) - envAttr, ok = attributes.Get(common.MetricAttributeEnvironment) + envAttr, ok = attributes.Get(attr.AWSLocalEnvironment) assert.True(t, ok) assert.Equal(t, "env2", envAttr.Str()) @@ -181,7 +181,7 @@ func TestResourceAttributesResolverWithCustomEnvironment(t *testing.T) { resourceAttributes.PutStr(semconv.AttributeDeploymentEnvironment, "env3") resolver.Process(attributes, resourceAttributes) - envAttr, ok = attributes.Get(common.MetricAttributeEnvironment) + envAttr, ok = attributes.Get(attr.AWSLocalEnvironment) assert.True(t, ok) assert.Equal(t, "env3", envAttr.Str()) }) From 39fce0bc39d6d650b8f5d105c5847504409fc5d2 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Tue, 16 Apr 2024 14:10:33 -0700 Subject: [PATCH 05/11] 1. bug fixes 2. rename SDK field in EMF log. --- .../awsapplicationsignals/common/types.go | 2 +- .../normalizer/attributesnormalizer_test.go | 2 +- .../internal/resolver/attributesresolver.go | 22 +++++++++-------- .../internal/resolver/kubernetes.go | 9 +++---- .../internal/resolver/kubernetes_test.go | 24 +++++++++---------- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/plugins/processors/awsapplicationsignals/common/types.go b/plugins/processors/awsapplicationsignals/common/types.go index 86615c1f2d..0abb6b2adf 100644 --- a/plugins/processors/awsapplicationsignals/common/types.go +++ b/plugins/processors/awsapplicationsignals/common/types.go @@ -20,7 +20,7 @@ const ( AttributeEC2AutoScalingGroupName = "EC2.AutoScalingGroupName" AttributeEC2InstanceId = "EC2.InstanceId" AttributePlatformType = "PlatformType" - AttributeSDK = "SDK" + AttributeSDK = "Telemetry.SDK" ) const ( diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go index c96fce4558..734af3e798 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go @@ -190,7 +190,7 @@ func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { } n.appendNewAttributes(tt.attributes, tt.resourceAttributes, tt.isTrace) - if value, ok := tt.attributes.Get("SDK"); !ok { + if value, ok := tt.attributes.Get("Telemetry.SDK"); !ok { if !tt.isTrace { t.Errorf("attribute is not found.") } diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go index bbe94ea921..e93e090cab 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -29,14 +29,12 @@ const ( ) var GenericInheritedAttributes = map[string]string{ - attr.AWSHostedInEnvironment: attr.AWSLocalEnvironment, semconv1.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, } // DefaultInheritedAttributes is an allow-list that also renames attributes from the resource detection processor var DefaultInheritedAttributes = map[string]string{ - attr.AWSHostedInEnvironment: attr.AWSLocalEnvironment, semconv1.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, attr.ResourceDetectionASG: common.AttributeEC2AutoScalingGroupName, attr.ResourceDetectionHostId: common.AttributeEC2InstanceId, @@ -110,18 +108,22 @@ func newResourceAttributesResolver(defaultEnvPrefix, platformType string, attrib func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcommon.Map) error { for attrKey, mappingKey := range h.attributeMap { if val, ok := resourceAttributes.Get(attrKey); ok { - attributes.PutStr(mappingKey, val.AsString()) + attributes.PutStr(mappingKey, val.Str()) } } if _, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { - if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { - if clusterName, ok := getECSClusterName(resourceAttributes); ok { - attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, clusterName)) + if val, found := resourceAttributes.Get(attr.AWSHostedInEnvironment); found { + attributes.PutStr(attr.AWSLocalEnvironment, val.Str()) + } else { + if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { + if clusterName, ok := getECSClusterName(resourceAttributes); ok { + attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, clusterName)) + } } - } - if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { - if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) + if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { + if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { + attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) + } } } } diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go index 2aa50ecc21..8dc5aea9b4 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go @@ -700,14 +700,11 @@ func (h *kubernetesResourceAttributesResolver) Process(attributes, resourceAttri namespace = "UnknownNamespace" } - if val, ok := resourceAttributes.Get(semconv.AttributeDeploymentEnvironment); !ok { + if val, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { env := GetDefaultEnvironment(h.platformCode, h.clusterName+"/"+namespace) - if len(env) > 255 { - env = env[:255] - } - attributes.PutStr(common.MetricAttributeEnvironment, env) + attributes.PutStr(attr.AWSLocalEnvironment, env) } else { - attributes.PutStr(common.MetricAttributeEnvironment, val.Str()) + attributes.PutStr(attr.AWSLocalEnvironment, val.Str()) } attributes.PutStr(common.AttributeK8SNamespace, namespace) diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go index b3a81634c2..42c037c319 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go @@ -855,7 +855,7 @@ func TestK8sResourceAttributesResolverOnEKS(t *testing.T) { map[string]string{}, map[string]string{ - common.MetricAttributeEnvironment: "eks:test-cluster/test-namespace-3", + attr.AWSLocalEnvironment: "eks:test-cluster/test-namespace-3", common.AttributeK8SNamespace: "test-namespace-3", common.AttributeEKSClusterName: "test-cluster", common.AttributeEC2InstanceId: "instance-id", @@ -869,7 +869,7 @@ func TestK8sResourceAttributesResolverOnEKS(t *testing.T) { semconv.AttributeDeploymentEnvironment: "custom-env", }, map[string]string{ - common.MetricAttributeEnvironment: "custom-env", + attr.AWSLocalEnvironment: "custom-env", common.AttributeK8SNamespace: "test-namespace-3", common.AttributeEKSClusterName: "test-cluster", common.AttributeEC2InstanceId: "instance-id", @@ -932,7 +932,7 @@ func TestK8sResourceAttributesResolverOnK8S(t *testing.T) { map[string]string{}, map[string]string{ - common.MetricAttributeEnvironment: "k8s:test-cluster/test-namespace-3", + attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", common.AttributeK8SNamespace: "test-namespace-3", common.AttributeK8SClusterName: "test-cluster", common.AttributeEC2InstanceId: "instance-id", @@ -946,7 +946,7 @@ func TestK8sResourceAttributesResolverOnK8S(t *testing.T) { semconv.AttributeDeploymentEnvironment: "custom-env", }, map[string]string{ - common.MetricAttributeEnvironment: "custom-env", + attr.AWSLocalEnvironment: "custom-env", common.AttributeK8SNamespace: "test-namespace-3", common.AttributeK8SClusterName: "test-cluster", common.AttributeEC2InstanceId: "instance-id", @@ -1006,10 +1006,10 @@ func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { map[string]string{}, map[string]string{ - common.MetricAttributeEnvironment: "k8s:test-cluster/test-namespace-3", - common.AttributeK8SNamespace: "test-namespace-3", - common.AttributeK8SClusterName: "test-cluster", - attr.ResourceDetectionHostName: "hostname", + attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + attr.ResourceDetectionHostName: "hostname", }, }, { @@ -1018,10 +1018,10 @@ func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { semconv.AttributeDeploymentEnvironment: "custom-env", }, map[string]string{ - common.MetricAttributeEnvironment: "custom-env", - common.AttributeK8SNamespace: "test-namespace-3", - common.AttributeK8SClusterName: "test-cluster", - attr.ResourceDetectionHostName: "hostname", + attr.AWSLocalEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + attr.ResourceDetectionHostName: "hostname", }, }, } From 200a5f8bb8a6c405a504fbbdd7e55d8562702fc0 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Fri, 19 Apr 2024 15:40:29 -0700 Subject: [PATCH 06/11] Rename Application Signals attributes in EMF logs --- .../awsapplicationsignals/common/types.go | 17 +++--- .../internal/attributes/attributes.go | 1 + .../metrics_limiter_test.go | 4 +- .../normalizer/attributesnormalizer.go | 49 +++++++++++------ .../normalizer/attributesnormalizer_test.go | 10 ++-- .../internal/resolver/attributesresolver.go | 33 ++++++------ .../resolver/attributesresolver_test.go | 4 +- .../internal/resolver/kubernetes.go | 2 +- .../internal/resolver/kubernetes_test.go | 54 +++++++++---------- .../appsignals_and_eks_config.yaml | 8 +-- .../appsignals_and_k8s_config.yaml | 8 +-- .../appsignals_fallback_and_eks_config.yaml | 8 +-- .../appsignals_over_fallback_config.yaml | 8 +-- .../sampleConfig/base_appsignals_config.yaml | 8 +-- .../base_appsignals_fallback_config.yaml | 8 +-- .../awsemf/appsignals_config_eks.yaml | 8 +-- .../awsemf/appsignals_config_generic.yaml | 8 +-- .../awsemf/appsignals_config_k8s.yaml | 8 +-- 18 files changed, 134 insertions(+), 112 deletions(-) diff --git a/plugins/processors/awsapplicationsignals/common/types.go b/plugins/processors/awsapplicationsignals/common/types.go index 0abb6b2adf..922570519d 100644 --- a/plugins/processors/awsapplicationsignals/common/types.go +++ b/plugins/processors/awsapplicationsignals/common/types.go @@ -14,13 +14,16 @@ const ( MetricAttributeRemoteResourceType = "RemoteResourceType" ) const ( - AttributeEKSClusterName = "EKS.Cluster" - AttributeK8SClusterName = "K8s.Cluster" - AttributeK8SNamespace = "K8s.Namespace" - AttributeEC2AutoScalingGroupName = "EC2.AutoScalingGroupName" - AttributeEC2InstanceId = "EC2.InstanceId" - AttributePlatformType = "PlatformType" - AttributeSDK = "Telemetry.SDK" + AttributeEKSClusterName = "EKS.Cluster" + AttributeK8SClusterName = "K8s.Cluster" + AttributeK8SNamespace = "K8s.Namespace" + AttributeEC2AutoScalingGroup = "EC2.AutoScalingGroup" + AttributeEC2InstanceId = "EC2.InstanceId" + AttributeHost = "Host" + AttributePlatformType = "PlatformType" + AttributeTelemetrySDK = "Telemetry.SDK" + AttributeTelemetryAgent = "Telemetry.Agent" + AttributeTelemetrySource = "Telemetry.Source" ) const ( diff --git a/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go b/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go index 0740511ed6..75e9a2fca1 100644 --- a/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go +++ b/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go @@ -5,6 +5,7 @@ package attributes const ( // aws attributes + AWSSpanKind = "aws.span.kind" AWSLocalService = "aws.local.service" AWSLocalEnvironment = "aws.local.environment" AWSLocalOperation = "aws.local.operation" diff --git a/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go index aa9cb426cb..e7b14e4978 100644 --- a/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go +++ b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go @@ -41,7 +41,9 @@ func TestAdmitAndRollup(t *testing.T) { admittedAttributes[uniqKey.AsString()] = attr } else { for _, indexedAttrKey := range awsDeclaredMetricAttributes { - if indexedAttrKey == "Environment" || indexedAttrKey == "Service" || indexedAttrKey == "RemoteService" { + if indexedAttrKey == common.MetricAttributeEnvironment || + indexedAttrKey == common.MetricAttributeLocalService || + indexedAttrKey == common.MetricAttributeRemoteService { continue } attrValue, _ := attr.Get(indexedAttrKey) diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go index 8955bdccd2..c083449651 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -8,21 +8,21 @@ import ( "strings" "go.opentelemetry.io/collector/pdata/pcommon" - conventions "go.opentelemetry.io/collector/semconv/v1.18.0" semconv "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" + "github.com/aws/amazon-cloudwatch-agent/internal/version" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) const ( // Length limits from Application Signals SLOs - MaxEnvironmentLength = 259 - MaxServiceNameLength = 255 + maxEnvironmentLength = 259 + maxServiceNameLength = 255 // Length limits from CloudWatch Metrics - DefaultMetricAttributeLength = 1024 + defaultMetricAttributeLength = 1024 ) type attributesNormalizer struct { @@ -89,7 +89,7 @@ func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Ma // for enforcing the limits on length. truncateAttributesByLength(attributes) n.renameAttributes(attributes, resourceAttributes, isTrace) - n.appendNewAttributes(attributes, resourceAttributes, isTrace) + n.normalizeTelemetryAttributes(attributes, resourceAttributes, isTrace) return nil } @@ -123,7 +123,7 @@ func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, re } } -func (n *attributesNormalizer) appendNewAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { +func (n *attributesNormalizer) normalizeTelemetryAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { if isTrace { return } @@ -140,13 +140,13 @@ func (n *attributesNormalizer) appendNewAttributes(attributes, resourceAttribute // TODO read telemetry.auto.version from telemetry.distro.* from v1.22 resourceAttributes.Range(func(k string, v pcommon.Value) bool { switch k { - case conventions.AttributeTelemetrySDKName: + case semconv.AttributeTelemetrySDKName: sdkName = strings.ReplaceAll(v.Str(), " ", "") - case conventions.AttributeTelemetrySDKLanguage: + case semconv.AttributeTelemetrySDKLanguage: sdkLang = strings.ReplaceAll(v.Str(), " ", "") - case conventions.AttributeTelemetrySDKVersion: + case semconv.AttributeTelemetrySDKVersion: sdkVersion = strings.ReplaceAll(v.Str(), " ", "") - case conventions.AttributeTelemetryAutoVersion: + case semconv.AttributeTelemetryAutoVersion: sdkAutoVersion = strings.ReplaceAll(v.Str(), " ", "") } return true @@ -155,7 +155,26 @@ func (n *attributesNormalizer) appendNewAttributes(attributes, resourceAttribute sdkVersion = sdkAutoVersion mode = instrumentationModeAuto } - attributes.PutStr(common.AttributeSDK, fmt.Sprintf("%s,%s,%s,%s", sdkName, sdkVersion, sdkLang, mode)) + attributes.PutStr(common.AttributeTelemetrySDK, fmt.Sprintf("%s,%s,%s,%s", sdkName, sdkVersion, sdkLang, mode)) + attributes.PutStr(common.AttributeTelemetryAgent, fmt.Sprintf("CWAgent/%s", version.Number())) + + var telemetrySource string + if val, ok := attributes.Get(attr.AWSSpanKind); ok { + switch val.Str() { + case "CLIENT": + telemetrySource = "ClientSpan" + case "SERVER": + telemetrySource = "ServerSpan" + case "PRODUCER": + telemetrySource = "ProducerSpan" + case "CONSUMER": + telemetrySource = "ConsumerSpan" + case "LOCAL_ROOT": + telemetrySource = "LocalRootSpan" + } + attributes.PutStr(common.AttributeTelemetrySource, telemetrySource) + attributes.Remove(attr.AWSSpanKind) + } } func rename(attrs pcommon.Map, renameMap map[string]string) { @@ -175,19 +194,19 @@ func rename(attrs pcommon.Map, renameMap map[string]string) { } func truncateAttributesByLength(attributes pcommon.Map) { - for attrKey, _ := range attributesRenamingForMetric { + for attrKey := range attributesRenamingForMetric { switch attrKey { case attr.AWSLocalEnvironment, attr.AWSRemoteEnvironment: if val, ok := attributes.Get(attrKey); ok { - attributes.PutStr(attrKey, truncateStringByLength(val.Str(), MaxEnvironmentLength)) + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), maxEnvironmentLength)) } case attr.AWSLocalService, attr.AWSRemoteService: if val, ok := attributes.Get(attrKey); ok { - attributes.PutStr(attrKey, truncateStringByLength(val.Str(), MaxServiceNameLength)) + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), maxServiceNameLength)) } default: if val, ok := attributes.Get(attrKey); ok { - attributes.PutStr(attrKey, truncateStringByLength(val.Str(), DefaultMetricAttributeLength)) + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), defaultMetricAttributeLength)) } } } diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go index 734af3e798..b8dff25844 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go @@ -123,13 +123,13 @@ func TestTruncateAttributes(t *testing.T) { truncateAttributesByLength(attributes) val, _ := attributes.Get(attr.AWSLocalEnvironment) - assert.True(t, len(val.Str()) == MaxEnvironmentLength) + assert.True(t, len(val.Str()) == maxEnvironmentLength) val, _ = attributes.Get(attr.AWSRemoteEnvironment) - assert.True(t, len(val.Str()) == MaxEnvironmentLength) + assert.True(t, len(val.Str()) == maxEnvironmentLength) val, _ = attributes.Get(attr.AWSLocalService) - assert.True(t, len(val.Str()) == MaxServiceNameLength) + assert.True(t, len(val.Str()) == maxServiceNameLength) val, _ = attributes.Get(attr.AWSRemoteService) - assert.True(t, len(val.Str()) == MaxServiceNameLength) + assert.True(t, len(val.Str()) == maxServiceNameLength) val, _ = attributes.Get(attr.AWSRemoteResourceIdentifier) assert.True(t, len(val.Str()) == 300) } @@ -188,7 +188,7 @@ func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { n := &attributesNormalizer{ logger: logger, } - n.appendNewAttributes(tt.attributes, tt.resourceAttributes, tt.isTrace) + n.normalizeTelemetryAttributes(tt.attributes, tt.resourceAttributes, tt.isTrace) if value, ok := tt.attributes.Get("Telemetry.SDK"); !ok { if !tt.isTrace { diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go index e93e090cab..b7da12888b 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -10,7 +10,7 @@ import ( "strings" "go.opentelemetry.io/collector/pdata/pcommon" - semconv1 "go.opentelemetry.io/collector/semconv/v1.17.0" + semconv "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" @@ -29,16 +29,16 @@ const ( ) var GenericInheritedAttributes = map[string]string{ - semconv1.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, + semconv.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, + attr.ResourceDetectionHostName: common.AttributeHost, } // DefaultInheritedAttributes is an allow-list that also renames attributes from the resource detection processor var DefaultInheritedAttributes = map[string]string{ - semconv1.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, - attr.ResourceDetectionASG: common.AttributeEC2AutoScalingGroupName, - attr.ResourceDetectionHostId: common.AttributeEC2InstanceId, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, + semconv.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, + attr.ResourceDetectionASG: common.AttributeEC2AutoScalingGroup, + attr.ResourceDetectionHostId: common.AttributeEC2InstanceId, + attr.ResourceDetectionHostName: common.AttributeHost, } type subResolver interface { @@ -85,9 +85,7 @@ func (r *attributesResolver) Process(attributes, resourceAttributes pcommon.Map, func (r *attributesResolver) Stop(ctx context.Context) error { var errs error for _, subResolver := range r.subResolvers { - if err := subResolver.Stop(ctx); err != nil { - errs = errors.Join(errs, err) - } + errs = errors.Join(errs, subResolver.Stop(ctx)) } return errs } @@ -117,18 +115,17 @@ func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcom } else { if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { if clusterName, ok := getECSClusterName(resourceAttributes); ok { - attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, clusterName)) + attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, clusterName)) } - } - if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { + } else if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) + attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) } } } } if _, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { - attributes.PutStr(attr.AWSLocalEnvironment, GetDefaultEnvironment(h.defaultEnvPrefix, AttributeEnvironmentDefault)) + attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, AttributeEnvironmentDefault)) } attributes.PutStr(common.AttributePlatformType, h.platformType) @@ -136,11 +133,11 @@ func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcom } func getECSClusterName(resourceAttributes pcommon.Map) (string, bool) { - if clusterAttr, ok := resourceAttributes.Get(semconv1.AttributeAWSECSClusterARN); ok { + if clusterAttr, ok := resourceAttributes.Get(semconv.AttributeAWSECSClusterARN); ok { parts := strings.Split(clusterAttr.Str(), "/") clusterName := parts[len(parts)-1] return clusterName, true - } else if taskAttr, ok := resourceAttributes.Get(semconv1.AttributeAWSECSTaskARN); ok { + } else if taskAttr, ok := resourceAttributes.Get(semconv.AttributeAWSECSTaskARN); ok { parts := strings.SplitAfterN(taskAttr.Str(), ":task/", 2) if len(parts) == 2 { taskParts := strings.Split(parts[1], "/") @@ -153,7 +150,7 @@ func getECSClusterName(resourceAttributes pcommon.Map) (string, bool) { return "", false } -func GetDefaultEnvironment(platformCode, val string) string { +func getDefaultEnvironment(platformCode, val string) string { return fmt.Sprintf("%s:%s", platformCode, val) } diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go index e30bb1ec64..ea6c1cd129 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.17.0" + semconv "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" @@ -125,7 +125,7 @@ func TestResourceAttributesResolverWithHostname(t *testing.T) { resourceAttributes.PutStr(attr.ResourceDetectionHostName, "hostname") resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.ResourceDetectionHostName) + envAttr, ok := attributes.Get(common.AttributeHost) assert.True(t, ok) assert.Equal(t, "hostname", envAttr.AsString()) } diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go index 8dc5aea9b4..979fad2900 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go @@ -701,7 +701,7 @@ func (h *kubernetesResourceAttributesResolver) Process(attributes, resourceAttri } if val, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { - env := GetDefaultEnvironment(h.platformCode, h.clusterName+"/"+namespace) + env := getDefaultEnvironment(h.platformCode, h.clusterName+"/"+namespace) attributes.PutStr(attr.AWSLocalEnvironment, env) } else { attributes.PutStr(attr.AWSLocalEnvironment, val.Str()) diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go index 42c037c319..0687b64b93 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go @@ -855,12 +855,12 @@ func TestK8sResourceAttributesResolverOnEKS(t *testing.T) { map[string]string{}, map[string]string{ - attr.AWSLocalEnvironment: "eks:test-cluster/test-namespace-3", - common.AttributeK8SNamespace: "test-namespace-3", - common.AttributeEKSClusterName: "test-cluster", - common.AttributeEC2InstanceId: "instance-id", - attr.ResourceDetectionHostName: "hostname", - common.AttributeEC2AutoScalingGroupName: "asg", + attr.AWSLocalEnvironment: "eks:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeEKSClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", }, }, { @@ -869,12 +869,12 @@ func TestK8sResourceAttributesResolverOnEKS(t *testing.T) { semconv.AttributeDeploymentEnvironment: "custom-env", }, map[string]string{ - attr.AWSLocalEnvironment: "custom-env", - common.AttributeK8SNamespace: "test-namespace-3", - common.AttributeEKSClusterName: "test-cluster", - common.AttributeEC2InstanceId: "instance-id", - attr.ResourceDetectionHostName: "hostname", - common.AttributeEC2AutoScalingGroupName: "asg", + attr.AWSLocalEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeEKSClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", }, }, } @@ -932,12 +932,12 @@ func TestK8sResourceAttributesResolverOnK8S(t *testing.T) { map[string]string{}, map[string]string{ - attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", - common.AttributeK8SNamespace: "test-namespace-3", - common.AttributeK8SClusterName: "test-cluster", - common.AttributeEC2InstanceId: "instance-id", - attr.ResourceDetectionHostName: "hostname", - common.AttributeEC2AutoScalingGroupName: "asg", + attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", }, }, { @@ -946,12 +946,12 @@ func TestK8sResourceAttributesResolverOnK8S(t *testing.T) { semconv.AttributeDeploymentEnvironment: "custom-env", }, map[string]string{ - attr.AWSLocalEnvironment: "custom-env", - common.AttributeK8SNamespace: "test-namespace-3", - common.AttributeK8SClusterName: "test-cluster", - common.AttributeEC2InstanceId: "instance-id", - attr.ResourceDetectionHostName: "hostname", - common.AttributeEC2AutoScalingGroupName: "asg", + attr.AWSLocalEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", }, }, } @@ -1009,7 +1009,7 @@ func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", common.AttributeK8SNamespace: "test-namespace-3", common.AttributeK8SClusterName: "test-cluster", - attr.ResourceDetectionHostName: "hostname", + common.AttributeHost: "hostname", }, }, { @@ -1021,7 +1021,7 @@ func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { attr.AWSLocalEnvironment: "custom-env", common.AttributeK8SNamespace: "test-namespace-3", common.AttributeK8SClusterName: "test-cluster", - attr.ResourceDetectionHostName: "hostname", + common.AttributeHost: "hostname", }, }, } @@ -1044,7 +1044,7 @@ func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) // EC2 related fields that should not exist for on-prem - _, exists := attributes.Get(common.AttributeEC2AutoScalingGroupName) + _, exists := attributes.Get(common.AttributeEC2AutoScalingGroup) assert.False(t, exists) _, exists = attributes.Get(common.AttributeEC2InstanceId) diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 7d36e78acf..fb91c0a6cf 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -22,8 +22,8 @@ exporters: - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency @@ -96,8 +96,8 @@ exporters: - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index 4f8cdd56e7..a49ebeeb37 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -22,8 +22,8 @@ exporters: - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency @@ -96,8 +96,8 @@ exporters: - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index 7321e14f4c..6a4d070a80 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -22,8 +22,8 @@ exporters: - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency @@ -96,8 +96,8 @@ exporters: - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index d74924d8f4..a8c3e656d9 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -22,8 +22,8 @@ exporters: - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency @@ -96,8 +96,8 @@ exporters: - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index ebde6d94d8..dd5c0e8d4d 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -22,8 +22,8 @@ exporters: - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency @@ -66,8 +66,8 @@ exporters: - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml index 511de4d458..11c0eefb2b 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml @@ -22,8 +22,8 @@ exporters: - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency @@ -66,8 +66,8 @@ exporters: - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml index 6ece7a381c..036c519717 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml @@ -8,8 +8,8 @@ metric_declarations: - [Environment, Service] label_matchers: - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' + - Telemetry.Source + regex: '^(ServerSpan|LocalRootSpan)$' metric_name_selectors: - Latency - Fault @@ -30,8 +30,8 @@ metric_declarations: - [RemoteService] label_matchers: - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' + - Telemetry.Source + regex: '^(ClientSpan|ProducerSpan|ConsumerSpan)$' metric_name_selectors: - Latency - Fault diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml index f48a575ba0..735f6df7da 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml @@ -8,8 +8,8 @@ metric_declarations: - [Environment, Service] label_matchers: - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' + - Telemetry.Source + regex: '^(ServerSpan|LocalRootSpan)$' metric_name_selectors: - Latency - Fault @@ -25,8 +25,8 @@ metric_declarations: - [RemoteService] label_matchers: - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' + - Telemetry.Source + regex: '^(ClientSpan|ProducerSpan|ConsumerSpan)$' metric_name_selectors: - Latency - Fault diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml index 6ece7a381c..ce85e50f78 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml @@ -8,8 +8,8 @@ metric_declarations: - [Environment, Service] label_matchers: - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ metric_name_selectors: - Latency - Fault @@ -30,8 +30,8 @@ metric_declarations: - [RemoteService] label_matchers: - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' + - Telemetry.Source + regex: '^(ClientSpan|ProducerSpan|ConsumerSpan)$' metric_name_selectors: - Latency - Fault From 421e99d89621f10baf59e494ea6b945f2104e90a Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Thu, 25 Apr 2024 10:12:45 -0700 Subject: [PATCH 07/11] Add invalid metric pruning --- .../awsapplicationsignals/common/types.go | 12 +++ .../normalizer/attributesnormalizer.go | 6 +- .../internal/prune/metric_pruner.go | 42 +++++++++ .../internal/prune/metric_pruner_test.go | 85 +++++++++++++++++++ .../awsapplicationsignals/processor.go | 19 +++-- 5 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go create mode 100644 plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go diff --git a/plugins/processors/awsapplicationsignals/common/types.go b/plugins/processors/awsapplicationsignals/common/types.go index 922570519d..a9e72a2fd3 100644 --- a/plugins/processors/awsapplicationsignals/common/types.go +++ b/plugins/processors/awsapplicationsignals/common/types.go @@ -13,6 +13,7 @@ const ( MetricAttributeRemoteResourceIdentifier = "RemoteResourceIdentifier" MetricAttributeRemoteResourceType = "RemoteResourceType" ) + const ( AttributeEKSClusterName = "EKS.Cluster" AttributeK8SClusterName = "K8s.Cluster" @@ -29,3 +30,14 @@ const ( const ( AttributeTmpReserved = "aws.tmp.reserved" ) + +var IndexableMetricAttributes = []string{ + MetricAttributeLocalService, + MetricAttributeLocalOperation, + MetricAttributeEnvironment, + MetricAttributeRemoteService, + MetricAttributeRemoteEnvironment, + MetricAttributeRemoteOperation, + MetricAttributeRemoteResourceIdentifier, + MetricAttributeRemoteResourceType, +} diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go index c083449651..fa19c75b29 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -84,9 +84,6 @@ func NewAttributesNormalizer(logger *zap.Logger) *attributesNormalizer { func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error { n.copyResourceAttributesToAttributes(attributes, resourceAttributes, isTrace) - // It's assumed that all attributes are initially inserted as trace attribute, and attributesRenamingForMetric - // contains all attributes that will be used for CloudWatch metric dimension. Therefore, we iterate the keys - // for enforcing the limits on length. truncateAttributesByLength(attributes) n.renameAttributes(attributes, resourceAttributes, isTrace) n.normalizeTelemetryAttributes(attributes, resourceAttributes, isTrace) @@ -194,6 +191,9 @@ func rename(attrs pcommon.Map, renameMap map[string]string) { } func truncateAttributesByLength(attributes pcommon.Map) { + // It's assumed that all attributes are initially inserted as trace attribute, and attributesRenamingForMetric + // contains all attributes that will be used for CloudWatch metric dimension. Therefore, we iterate the keys + // for enforcing the limits on length. for attrKey := range attributesRenamingForMetric { switch attrKey { case attr.AWSLocalEnvironment, attr.AWSRemoteEnvironment: diff --git a/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go new file mode 100644 index 0000000000..5fcca636d4 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go @@ -0,0 +1,42 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package prune + +import ( + "errors" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" +) + +type MetricPruner struct { +} + +func (p *MetricPruner) ShouldBeDropped(attributes pcommon.Map) (bool, error) { + for _, attributeKey := range common.IndexableMetricAttributes { + if val, ok := attributes.Get(attributeKey); ok { + if !isAsciiPrintable(val.Str()) { + return true, errors.New("Metric attribute " + attributeKey + " must contain only ASCII characters.") + } + } + } + return false, nil +} + +func NewPruner() *MetricPruner { + return &MetricPruner{} +} + +func isAsciiPrintable(val string) bool { + nonWhitespaceFound := false + for _, c := range val { + if c < 32 || c > 126 { + return false + } else if !nonWhitespaceFound && c != 32 { + nonWhitespaceFound = true + } + } + return nonWhitespaceFound +} diff --git a/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go new file mode 100644 index 0000000000..4a14c5e9a7 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package prune + +import ( + "testing" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" +) + +func TestMetricPrunerWithIndexableAttribute(t *testing.T) { + tests := []struct { + name string + val string + want bool + }{ + { + "testShouldDropChineseChar", + "漢", + true, + }, { + "testShouldDropSymbolChar", + "€, £, µ", + true, + }, { + "testShouldDropAllBlackSpace", + " ", + true, + }, + { + "testShouldDropAllTab", + " ", + true, + }, { + "testShouldKeepEnglishWord", + "abcdefg-", + false, + }, + } + + p := &MetricPruner{} + for _, tt := range tests { + attributes := pcommon.NewMap() + attributes.PutStr(common.MetricAttributeLocalService, tt.val) + t.Run(tt.name, func(t *testing.T) { + got, _ := p.ShouldBeDropped(attributes) + if got != tt.want { + t.Errorf("ShouldBeDropped() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMetricPrunerWithNonIndexableAttribute(t *testing.T) { + tests := []struct { + name string + val string + want bool + }{ + { + "testShouldKeepChineseChar", + "漢", + false, + }, { + "testShouldKeepEnglishWord", + "abcdefg-", + false, + }, + } + + p := &MetricPruner{} + for _, tt := range tests { + attributes := pcommon.NewMap() + attributes.PutStr(common.AttributeEC2InstanceId, tt.val) + t.Run(tt.name, func(t *testing.T) { + got, _ := p.ShouldBeDropped(attributes) + if got != tt.want { + t.Errorf("ShouldBeDropped() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/processors/awsapplicationsignals/processor.go b/plugins/processors/awsapplicationsignals/processor.go index d83a5783b7..8fc3e25bb7 100644 --- a/plugins/processors/awsapplicationsignals/processor.go +++ b/plugins/processors/awsapplicationsignals/processor.go @@ -17,14 +17,14 @@ import ( appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/normalizer" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/prune" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/resolver" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) const ( - failedToProcessAttribute = "failed to process attributes" - failedToProcessAttributeWithCustomRule = "failed to process attributes with custom rule, will drop the metric" - failedToProcessAttributeWithLimiter = "failed to process attributes with limiter, keep the data" + failedToProcessAttribute = "failed to process attributes" + failedToProcessAttributeWithLimiter = "failed to process attributes with limiter, keep the data" ) var metricCaser = cases.Title(language.English) @@ -75,9 +75,10 @@ func (ap *awsapplicationsignalsprocessor) StartMetrics(ctx context.Context, _ co ap.replaceActions = rules.NewReplacer(ap.config.Rules, !limiterConfig.Disabled) + pruner := prune.NewPruner() keeper := rules.NewKeeper(ap.config.Rules, !limiterConfig.Disabled) dropper := rules.NewDropper(ap.config.Rules) - ap.allowlistMutators = []allowListMutator{keeper, dropper} + ap.allowlistMutators = []allowListMutator{pruner, keeper, dropper} return nil } @@ -164,7 +165,7 @@ func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Cont for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -199,7 +200,7 @@ func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Cont for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -234,7 +235,7 @@ func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Cont for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -269,7 +270,7 @@ func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Cont for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -304,7 +305,7 @@ func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Cont for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true From 6b6559316315680e113eb47bbc3e88e6625038e4 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Fri, 26 Apr 2024 15:04:43 -0700 Subject: [PATCH 08/11] fix null instance id in trace --- .../internal/normalizer/attributesnormalizer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go index fa19c75b29..dd08d63f44 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -183,7 +183,6 @@ func rename(attrs pcommon.Map, renameMap map[string]string) { // only rename host.id if the pod name is set if host, ok := attrs.Get("host.id"); ok { attrs.PutStr("K8s.Node", host.AsString()) - attrs.Remove("host.id") } } } From 3ddb83b640d98d4b6f1fc960949f42adf76c88eb Mon Sep 17 00:00:00 2001 From: "Mengyi Zhou (bjrara)" Date: Thu, 16 May 2024 12:58:51 -0700 Subject: [PATCH 09/11] [GA] Autofill ECS cluster info if it's missing (#1180) Merge as the changes have been reviewed in the previous PR. --- .../normalizer/attributesnormalizer.go | 35 +++++++++++++------ .../normalizer/attributesnormalizer_test.go | 5 +-- .../internal/resolver/attributesresolver.go | 6 ++-- .../resolver/attributesresolver_test.go | 2 +- .../internal/resolver/kubernetes.go | 2 +- .../internal/resolver/kubernetes_test.go | 2 +- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go index dd08d63f44..b0966bca2c 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -8,7 +8,8 @@ import ( "strings" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.18.0" + deprecatedsemconv "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" "github.com/aws/amazon-cloudwatch-agent/internal/version" @@ -126,28 +127,38 @@ func (n *attributesNormalizer) normalizeTelemetryAttributes(attributes, resource } var ( - sdkName string - sdkVersion string + sdkName string + sdkVersion string + sdkLang string + ) + var ( + sdkAutoName string sdkAutoVersion string - sdkLang string ) sdkName, sdkVersion, sdkLang = "-", "-", "-" mode := instrumentationModeManual - // TODO read telemetry.auto.version from telemetry.distro.* from v1.22 resourceAttributes.Range(func(k string, v pcommon.Value) bool { switch k { case semconv.AttributeTelemetrySDKName: - sdkName = strings.ReplaceAll(v.Str(), " ", "") + sdkName = removeWhitespaces(v.Str()) case semconv.AttributeTelemetrySDKLanguage: - sdkLang = strings.ReplaceAll(v.Str(), " ", "") + sdkLang = removeWhitespaces(v.Str()) case semconv.AttributeTelemetrySDKVersion: - sdkVersion = strings.ReplaceAll(v.Str(), " ", "") - case semconv.AttributeTelemetryAutoVersion: - sdkAutoVersion = strings.ReplaceAll(v.Str(), " ", "") + sdkVersion = removeWhitespaces(v.Str()) + } + switch k { + case semconv.AttributeTelemetryDistroName: + sdkAutoName = removeWhitespaces(v.Str()) + case deprecatedsemconv.AttributeTelemetryAutoVersion, semconv.AttributeTelemetryDistroVersion: + sdkAutoVersion = removeWhitespaces(v.Str()) } return true }) + if sdkAutoName != "" { + sdkName = sdkAutoName + mode = instrumentationModeAuto + } if sdkAutoVersion != "" { sdkVersion = sdkAutoVersion mode = instrumentationModeAuto @@ -217,3 +228,7 @@ func truncateStringByLength(val string, length int) string { } return val } + +func removeWhitespaces(val string) string { + return strings.ReplaceAll(val, " ", "") +} diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go index b8dff25844..18a45fcfe2 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go @@ -8,7 +8,8 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" - conventions "go.opentelemetry.io/collector/semconv/v1.6.1" + deprecatedsemconv "go.opentelemetry.io/collector/semconv/v1.18.0" + conventions "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" @@ -139,7 +140,7 @@ func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { completeResourceAttributes := pcommon.NewMap() completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKName, "opentelemetry") - completeResourceAttributes.PutStr(conventions.AttributeTelemetryAutoVersion, "0.0.1 auto") + completeResourceAttributes.PutStr(deprecatedsemconv.AttributeTelemetryAutoVersion, "0.0.1 auto") completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKVersion, "0.0.1 test") completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKLanguage, "go") diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go index b7da12888b..94d7487606 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -10,7 +10,7 @@ import ( "strings" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" @@ -114,7 +114,9 @@ func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcom attributes.PutStr(attr.AWSLocalEnvironment, val.Str()) } else { if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { - if clusterName, ok := getECSClusterName(resourceAttributes); ok { + if clusterName, _ := getECSClusterName(resourceAttributes); clusterName != "" { + attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, clusterName)) + } else if clusterName = ecsutil.GetECSUtilSingleton().Cluster; clusterName != "" { attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, clusterName)) } } else if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go index ea6c1cd129..cf8fd05f57 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go index 979fad2900..fe5712976b 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go @@ -17,7 +17,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.17.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go index 0687b64b93..20cb17a5cb 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go @@ -14,7 +14,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.17.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" From b7eaad47d022bd9265ea9b0ece67e790e75c8a74 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Fri, 17 May 2024 11:16:34 -0700 Subject: [PATCH 10/11] extract getLocalEnvironment for resolver in ApplicationSignals --- .../normalizer/attributesnormalizer_test.go | 12 ++--- .../internal/resolver/attributesresolver.go | 44 ++++++++++--------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go index 18a45fcfe2..812b5a4142 100644 --- a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" deprecatedsemconv "go.opentelemetry.io/collector/semconv/v1.18.0" - conventions "go.opentelemetry.io/collector/semconv/v1.22.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" @@ -139,14 +139,14 @@ func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { logger, _ := zap.NewDevelopment() completeResourceAttributes := pcommon.NewMap() - completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKName, "opentelemetry") + completeResourceAttributes.PutStr(semconv.AttributeTelemetrySDKName, "opentelemetry") completeResourceAttributes.PutStr(deprecatedsemconv.AttributeTelemetryAutoVersion, "0.0.1 auto") - completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKVersion, "0.0.1 test") - completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKLanguage, "go") + completeResourceAttributes.PutStr(semconv.AttributeTelemetrySDKVersion, "0.0.1 test") + completeResourceAttributes.PutStr(semconv.AttributeTelemetrySDKLanguage, "go") incompleteResourceAttributes := pcommon.NewMap() - incompleteResourceAttributes.PutStr(conventions.AttributeTelemetrySDKName, "opentelemetry") - incompleteResourceAttributes.PutStr(conventions.AttributeTelemetrySDKVersion, "0.0.1 test") + incompleteResourceAttributes.PutStr(semconv.AttributeTelemetrySDKName, "opentelemetry") + incompleteResourceAttributes.PutStr(semconv.AttributeTelemetrySDKVersion, "0.0.1 test") tests := []struct { name string diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go index 94d7487606..cab4106676 100644 --- a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -109,31 +109,33 @@ func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcom attributes.PutStr(mappingKey, val.Str()) } } - if _, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { - if val, found := resourceAttributes.Get(attr.AWSHostedInEnvironment); found { - attributes.PutStr(attr.AWSLocalEnvironment, val.Str()) - } else { - if h.defaultEnvPrefix == appsignalsconfig.PlatformECS { - if clusterName, _ := getECSClusterName(resourceAttributes); clusterName != "" { - attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, clusterName)) - } else if clusterName = ecsutil.GetECSUtilSingleton().Cluster; clusterName != "" { - attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, clusterName)) - } - } else if h.defaultEnvPrefix == appsignalsconfig.PlatformEC2 { - if asgAttr, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, asgAttr.Str())) - } - } - } - } - if _, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { - attributes.PutStr(attr.AWSLocalEnvironment, getDefaultEnvironment(h.defaultEnvPrefix, AttributeEnvironmentDefault)) - } + attributes.PutStr(attr.AWSLocalEnvironment, getLocalEnvironment(attributes, resourceAttributes, h.defaultEnvPrefix)) attributes.PutStr(common.AttributePlatformType, h.platformType) - return nil } +func getLocalEnvironment(attributes, resourceAttributes pcommon.Map, defaultEnvPrefix string) string { + if val, ok := attributes.Get(attr.AWSLocalEnvironment); ok { + return val.Str() + } + if val, found := resourceAttributes.Get(attr.AWSHostedInEnvironment); found { + return val.Str() + } + if defaultEnvPrefix == appsignalsconfig.PlatformECS { + if clusterName, _ := getECSClusterName(resourceAttributes); clusterName != "" { + return getDefaultEnvironment(defaultEnvPrefix, clusterName) + } + if clusterName := ecsutil.GetECSUtilSingleton().Cluster; clusterName != "" { + return getDefaultEnvironment(defaultEnvPrefix, clusterName) + } + } else if defaultEnvPrefix == appsignalsconfig.PlatformEC2 { + if asgAttr, found := resourceAttributes.Get(attr.ResourceDetectionASG); found { + return getDefaultEnvironment(defaultEnvPrefix, asgAttr.Str()) + } + } + return getDefaultEnvironment(defaultEnvPrefix, AttributeEnvironmentDefault) +} + func getECSClusterName(resourceAttributes pcommon.Map) (string, bool) { if clusterAttr, ok := resourceAttributes.Get(semconv.AttributeAWSECSClusterARN); ok { parts := strings.Split(clusterAttr.Str(), "/") From 3a4959bdb220ad77c4a039c7315c931584b4b650 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 17 May 2024 19:34:35 -0700 Subject: [PATCH 11/11] Fix Release Testing for Java and Python EKS/EC2/ASG (#1181) --- ...lication-signals-java-e2e-ec2-asg-test.yml | 168 ++++++++++ ...application-signals-java-e2e-ec2-test.yml} | 25 +- ...application-signals-java-e2e-eks-test.yml} | 30 +- ...cation-signals-python-e2e-ec2-asg-test.yml | 167 +++++++++ ...pplication-signals-python-e2e-ec2-test.yml | 160 +++++++++ ...pplication-signals-python-e2e-eks-test.yml | 317 ++++++++++++++++++ .github/workflows/integration-test.yml | 46 ++- 7 files changed, 888 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/application-signals-java-e2e-ec2-asg-test.yml rename .github/workflows/{appsignals-e2e-ec2-test.yml => application-signals-java-e2e-ec2-test.yml} (85%) rename .github/workflows/{appsignals-e2e-eks-test.yml => application-signals-java-e2e-eks-test.yml} (90%) create mode 100644 .github/workflows/application-signals-python-e2e-ec2-asg-test.yml create mode 100644 .github/workflows/application-signals-python-e2e-ec2-test.yml create mode 100644 .github/workflows/application-signals-python-e2e-eks-test.yml diff --git a/.github/workflows/application-signals-java-e2e-ec2-asg-test.yml b/.github/workflows/application-signals-java-e2e-ec2-asg-test.yml new file mode 100644 index 0000000000..af25972507 --- /dev/null +++ b/.github/workflows/application-signals-java-e2e-ec2-asg-test.yml @@ -0,0 +1,168 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the E2E test for App Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: App Signals Enablement E2E Testing - EC2 ASG Use Case +on: + workflow_call: + +permissions: + id-token: write + contents: read + +env: + # The presence of this env var is required for use by terraform and AWS CLI commands + # It is not redundant + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_FRONTEND_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/main-service.jar" + SAMPLE_APP_REMOTE_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/remote-service.jar" + GET_ADOT_JAR_COMMAND: "aws s3 cp s3://adot-main-build-staging-jar/aws-opentelemetry-agent.jar ./adot.jar" + GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + +jobs: + e2e-ec2-single-asg-test: + runs-on: ubuntu-latest + steps: + - name: Get testing resources from aws-application-signals-test-framework + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: add-ec2-platform-support + + - name: Generate testing id + run: echo TESTING_ID="java-asg-${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/ec2/asg + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="sample_app_jar=${{ env.SAMPLE_APP_FRONTEND_SERVICE_JAR }}" \ + -var="sample_remote_app_jar=${{ env.SAMPLE_APP_REMOTE_SERVICE_JAR }}" \ + -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \ + -var="get_adot_jar_command=${{ env.GET_ADOT_JAR_COMMAND }}" + + - name: Get sample app and EC2 instance information + working-directory: terraform/ec2/asg + run: | + main_service_instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names ec2-single-asg-${{ env.TESTING_ID }} --region ${{ env.AWS_DEFAULT_REGION }} --query "AutoScalingGroups[].Instances[0].InstanceId" --output text) + main_service_public_ip=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PublicIpAddress" --output text) + main_service_private_dns_name=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PrivateDnsName" --output text) + echo "INSTANCE_ID=$main_service_instance_id" >> $GITHUB_ENV + echo "MAIN_SERVICE_ENDPOINT=$main_service_public_ip:8080" >> $GITHUB_ENV + echo "PRIVATE_DNS_NAME=$main_service_private_dns_name" >> $GITHUB_ENV + echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV + echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call" + + # Validation for pulse telemetry data + - name: Validate generated EMF logs + id: log-validation + run: ./gradlew validator:run --args='-c java/ec2/asg/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Validate generated metrics + id: metric-validation + if: (success() || steps.log-validation-1.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c java/ec2/asg/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Validate generated traces + id: trace-validation + if: (success() || steps.log-validation-1.outcome == 'failure' || steps.metric-validation-1.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c java/ec2/asg/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + # Clean up Procedures + - name: Terraform destroy + if: always() + continue-on-error: true + working-directory: terraform/ec2/asg + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" diff --git a/.github/workflows/appsignals-e2e-ec2-test.yml b/.github/workflows/application-signals-java-e2e-ec2-test.yml similarity index 85% rename from .github/workflows/appsignals-e2e-ec2-test.yml rename to .github/workflows/application-signals-java-e2e-ec2-test.yml index 95464b7918..79052fae38 100644 --- a/.github/workflows/appsignals-e2e-ec2-test.yml +++ b/.github/workflows/application-signals-java-e2e-ec2-test.yml @@ -18,9 +18,9 @@ env: APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} SAMPLE_APP_FRONTEND_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/main-service.jar" SAMPLE_APP_REMOTE_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/remote-service.jar" - GET_ADOT_JAR_COMMAND: "wget -O adot.jar https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest/download/aws-opentelemetry-agent.jar" + GET_ADOT_JAR_COMMAND: "aws s3 cp s3://adot-main-build-staging-jar/aws-opentelemetry-agent.jar ./adot.jar" GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" - METRIC_NAMESPACE: AppSignals + METRIC_NAMESPACE: ApplicationSignals LOG_GROUP_NAME: /aws/application-signals/data jobs: @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@v4 with: repository: aws-observability/aws-application-signals-test-framework - ref: main + ref: ga-release - name: Generate testing id run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV @@ -70,6 +70,7 @@ jobs: run: | echo "MAIN_SERVICE_ENDPOINT=$(terraform output sample_app_main_service_public_dns):8080" >> $GITHUB_ENV echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + echo "MAIN_SERVICE_INSTANCE_ID=$(terraform output main_service_instance_id)" >> $GITHUB_ENV - name: Wait for app endpoint to come online id: endpoint-check @@ -91,10 +92,10 @@ jobs: - name: Call all test APIs continue-on-error: true run: | - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call/; echo - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call/; echo - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}/; echo - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/; echo + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call" - name: Build Gradle working-directory: ${{ env.TEST_RESOURCES_FOLDER }} @@ -113,8 +114,9 @@ jobs: --log-group ${{ env.LOG_GROUP_NAME }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --request-body ip=${{ env.REMOTE_SERVICE_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' - name: Validate generated metrics @@ -130,8 +132,9 @@ jobs: --log-group ${{ env.LOG_GROUP_NAME }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --request-body ip=${{ env.REMOTE_SERVICE_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' - name: Validate generated traces @@ -147,8 +150,9 @@ jobs: --log-group ${{ env.LOG_GROUP_NAME }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --request-body ip=${{ env.REMOTE_SERVICE_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' # Clean up Procedures @@ -159,4 +163,3 @@ jobs: run: | terraform destroy -auto-approve \ -var="test_id=${{ env.TESTING_ID }}" - diff --git a/.github/workflows/appsignals-e2e-eks-test.yml b/.github/workflows/application-signals-java-e2e-eks-test.yml similarity index 90% rename from .github/workflows/appsignals-e2e-eks-test.yml rename to .github/workflows/application-signals-java-e2e-eks-test.yml index 011f4df596..7a5e64d0c4 100644 --- a/.github/workflows/appsignals-e2e-eks-test.yml +++ b/.github/workflows/application-signals-java-e2e-eks-test.yml @@ -24,9 +24,10 @@ env: SAMPLE_APP_NAMESPACE: sample-app-namespace SAMPLE_APP_FRONTEND_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_FRONTEND_SVC_IMG }} SAMPLE_APP_REMOTE_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_REMOTE_SVC_IMG }} - METRIC_NAMESPACE: AppSignals + METRIC_NAMESPACE: ApplicationSignals LOG_GROUP_NAME: /aws/application-signals/data ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" + APPLICATION_SIGNALS_ADOT_IMAGE: 611364707713.dkr.ecr.us-west-2.amazonaws.com/adot-autoinstrumentation-java-operator-staging:1.33.0-SNAPSHOT-91cbba8 jobs: appsignals-e2e-test: @@ -36,7 +37,7 @@ jobs: uses: actions/checkout@v4 with: repository: aws-observability/aws-application-signals-test-framework - ref: main + ref: ga-release - name: Download enablement script uses: actions/checkout@v4 @@ -131,6 +132,15 @@ jobs: run: | kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' kubectl delete pods --all -n amazon-cloudwatch + sleep 10 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + + - name: Patch the ADOT image and restart CloudWatch pods + run: | + kubectl patch deploy -namazon-cloudwatch amazon-cloudwatch-observability-controller-manager --type='json' \ + -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/args/1", "value": "--auto-instrumentation-java-image=${{ env.APPLICATION_SIGNALS_ADOT_IMAGE }}"}]' + kubectl delete pods --all -n amazon-cloudwatch + sleep 10 kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch # Application pods need to be restarted for the @@ -202,10 +212,10 @@ jobs: - name: Call all test APIs continue-on-error: true run: | - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/outgoing-http-call/; echo - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/aws-sdk-call/; echo - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}/; echo - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/client-call/; echo + curl -S -s "http://${{ env.APP_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.APP_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.APP_ENDPOINT }}/client-call" - name: Build Gradle working-directory: ${{ env.TEST_RESOURCES_FOLDER }} @@ -226,7 +236,7 @@ jobs: --platform-info ${{ inputs.test-cluster-name }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} - --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} --rollup' - name: Call endpoints and validate generated metrics @@ -244,7 +254,7 @@ jobs: --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} - --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} --rollup' - name: Call endpoints and validate generated traces @@ -261,7 +271,7 @@ jobs: --platform-info ${{ inputs.test-cluster-name }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} - --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} --rollup' # Clean up Procedures @@ -312,4 +322,4 @@ jobs: --name service-account-${{ env.TESTING_ID }} \ --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ --cluster ${{ inputs.test-cluster-name }} \ - --region ${{ env.AWS_DEFAULT_REGION }} \ No newline at end of file + --region ${{ env.AWS_DEFAULT_REGION }} diff --git a/.github/workflows/application-signals-python-e2e-ec2-asg-test.yml b/.github/workflows/application-signals-python-e2e-ec2-asg-test.yml new file mode 100644 index 0000000000..553e792e10 --- /dev/null +++ b/.github/workflows/application-signals-python-e2e-ec2-asg-test.yml @@ -0,0 +1,167 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the Python E2E Canary test for Application Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: Application Signals Enablement E2E Testing - Python EC2 Asg Use Case +on: + workflow_call: + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-us-east-1/python-sample-app.zip + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + ADOT_WHEEL_NAME: ${{ inputs.staging_wheel_name }} + TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE} + GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" + GET_ADOT_WHEEL_COMMAND: "aws s3 cp s3://metric-schema-changes/aws_opentelemetry_distro-0.2.0-py3-none-any.whl ./aws_opentelemetry_distro-0.2.0-py3-none-any.whl && python3.9 -m pip install aws_opentelemetry_distro-0.2.0-py3-none-any.whl" + +jobs: + python-e2e-ec2-asg-test: + runs-on: ubuntu-latest + steps: + - name: Get testing resources from aws-application-signals-test-framework + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: add-ec2-platform-for-python-ga + + - name: Generate testing id + run: echo TESTING_ID="python-asg-${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/python/ec2/asg + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \ + -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \ + -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" + + - name: Get sample app and EC2 instance information + working-directory: terraform/python/ec2/asg + run: | + main_service_instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names python-ec2-single-asg-${{ env.TESTING_ID }} --region ${{ env.AWS_DEFAULT_REGION }} --query "AutoScalingGroups[].Instances[0].InstanceId" --output text) + main_service_public_ip=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PublicIpAddress" --output text) + main_service_private_dns_name=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PrivateDnsName" --output text) + echo "INSTANCE_ID=$main_service_instance_id" >> $GITHUB_ENV + echo "MAIN_SERVICE_ENDPOINT=$main_service_public_ip:8000" >> $GITHUB_ENV + echo "PRIVATE_DNS_NAME=$main_service_private_dns_name" >> $GITHUB_ENV + echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV + echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call" + + - name: Build Gradle + run: ./gradlew + + # Validation for pulse telemetry data + - name: Validate generated EMF logs + id: log-validation + run: ./gradlew validator:run --args='-c python/ec2/asg/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info python-ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --rollup' + + - name: Validate generated metrics + id: metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/asg/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info python-ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --rollup' + + - name: Validate generated traces + id: trace-validation + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/asg/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info python-ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --rollup' + + # Clean up Procedures + - name: Terraform destroy + if: always() + continue-on-error: true + working-directory: terraform/python/ec2/asg + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ No newline at end of file diff --git a/.github/workflows/application-signals-python-e2e-ec2-test.yml b/.github/workflows/application-signals-python-e2e-ec2-test.yml new file mode 100644 index 0000000000..470bb492ac --- /dev/null +++ b/.github/workflows/application-signals-python-e2e-ec2-test.yml @@ -0,0 +1,160 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the Python E2E Canary test for Application Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: Application Signals Enablement E2E Testing - Python EC2 Use Case +on: + workflow_call: + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-us-east-1/python-sample-app.zip + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE} + GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" + GET_ADOT_WHEEL_COMMAND: "aws s3 cp s3://metric-schema-changes/aws_opentelemetry_distro-0.2.0-py3-none-any.whl ./aws_opentelemetry_distro-0.2.0-py3-none-any.whl && python3.9 -m pip install aws_opentelemetry_distro-0.2.0-py3-none-any.whl" + +jobs: + python-e2e-ec2-test: + runs-on: ubuntu-latest + steps: + - name: Get testing resources from aws-application-signals-test-framework + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: ga-python + + - name: Generate testing id + run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/python/ec2 + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \ + -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \ + -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" + + - name: Get the ec2 instance ami id + working-directory: terraform/python/ec2 + run: | + echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV + + - name: Get the sample app endpoint + working-directory: terraform/python/ec2 + run: | + echo "MAIN_SERVICE_ENDPOINT=$(terraform output sample_app_main_service_public_dns):8000" >> $GITHUB_ENV + echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + echo "MAIN_SERVICE_INSTANCE_ID=$(terraform output main_service_instance_id)" >> $GITHUB_ENV + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call/; echo + curl -S -s -o /dev/null "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/; echo + + - name: Build Gradle + run: ./gradlew + + # Validation for pulse telemetry data + - name: Validate generated EMF logs + id: log-validation + run: ./gradlew validator:run --args='-c python/ec2/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + + - name: Validate generated metrics + id: metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + + - name: Validate generated traces + id: trace-validation + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + + # Clean up Procedures + - name: Terraform destroy + if: always() + continue-on-error: true + working-directory: terraform/ec2 + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ No newline at end of file diff --git a/.github/workflows/application-signals-python-e2e-eks-test.yml b/.github/workflows/application-signals-python-e2e-eks-test.yml new file mode 100644 index 0000000000..f33e12a553 --- /dev/null +++ b/.github/workflows/application-signals-python-e2e-eks-test.yml @@ -0,0 +1,317 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the E2E test for Application Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: Application Signals Enablement E2E Testing - Python EKS +on: + workflow_call: + inputs: + test-cluster-name: + required: true + type: string + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_NAMESPACE: sample-app-namespace + SAMPLE_APP_FRONTEND_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_FRONTEND_SVC_IMG }} + SAMPLE_APP_REMOTE_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_REMOTE_SVC_IMG }} + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" + APPLICATION_SIGNALS_ADOT_IMAGE: 637423224110.dkr.ecr.us-east-1.amazonaws.com/aws-observability/adot-autoinstrumentation-python-staging:0.2.0-408d938 + +jobs: + python-e2e-eks-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: ga-python + + - name: Download enablement script + uses: actions/checkout@v4 + with: + repository: aws-observability/application-signals-demo + ref: main + path: enablement-script + sparse-checkout: | + scripts/eks/appsignals/enable-app-signals.sh + scripts/eks/appsignals/clean-app-signals.sh + sparse-checkout-cone-mode: false + + - name: Generate testing id + run: echo TESTING_ID="${{ env.AWS_DEFAULT_REGION }}-${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # local directory to store the kubernetes config + - name: Create kubeconfig directory + run: mkdir -p ${{ github.workspace }}/.kube + + - name: Set KUBECONFIG environment variable + run: echo KUBECONFIG="${{ github.workspace }}/.kube/config" >> $GITHUB_ENV + + - name: Set up kubeconfig + run: aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ env.AWS_DEFAULT_REGION }} + + - name: Install eksctl + run: | + mkdir ${{ github.workspace }}/eksctl + curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz" + tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz + echo "${{ github.workspace }}/eksctl" >> $GITHUB_PATH + + - name: Create role for AWS access from the sample app + id: create_service_account + run: | + eksctl create iamserviceaccount \ + --name service-account-${{ env.TESTING_ID }} \ + --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ + --cluster ${{ inputs.test-cluster-name }} \ + --role-name eks-s3-access-${{ env.TESTING_ID }} \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \ + --region ${{ env.AWS_DEFAULT_REGION }} \ + --approve + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/python/eks + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="kube_directory_path=${{ github.workspace }}/.kube" \ + -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ + -var="eks_cluster_context_name=$(kubectl config current-context)" \ + -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ + -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ + -var="python_app_image=654654176582.dkr.ecr.us-east-1.amazonaws.com/appsignals-python-django-main-service" \ + -var="python_remote_app_image=654654176582.dkr.ecr.us-east-1.amazonaws.com/appsignals-python-django-remote-service" + + # Enable App Signals on the test cluster + - name: Enable App Signals + working-directory: enablement-script/scripts/eks/appsignals + run: | + ./enable-app-signals.sh \ + ${{ inputs.test-cluster-name }} \ + ${{ env.AWS_DEFAULT_REGION }} \ + ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Save CloudWatch image to environment before patching + run: | + echo "OLD_CW_AGENT_IMAGE"=$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o json | \ + jq '.items[0].status.containerStatuses[0].image') >> $GITHUB_ENV + + - name: Patch the CloudWatch Agent image and restart CloudWatch pods + run: | + kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' + kubectl delete pods --all -n amazon-cloudwatch + sleep 10 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + + - name: Patch the ADOT image and restart CloudWatch pods + run: | + kubectl patch deploy -namazon-cloudwatch amazon-cloudwatch-observability-controller-manager --type='json' \ + -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/args/2", "value": "--auto-instrumentation-python-image=${{ env.APPLICATION_SIGNALS_ADOT_IMAGE }}"}]' + kubectl delete pods --all -n amazon-cloudwatch + sleep 10 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + + # Application pods need to be restarted for the + # app signals instrumentation to take effect + - name: Restart the app pods + run: kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Wait for sample app pods to come up + run: | + kubectl wait --for=condition=Ready pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }} \ + + - name: Get remote service deployment name and IP + run: | + echo "REMOTE_SERVICE_DEPLOYMENT_NAME=$(kubectl get deployments -n ${{ env.SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].metadata.name}')" >> $GITHUB_ENV + echo "REMOTE_SERVICE_POD_IP=$(kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].status.podIP}')" >> $GITHUB_ENV + + - name: Log pod ADOT image ID + run: | + kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --output json | \ + jq '.items[0].status.initContainerStatuses[0].imageID' + + - name: Log pod CWAgent Operator image ID + run: | + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-observability -o json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + - name: Log pod FluentBit image ID + run: | + kubectl get pods -n amazon-cloudwatch -l k8s-app=fluent-bit -o json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + - name: Log pod CWAgent image ID and save image to the environment + run: | + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + echo "NEW_CW_AGENT_IMAGE"=$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o json | \ + jq '.items[0].status.containerStatuses[0].image') >> $GITHUB_ENV + +# - name: Check if CW Agent image has changed +# run: | +# if [ ${{ env.OLD_CW_AGENT_IMAGE }} = ${{ env.NEW_CW_AGENT_IMAGE }} ]; then +# echo "Operator image did not change" +# exit 1 +# fi + + - name: Get the sample app endpoint + run: | + echo "APP_ENDPOINT=$(terraform output python_app_endpoint)" >> $GITHUB_ENV + working-directory: terraform/python/eks + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.APP_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/outgoing-http-call"; echo + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/client-call"; echo + + - name: Build Gradle + run: ./gradlew + + # Validation for application signals telemetry data + - name: Call endpoint and validate generated EMF logs + id: log-validation + if: steps.endpoint-check.outcome == 'success' && !cancelled() + run: ./gradlew validator:run --args='-c python/eks/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --platform-info ${{ inputs.test-cluster-name }} + --service-name python-application-${{ env.TESTING_ID }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Call endpoints and validate generated metrics + id: metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/eks/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --platform-info ${{ inputs.test-cluster-name }} + --service-name python-application-${{ env.TESTING_ID }} + --remote-service-name python-remote-application-${{ env.TESTING_ID }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Call endpoints and validate generated traces + id: trace-validation + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/eks/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --log-group ${{ env.LOG_GROUP_NAME }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --platform-info ${{ inputs.test-cluster-name }} + --service-name python-application-${{ env.TESTING_ID }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + # Clean up Procedures + + - name: Remove log group deletion command + if: always() + working-directory: enablement-script/scripts/eks/appsignals + run: | + delete_log_group="aws logs delete-log-group --log-group-name '${{ env.LOG_GROUP_NAME }}' --region \$REGION" + sed -i "s#$delete_log_group##g" clean-app-signals.sh + + - name: Clean Up Application Signals + if: always() + continue-on-error: true + working-directory: enablement-script + run: | + ./clean-app-signals.sh \ + ${{ inputs.test-cluster-name }} \ + ${{ inputs.aws-region }} \ + ${{ env.SAMPLE_APP_NAMESPACE }} + + # This step also deletes lingering resources from previous test runs + - name: Delete all sample app resources + if: always() + continue-on-error: true + timeout-minutes: 10 + run: kubectl delete namespace ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Terraform destroy + if: always() + continue-on-error: true + timeout-minutes: 5 + working-directory: terraform/python/eks + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="aws_region=${{ inputs.aws-region }}" \ + -var="kube_directory_path=${{ github.workspace }}/.kube" \ + -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ + -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ + -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ + -var="python_app_image=${{ env.ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws-region }}.amazonaws.com/${{ secrets.APP_SIGNALS_PYTHON_E2E_FE_SA_IMG }}" \ + -var="python_remote_app_image=${{ env.ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws-region }}.amazonaws.com/${{ secrets.APP_SIGNALS_PYTHON_E2E_RE_SA_IMG }}" + + - name: Remove aws access service account + if: always() + continue-on-error: true + run: | + eksctl delete iamserviceaccount \ + --name service-account-${{ env.TESTING_ID }} \ + --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ + --cluster ${{ inputs.test-cluster-name }} \ + --region ${{ inputs.aws-region }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index cf58ab81d2..4c806cba83 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1298,10 +1298,10 @@ jobs: retry_wait_seconds: 5 command: cd terraform/stress && terraform destroy --auto-approve - EKSEndToEndTest: + JavaEKSEndToEndTest: name: "AppSignals E2E EKS Test" needs: [ BuildAndUpload, BuildDocker ] - uses: ./.github/workflows/appsignals-e2e-eks-test.yml + uses: ./.github/workflows/application-signals-java-e2e-eks-test.yml permissions: id-token: write contents: read @@ -1309,10 +1309,48 @@ jobs: with: test-cluster-name: 'e2e-cw-agent-test' - EC2EndToEndTest: + JavaEC2EndToEndTest: name: "AppSignals E2E EC2 Test" needs: [ BuildAndUpload, BuildDocker ] - uses: ./.github/workflows/appsignals-e2e-ec2-test.yml + uses: ./.github/workflows/application-signals-java-e2e-ec2-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + + PythonEKSEndToEndTest: + name: "AppSignals E2E EKS Test" + needs: [ BuildAndUpload, BuildDocker, JavaEKSEndToEndTest ] + uses: ./.github/workflows/application-signals-python-e2e-eks-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + with: + test-cluster-name: 'e2e-cw-agent-test' + + PythonEC2EndToEndTest: + name: "AppSignals E2E EC2 Test" + needs: [ BuildAndUpload, BuildDocker, JavaEC2EndToEndTest ] + uses: ./.github/workflows/application-signals-python-e2e-ec2-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + + JavaEC2ASGEndToEndTest: + name: "AppSignals E2E EC2 Test" + needs: [ BuildAndUpload, BuildDocker ] + uses: ./.github/workflows/application-signals-java-e2e-ec2-asg-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + + PythonEC2ASGEndToEndTest: + name: "AppSignals E2E EC2 Test" + needs: [ BuildAndUpload, BuildDocker ] + uses: ./.github/workflows/application-signals-python-e2e-ec2-asg-test.yml permissions: id-token: write contents: read