From 4193fa17b3759581753480cd378f0c6ef47043bb Mon Sep 17 00:00:00 2001 From: "Mengyi Zhou (bjrara)" Date: Mon, 20 May 2024 14:27:23 -0700 Subject: [PATCH] [Application Signals] Improve metric schema for EKS, Native K8s, EC2 (#1179) Co-authored-by: Harry Co-authored-by: Lisa Guo --- ...lication-signals-java-e2e-ec2-asg-test.yml | 168 ++++++++++ ...application-signals-java-e2e-ec2-test.yml} | 27 +- ...application-signals-java-e2e-eks-test.yml} | 32 +- ...cation-signals-python-e2e-ec2-asg-test.yml | 167 +++++++++ ...pplication-signals-python-e2e-ec2-test.yml | 160 +++++++++ ...pplication-signals-python-e2e-eks-test.yml | 317 ++++++++++++++++++ .github/workflows/integration-test.yml | 46 ++- .../README.md | 36 +- .../awsapplicationsignals/common/types.go | 43 +++ .../config/config.go | 4 +- .../config/config_test.go | 0 .../config/resolvers.go | 2 + .../config/resolvers_test.go | 0 .../factory.go | 10 +- .../factory_test.go | 6 +- .../internal/attributes/attributes.go | 24 ++ .../cardinalitycontrol/count_min_sketch.go | 0 .../count_min_sketch_test.go | 0 .../cardinalitycontrol/metrics_limiter.go | 20 +- .../metrics_limiter_test.go | 29 +- .../normalizer/attributesnormalizer.go | 234 +++++++++++++ .../normalizer/attributesnormalizer_test.go | 59 +++- .../internal/prune/metric_pruner.go | 42 +++ .../internal/prune/metric_pruner_test.go | 85 +++++ .../internal/resolver/attributesresolver.go | 163 +++++++++ .../resolver/attributesresolver_test.go | 247 ++++++++++++++ .../internal/resolver/kubernetes.go | 70 ++-- .../internal/resolver/kubernetes_test.go | 264 +++++++++++---- .../processor.go | 45 +-- .../processor_test.go | 12 +- .../rules/common.go | 31 +- .../rules/common_test.go | 46 +++ .../rules/dropper.go | 0 .../rules/dropper_test.go | 0 .../rules/keeper.go | 2 +- .../rules/keeper_test.go | 0 .../rules/replacer.go | 17 +- .../rules/replacer_test.go | 71 ++++ .../testdata/config_eks.yaml | 0 .../testdata/config_generic.yaml | 0 .../processors/awsappsignals/common/types.go | 29 -- .../internal/attributes/attributes.go | 42 --- .../normalizer/attributesnormalizer.go | 154 --------- .../internal/resolver/attributesresolver.go | 102 ------ .../resolver/attributesresolver_test.go | 119 ------- .../awsappsignals/internal/resolver/ec2.go | 59 ---- .../internal/resolver/ec2_test.go | 148 -------- .../awsappsignals/rules/common_test.go | 23 -- service/defaultcomponents/components.go | 4 +- translator/config/mode.go | 4 + .../appsignals_and_eks_config.yaml | 90 +++-- .../appsignals_and_k8s_config.yaml | 94 +++--- .../appsignals_fallback_and_eks_config.yaml | 90 +++-- .../appsignals_over_fallback_config.yaml | 90 +++-- .../sampleConfig/base_appsignals_config.yaml | 48 +-- .../base_appsignals_fallback_config.yaml | 48 +-- .../awsemf/appsignals_config_ec2.yaml | 33 -- .../awsemf/appsignals_config_eks.yaml | 40 +-- .../awsemf/appsignals_config_generic.yaml | 30 +- .../awsemf/appsignals_config_k8s.yaml | 40 +-- .../otel/exporter/awsemf/translator.go | 25 +- .../otel/exporter/awsxray/translator.go | 33 +- .../otel/exporter/awsxray/translator_test.go | 24 +- .../translator.go | 6 +- .../translator_test.go | 2 +- .../testdata/config_ec2.yaml | 0 .../testdata/config_eks.yaml | 0 .../testdata/config_generic.yaml | 0 .../testdata/config_k8s.yaml | 0 .../testdata/invalidRulesConfig.json | 0 .../testdata/validRulesConfig.json | 0 .../testdata/validRulesConfigEKS.yaml | 0 .../testdata/validRulesConfigGeneric.yaml | 0 .../translator.go | 36 +- .../translator_test.go | 18 +- translator/translate/otel/translate_otel.go | 6 +- 76 files changed, 2602 insertions(+), 1314 deletions(-) create mode 100644 .github/workflows/application-signals-java-e2e-ec2-asg-test.yml rename .github/workflows/{appsignals-e2e-ec2-test.yml => application-signals-java-e2e-ec2-test.yml} (85%) rename .github/workflows/{appsignals-e2e-eks-test.yml => application-signals-java-e2e-eks-test.yml} (90%) create mode 100644 .github/workflows/application-signals-python-e2e-ec2-asg-test.yml create mode 100644 .github/workflows/application-signals-python-e2e-ec2-test.yml create mode 100644 .github/workflows/application-signals-python-e2e-eks-test.yml rename plugins/processors/{awsappsignals => awsapplicationsignals}/README.md (92%) create mode 100644 plugins/processors/awsapplicationsignals/common/types.go rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/config.go (95%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/config_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/resolvers.go (95%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/config/resolvers_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/factory.go (89%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/factory_test.go (97%) create mode 100644 plugins/processors/awsapplicationsignals/internal/attributes/attributes.go rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/count_min_sketch.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/count_min_sketch_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/metrics_limiter.go (95%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/cardinalitycontrol/metrics_limiter_test.go (87%) create mode 100644 plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/normalizer/attributesnormalizer_test.go (68%) create mode 100644 plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go create mode 100644 plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go create mode 100644 plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go create mode 100644 plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/resolver/kubernetes.go (91%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/internal/resolver/kubernetes_test.go (82%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/processor.go (85%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/processor_test.go (97%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/common.go (70%) create mode 100644 plugins/processors/awsapplicationsignals/rules/common_test.go rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/dropper.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/dropper_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/keeper.go (97%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/keeper_test.go (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/replacer.go (80%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/rules/replacer_test.go (77%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/testdata/config_eks.yaml (100%) rename plugins/processors/{awsappsignals => awsapplicationsignals}/testdata/config_generic.yaml (100%) delete mode 100644 plugins/processors/awsappsignals/common/types.go delete mode 100644 plugins/processors/awsappsignals/internal/attributes/attributes.go delete mode 100644 plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go delete mode 100644 plugins/processors/awsappsignals/internal/resolver/attributesresolver.go delete mode 100644 plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go delete mode 100644 plugins/processors/awsappsignals/internal/resolver/ec2.go delete mode 100644 plugins/processors/awsappsignals/internal/resolver/ec2_test.go delete mode 100644 plugins/processors/awsappsignals/rules/common_test.go delete mode 100644 translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml rename translator/translate/otel/pipeline/{appsignals => applicationsignals}/translator.go (93%) rename translator/translate/otel/pipeline/{appsignals => applicationsignals}/translator_test.go (99%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_ec2.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_eks.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_generic.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/config_k8s.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/invalidRulesConfig.json (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/validRulesConfig.json (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/validRulesConfigEKS.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/testdata/validRulesConfigGeneric.yaml (100%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/translator.go (89%) rename translator/translate/otel/processor/{awsappsignals => awsapplicationsignals}/translator_test.go (91%) diff --git a/.github/workflows/application-signals-java-e2e-ec2-asg-test.yml b/.github/workflows/application-signals-java-e2e-ec2-asg-test.yml new file mode 100644 index 0000000000..af25972507 --- /dev/null +++ b/.github/workflows/application-signals-java-e2e-ec2-asg-test.yml @@ -0,0 +1,168 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the E2E test for App Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: App Signals Enablement E2E Testing - EC2 ASG Use Case +on: + workflow_call: + +permissions: + id-token: write + contents: read + +env: + # The presence of this env var is required for use by terraform and AWS CLI commands + # It is not redundant + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_FRONTEND_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/main-service.jar" + SAMPLE_APP_REMOTE_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/remote-service.jar" + GET_ADOT_JAR_COMMAND: "aws s3 cp s3://adot-main-build-staging-jar/aws-opentelemetry-agent.jar ./adot.jar" + GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + +jobs: + e2e-ec2-single-asg-test: + runs-on: ubuntu-latest + steps: + - name: Get testing resources from aws-application-signals-test-framework + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: add-ec2-platform-support + + - name: Generate testing id + run: echo TESTING_ID="java-asg-${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/ec2/asg + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="sample_app_jar=${{ env.SAMPLE_APP_FRONTEND_SERVICE_JAR }}" \ + -var="sample_remote_app_jar=${{ env.SAMPLE_APP_REMOTE_SERVICE_JAR }}" \ + -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \ + -var="get_adot_jar_command=${{ env.GET_ADOT_JAR_COMMAND }}" + + - name: Get sample app and EC2 instance information + working-directory: terraform/ec2/asg + run: | + main_service_instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names ec2-single-asg-${{ env.TESTING_ID }} --region ${{ env.AWS_DEFAULT_REGION }} --query "AutoScalingGroups[].Instances[0].InstanceId" --output text) + main_service_public_ip=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PublicIpAddress" --output text) + main_service_private_dns_name=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PrivateDnsName" --output text) + echo "INSTANCE_ID=$main_service_instance_id" >> $GITHUB_ENV + echo "MAIN_SERVICE_ENDPOINT=$main_service_public_ip:8080" >> $GITHUB_ENV + echo "PRIVATE_DNS_NAME=$main_service_private_dns_name" >> $GITHUB_ENV + echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV + echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call" + + # Validation for pulse telemetry data + - name: Validate generated EMF logs + id: log-validation + run: ./gradlew validator:run --args='-c java/ec2/asg/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Validate generated metrics + id: metric-validation + if: (success() || steps.log-validation-1.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c java/ec2/asg/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Validate generated traces + id: trace-validation + if: (success() || steps.log-validation-1.outcome == 'failure' || steps.metric-validation-1.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c java/ec2/asg/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + # Clean up Procedures + - name: Terraform destroy + if: always() + continue-on-error: true + working-directory: terraform/ec2/asg + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" diff --git a/.github/workflows/appsignals-e2e-ec2-test.yml b/.github/workflows/application-signals-java-e2e-ec2-test.yml similarity index 85% rename from .github/workflows/appsignals-e2e-ec2-test.yml rename to .github/workflows/application-signals-java-e2e-ec2-test.yml index da1bd70139..79052fae38 100644 --- a/.github/workflows/appsignals-e2e-ec2-test.yml +++ b/.github/workflows/application-signals-java-e2e-ec2-test.yml @@ -18,10 +18,10 @@ env: APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} SAMPLE_APP_FRONTEND_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/main-service.jar" SAMPLE_APP_REMOTE_SERVICE_JAR: "s3://aws-appsignals-sample-app-prod-us-east-1/remote-service.jar" - GET_ADOT_JAR_COMMAND: "wget -O adot.jar https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest/download/aws-opentelemetry-agent.jar" + GET_ADOT_JAR_COMMAND: "aws s3 cp s3://adot-main-build-staging-jar/aws-opentelemetry-agent.jar ./adot.jar" GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" - METRIC_NAMESPACE: AppSignals - LOG_GROUP_NAME: /aws/appsignals/generic + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data jobs: e2e-ec2-test: @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@v4 with: repository: aws-observability/aws-application-signals-test-framework - ref: main + ref: ga-release - name: Generate testing id run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV @@ -70,6 +70,7 @@ jobs: run: | echo "MAIN_SERVICE_ENDPOINT=$(terraform output sample_app_main_service_public_dns):8080" >> $GITHUB_ENV echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + echo "MAIN_SERVICE_INSTANCE_ID=$(terraform output main_service_instance_id)" >> $GITHUB_ENV - name: Wait for app endpoint to come online id: endpoint-check @@ -91,10 +92,10 @@ jobs: - name: Call all test APIs continue-on-error: true run: | - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call/; echo - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call/; echo - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}/; echo - curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/; echo + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call" - name: Build Gradle working-directory: ${{ env.TEST_RESOURCES_FOLDER }} @@ -113,8 +114,9 @@ jobs: --log-group ${{ env.LOG_GROUP_NAME }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --request-body ip=${{ env.REMOTE_SERVICE_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' - name: Validate generated metrics @@ -130,8 +132,9 @@ jobs: --log-group ${{ env.LOG_GROUP_NAME }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --request-body ip=${{ env.REMOTE_SERVICE_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' - name: Validate generated traces @@ -147,8 +150,9 @@ jobs: --log-group ${{ env.LOG_GROUP_NAME }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --request-body ip=${{ env.REMOTE_SERVICE_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' # Clean up Procedures @@ -159,4 +163,3 @@ jobs: run: | terraform destroy -auto-approve \ -var="test_id=${{ env.TESTING_ID }}" - diff --git a/.github/workflows/appsignals-e2e-eks-test.yml b/.github/workflows/application-signals-java-e2e-eks-test.yml similarity index 90% rename from .github/workflows/appsignals-e2e-eks-test.yml rename to .github/workflows/application-signals-java-e2e-eks-test.yml index 0d905deb10..7a5e64d0c4 100644 --- a/.github/workflows/appsignals-e2e-eks-test.yml +++ b/.github/workflows/application-signals-java-e2e-eks-test.yml @@ -24,9 +24,10 @@ env: SAMPLE_APP_NAMESPACE: sample-app-namespace SAMPLE_APP_FRONTEND_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_FRONTEND_SVC_IMG }} SAMPLE_APP_REMOTE_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_REMOTE_SVC_IMG }} - METRIC_NAMESPACE: AppSignals - LOG_GROUP_NAME: /aws/appsignals/eks + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" + APPLICATION_SIGNALS_ADOT_IMAGE: 611364707713.dkr.ecr.us-west-2.amazonaws.com/adot-autoinstrumentation-java-operator-staging:1.33.0-SNAPSHOT-91cbba8 jobs: appsignals-e2e-test: @@ -36,7 +37,7 @@ jobs: uses: actions/checkout@v4 with: repository: aws-observability/aws-application-signals-test-framework - ref: main + ref: ga-release - name: Download enablement script uses: actions/checkout@v4 @@ -131,6 +132,15 @@ jobs: run: | kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' kubectl delete pods --all -n amazon-cloudwatch + sleep 10 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + + - name: Patch the ADOT image and restart CloudWatch pods + run: | + kubectl patch deploy -namazon-cloudwatch amazon-cloudwatch-observability-controller-manager --type='json' \ + -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/args/1", "value": "--auto-instrumentation-java-image=${{ env.APPLICATION_SIGNALS_ADOT_IMAGE }}"}]' + kubectl delete pods --all -n amazon-cloudwatch + sleep 10 kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch # Application pods need to be restarted for the @@ -202,10 +212,10 @@ jobs: - name: Call all test APIs continue-on-error: true run: | - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/outgoing-http-call/; echo - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/aws-sdk-call/; echo - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}/; echo - curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/client-call/; echo + curl -S -s "http://${{ env.APP_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.APP_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.APP_ENDPOINT }}/client-call" - name: Build Gradle working-directory: ${{ env.TEST_RESOURCES_FOLDER }} @@ -226,7 +236,7 @@ jobs: --platform-info ${{ inputs.test-cluster-name }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} - --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} --rollup' - name: Call endpoints and validate generated metrics @@ -244,7 +254,7 @@ jobs: --service-name sample-application-${{ env.TESTING_ID }} --remote-service-name sample-remote-application-${{ env.TESTING_ID }} --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} - --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} --rollup' - name: Call endpoints and validate generated traces @@ -261,7 +271,7 @@ jobs: --platform-info ${{ inputs.test-cluster-name }} --service-name sample-application-${{ env.TESTING_ID }} --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} - --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} --rollup' # Clean up Procedures @@ -312,4 +322,4 @@ jobs: --name service-account-${{ env.TESTING_ID }} \ --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ --cluster ${{ inputs.test-cluster-name }} \ - --region ${{ env.AWS_DEFAULT_REGION }} \ No newline at end of file + --region ${{ env.AWS_DEFAULT_REGION }} diff --git a/.github/workflows/application-signals-python-e2e-ec2-asg-test.yml b/.github/workflows/application-signals-python-e2e-ec2-asg-test.yml new file mode 100644 index 0000000000..553e792e10 --- /dev/null +++ b/.github/workflows/application-signals-python-e2e-ec2-asg-test.yml @@ -0,0 +1,167 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the Python E2E Canary test for Application Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: Application Signals Enablement E2E Testing - Python EC2 Asg Use Case +on: + workflow_call: + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-us-east-1/python-sample-app.zip + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + ADOT_WHEEL_NAME: ${{ inputs.staging_wheel_name }} + TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE} + GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" + GET_ADOT_WHEEL_COMMAND: "aws s3 cp s3://metric-schema-changes/aws_opentelemetry_distro-0.2.0-py3-none-any.whl ./aws_opentelemetry_distro-0.2.0-py3-none-any.whl && python3.9 -m pip install aws_opentelemetry_distro-0.2.0-py3-none-any.whl" + +jobs: + python-e2e-ec2-asg-test: + runs-on: ubuntu-latest + steps: + - name: Get testing resources from aws-application-signals-test-framework + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: add-ec2-platform-for-python-ga + + - name: Generate testing id + run: echo TESTING_ID="python-asg-${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/python/ec2/asg + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \ + -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \ + -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" + + - name: Get sample app and EC2 instance information + working-directory: terraform/python/ec2/asg + run: | + main_service_instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names python-ec2-single-asg-${{ env.TESTING_ID }} --region ${{ env.AWS_DEFAULT_REGION }} --query "AutoScalingGroups[].Instances[0].InstanceId" --output text) + main_service_public_ip=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PublicIpAddress" --output text) + main_service_private_dns_name=$(aws ec2 describe-instances --instance-ids $main_service_instance_id --region ${{ env.AWS_DEFAULT_REGION }} --query "Reservations[].Instances[].PrivateDnsName" --output text) + echo "INSTANCE_ID=$main_service_instance_id" >> $GITHUB_ENV + echo "MAIN_SERVICE_ENDPOINT=$main_service_public_ip:8000" >> $GITHUB_ENV + echo "PRIVATE_DNS_NAME=$main_service_private_dns_name" >> $GITHUB_ENV + echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV + echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}" + curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call" + + - name: Build Gradle + run: ./gradlew + + # Validation for pulse telemetry data + - name: Validate generated EMF logs + id: log-validation + run: ./gradlew validator:run --args='-c python/ec2/asg/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info python-ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --rollup' + + - name: Validate generated metrics + id: metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/asg/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info python-ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --rollup' + + - name: Validate generated traces + id: trace-validation + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/asg/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --platform-info python-ec2-single-asg-${{ env.TESTING_ID }} + --instance-id ${{ env.INSTANCE_ID }} + --private-dns-name ${{ env.PRIVATE_DNS_NAME }} + --rollup' + + # Clean up Procedures + - name: Terraform destroy + if: always() + continue-on-error: true + working-directory: terraform/python/ec2/asg + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ No newline at end of file diff --git a/.github/workflows/application-signals-python-e2e-ec2-test.yml b/.github/workflows/application-signals-python-e2e-ec2-test.yml new file mode 100644 index 0000000000..470bb492ac --- /dev/null +++ b/.github/workflows/application-signals-python-e2e-ec2-test.yml @@ -0,0 +1,160 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the Python E2E Canary test for Application Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: Application Signals Enablement E2E Testing - Python EC2 Use Case +on: + workflow_call: + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-us-east-1/python-sample-app.zip + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE} + GET_CW_AGENT_RPM_COMMAND: "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm ./cw-agent.rpm" + GET_ADOT_WHEEL_COMMAND: "aws s3 cp s3://metric-schema-changes/aws_opentelemetry_distro-0.2.0-py3-none-any.whl ./aws_opentelemetry_distro-0.2.0-py3-none-any.whl && python3.9 -m pip install aws_opentelemetry_distro-0.2.0-py3-none-any.whl" + +jobs: + python-e2e-ec2-test: + runs-on: ubuntu-latest + steps: + - name: Get testing resources from aws-application-signals-test-framework + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: ga-python + + - name: Generate testing id + run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/python/ec2 + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \ + -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \ + -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" + + - name: Get the ec2 instance ami id + working-directory: terraform/python/ec2 + run: | + echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV + + - name: Get the sample app endpoint + working-directory: terraform/python/ec2 + run: | + echo "MAIN_SERVICE_ENDPOINT=$(terraform output sample_app_main_service_public_dns):8000" >> $GITHUB_ENV + echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_public_ip)" >> $GITHUB_ENV + echo "MAIN_SERVICE_INSTANCE_ID=$(terraform output main_service_instance_id)" >> $GITHUB_ENV + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call/; echo + curl -S -s -o /dev/null "http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null "http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/; echo + + - name: Build Gradle + run: ./gradlew + + # Validation for pulse telemetry data + - name: Validate generated EMF logs + id: log-validation + run: ./gradlew validator:run --args='-c python/ec2/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + + - name: Validate generated metrics + id: metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + + - name: Validate generated traces + id: trace-validation + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + + # Clean up Procedures + - name: Terraform destroy + if: always() + continue-on-error: true + working-directory: terraform/ec2 + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ No newline at end of file diff --git a/.github/workflows/application-signals-python-e2e-eks-test.yml b/.github/workflows/application-signals-python-e2e-eks-test.yml new file mode 100644 index 0000000000..f33e12a553 --- /dev/null +++ b/.github/workflows/application-signals-python-e2e-eks-test.yml @@ -0,0 +1,317 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +# This is a reusable workflow for running the E2E test for Application Signals. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: Application Signals Enablement E2E Testing - Python EKS +on: + workflow_call: + inputs: + test-cluster-name: + required: true + type: string + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + APP_SIGNALS_E2E_TEST_ACCOUNT_ID: ${{ secrets.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + SAMPLE_APP_NAMESPACE: sample-app-namespace + SAMPLE_APP_FRONTEND_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_FRONTEND_SVC_IMG }} + SAMPLE_APP_REMOTE_SERVICE_IMAGE: ${{ secrets.APP_SIGNALS_E2E_SAMPLE_APP_REMOTE_SVC_IMG }} + METRIC_NAMESPACE: ApplicationSignals + LOG_GROUP_NAME: /aws/application-signals/data + ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" + APPLICATION_SIGNALS_ADOT_IMAGE: 637423224110.dkr.ecr.us-east-1.amazonaws.com/aws-observability/adot-autoinstrumentation-python-staging:0.2.0-408d938 + +jobs: + python-e2e-eks-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: aws-observability/aws-application-signals-test-framework + ref: ga-python + + - name: Download enablement script + uses: actions/checkout@v4 + with: + repository: aws-observability/application-signals-demo + ref: main + path: enablement-script + sparse-checkout: | + scripts/eks/appsignals/enable-app-signals.sh + scripts/eks/appsignals/clean-app-signals.sh + sparse-checkout-cone-mode: false + + - name: Generate testing id + run: echo TESTING_ID="${{ env.AWS_DEFAULT_REGION }}-${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APP_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # local directory to store the kubernetes config + - name: Create kubeconfig directory + run: mkdir -p ${{ github.workspace }}/.kube + + - name: Set KUBECONFIG environment variable + run: echo KUBECONFIG="${{ github.workspace }}/.kube/config" >> $GITHUB_ENV + + - name: Set up kubeconfig + run: aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ env.AWS_DEFAULT_REGION }} + + - name: Install eksctl + run: | + mkdir ${{ github.workspace }}/eksctl + curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz" + tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz + echo "${{ github.workspace }}/eksctl" >> $GITHUB_PATH + + - name: Create role for AWS access from the sample app + id: create_service_account + run: | + eksctl create iamserviceaccount \ + --name service-account-${{ env.TESTING_ID }} \ + --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ + --cluster ${{ inputs.test-cluster-name }} \ + --role-name eks-s3-access-${{ env.TESTING_ID }} \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \ + --region ${{ env.AWS_DEFAULT_REGION }} \ + --approve + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: terraform/python/eks + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="aws_region=${{ env.AWS_DEFAULT_REGION }}" \ + -var="kube_directory_path=${{ github.workspace }}/.kube" \ + -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ + -var="eks_cluster_context_name=$(kubectl config current-context)" \ + -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ + -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ + -var="python_app_image=654654176582.dkr.ecr.us-east-1.amazonaws.com/appsignals-python-django-main-service" \ + -var="python_remote_app_image=654654176582.dkr.ecr.us-east-1.amazonaws.com/appsignals-python-django-remote-service" + + # Enable App Signals on the test cluster + - name: Enable App Signals + working-directory: enablement-script/scripts/eks/appsignals + run: | + ./enable-app-signals.sh \ + ${{ inputs.test-cluster-name }} \ + ${{ env.AWS_DEFAULT_REGION }} \ + ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Save CloudWatch image to environment before patching + run: | + echo "OLD_CW_AGENT_IMAGE"=$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o json | \ + jq '.items[0].status.containerStatuses[0].image') >> $GITHUB_ENV + + - name: Patch the CloudWatch Agent image and restart CloudWatch pods + run: | + kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' + kubectl delete pods --all -n amazon-cloudwatch + sleep 10 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + + - name: Patch the ADOT image and restart CloudWatch pods + run: | + kubectl patch deploy -namazon-cloudwatch amazon-cloudwatch-observability-controller-manager --type='json' \ + -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/args/2", "value": "--auto-instrumentation-python-image=${{ env.APPLICATION_SIGNALS_ADOT_IMAGE }}"}]' + kubectl delete pods --all -n amazon-cloudwatch + sleep 10 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + + # Application pods need to be restarted for the + # app signals instrumentation to take effect + - name: Restart the app pods + run: kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Wait for sample app pods to come up + run: | + kubectl wait --for=condition=Ready pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }} \ + + - name: Get remote service deployment name and IP + run: | + echo "REMOTE_SERVICE_DEPLOYMENT_NAME=$(kubectl get deployments -n ${{ env.SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].metadata.name}')" >> $GITHUB_ENV + echo "REMOTE_SERVICE_POD_IP=$(kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].status.podIP}')" >> $GITHUB_ENV + + - name: Log pod ADOT image ID + run: | + kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --output json | \ + jq '.items[0].status.initContainerStatuses[0].imageID' + + - name: Log pod CWAgent Operator image ID + run: | + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-observability -o json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + - name: Log pod FluentBit image ID + run: | + kubectl get pods -n amazon-cloudwatch -l k8s-app=fluent-bit -o json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + - name: Log pod CWAgent image ID and save image to the environment + run: | + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + echo "NEW_CW_AGENT_IMAGE"=$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o json | \ + jq '.items[0].status.containerStatuses[0].image') >> $GITHUB_ENV + +# - name: Check if CW Agent image has changed +# run: | +# if [ ${{ env.OLD_CW_AGENT_IMAGE }} = ${{ env.NEW_CW_AGENT_IMAGE }} ]; then +# echo "Operator image did not change" +# exit 1 +# fi + + - name: Get the sample app endpoint + run: | + echo "APP_ENDPOINT=$(terraform output python_app_endpoint)" >> $GITHUB_ENV + working-directory: terraform/python/eks + + - name: Wait for app endpoint to come online + id: endpoint-check + run: | + attempt_counter=0 + max_attempts=30 + until $(curl --output /dev/null --silent --head --fail http://${{ env.APP_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # This steps increases the speed of the validation by creating the telemetry data in advance + - name: Call all test APIs + continue-on-error: true + run: | + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/outgoing-http-call"; echo + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}"; echo + curl -S -s -o /dev/null "http://${{ env.APP_ENDPOINT }}/client-call"; echo + + - name: Build Gradle + run: ./gradlew + + # Validation for application signals telemetry data + - name: Call endpoint and validate generated EMF logs + id: log-validation + if: steps.endpoint-check.outcome == 'success' && !cancelled() + run: ./gradlew validator:run --args='-c python/eks/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --platform-info ${{ inputs.test-cluster-name }} + --service-name python-application-${{ env.TESTING_ID }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Call endpoints and validate generated metrics + id: metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/eks/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --platform-info ${{ inputs.test-cluster-name }} + --service-name python-application-${{ env.TESTING_ID }} + --remote-service-name python-remote-application-${{ env.TESTING_ID }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + - name: Call endpoints and validate generated traces + id: trace-validation + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/eks/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ env.APP_SIGNALS_E2E_TEST_ACCOUNT_ID }} + --log-group ${{ env.LOG_GROUP_NAME }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --platform-info ${{ inputs.test-cluster-name }} + --service-name python-application-${{ env.TESTING_ID }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }} + --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }} + --rollup' + + # Clean up Procedures + + - name: Remove log group deletion command + if: always() + working-directory: enablement-script/scripts/eks/appsignals + run: | + delete_log_group="aws logs delete-log-group --log-group-name '${{ env.LOG_GROUP_NAME }}' --region \$REGION" + sed -i "s#$delete_log_group##g" clean-app-signals.sh + + - name: Clean Up Application Signals + if: always() + continue-on-error: true + working-directory: enablement-script + run: | + ./clean-app-signals.sh \ + ${{ inputs.test-cluster-name }} \ + ${{ inputs.aws-region }} \ + ${{ env.SAMPLE_APP_NAMESPACE }} + + # This step also deletes lingering resources from previous test runs + - name: Delete all sample app resources + if: always() + continue-on-error: true + timeout-minutes: 10 + run: kubectl delete namespace ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Terraform destroy + if: always() + continue-on-error: true + timeout-minutes: 5 + working-directory: terraform/python/eks + run: | + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="aws_region=${{ inputs.aws-region }}" \ + -var="kube_directory_path=${{ github.workspace }}/.kube" \ + -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ + -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ + -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ + -var="python_app_image=${{ env.ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws-region }}.amazonaws.com/${{ secrets.APP_SIGNALS_PYTHON_E2E_FE_SA_IMG }}" \ + -var="python_remote_app_image=${{ env.ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws-region }}.amazonaws.com/${{ secrets.APP_SIGNALS_PYTHON_E2E_RE_SA_IMG }}" + + - name: Remove aws access service account + if: always() + continue-on-error: true + run: | + eksctl delete iamserviceaccount \ + --name service-account-${{ env.TESTING_ID }} \ + --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ + --cluster ${{ inputs.test-cluster-name }} \ + --region ${{ inputs.aws-region }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index cf58ab81d2..4c806cba83 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1298,10 +1298,10 @@ jobs: retry_wait_seconds: 5 command: cd terraform/stress && terraform destroy --auto-approve - EKSEndToEndTest: + JavaEKSEndToEndTest: name: "AppSignals E2E EKS Test" needs: [ BuildAndUpload, BuildDocker ] - uses: ./.github/workflows/appsignals-e2e-eks-test.yml + uses: ./.github/workflows/application-signals-java-e2e-eks-test.yml permissions: id-token: write contents: read @@ -1309,10 +1309,48 @@ jobs: with: test-cluster-name: 'e2e-cw-agent-test' - EC2EndToEndTest: + JavaEC2EndToEndTest: name: "AppSignals E2E EC2 Test" needs: [ BuildAndUpload, BuildDocker ] - uses: ./.github/workflows/appsignals-e2e-ec2-test.yml + uses: ./.github/workflows/application-signals-java-e2e-ec2-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + + PythonEKSEndToEndTest: + name: "AppSignals E2E EKS Test" + needs: [ BuildAndUpload, BuildDocker, JavaEKSEndToEndTest ] + uses: ./.github/workflows/application-signals-python-e2e-eks-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + with: + test-cluster-name: 'e2e-cw-agent-test' + + PythonEC2EndToEndTest: + name: "AppSignals E2E EC2 Test" + needs: [ BuildAndUpload, BuildDocker, JavaEC2EndToEndTest ] + uses: ./.github/workflows/application-signals-python-e2e-ec2-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + + JavaEC2ASGEndToEndTest: + name: "AppSignals E2E EC2 Test" + needs: [ BuildAndUpload, BuildDocker ] + uses: ./.github/workflows/application-signals-java-e2e-ec2-asg-test.yml + permissions: + id-token: write + contents: read + secrets: inherit + + PythonEC2ASGEndToEndTest: + name: "AppSignals E2E EC2 Test" + needs: [ BuildAndUpload, BuildDocker ] + uses: ./.github/workflows/application-signals-python-e2e-ec2-asg-test.yml permissions: id-token: write contents: read diff --git a/plugins/processors/awsappsignals/README.md b/plugins/processors/awsapplicationsignals/README.md similarity index 92% rename from plugins/processors/awsappsignals/README.md rename to plugins/processors/awsapplicationsignals/README.md index 4906b69a46..36893c2528 100644 --- a/plugins/processors/awsappsignals/README.md +++ b/plugins/processors/awsapplicationsignals/README.md @@ -50,36 +50,36 @@ A replacements section defines a matching against the dimensions of incoming met ## AWS AppSignals Processor Configuration Example ```yaml -awsappsignals: +awsapplicationsignals: resolvers: ["eks"] rules: - selectors: - dimension: Operation - match: "POST *" + match: "POST *" - dimension: RemoteService - match: "*" - action: keep - rule_name: "keep01" + match: "*" + action: keep + rule_name: "keep01" - selectors: - dimension: Operation - match: "GET *" + match: "GET *" - dimension: RemoteService - match: "*" - action: keep - rule_name: "keep02" + match: "*" + action: keep + rule_name: "keep02" - selectors: - dimension: Operation - match: "POST *" - action: drop - rule_name: "drop01" + match: "POST *" + action: drop + rule_name: "drop01" - selectors: - dimension: Operation - match: "*" - replacements: - - target_dimension: RemoteOperation - value: "This is a test string" - action: replace - rule_name: "replace01" + match: "*" + replacements: + - target_dimension: RemoteOperation + value: "This is a test string" + action: replace + rule_name: "replace01" ``` ## Amazon CloudWatch Agent Configuration Example diff --git a/plugins/processors/awsapplicationsignals/common/types.go b/plugins/processors/awsapplicationsignals/common/types.go new file mode 100644 index 0000000000..a9e72a2fd3 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/common/types.go @@ -0,0 +1,43 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package common + +const ( + MetricAttributeLocalService = "Service" + MetricAttributeLocalOperation = "Operation" + MetricAttributeEnvironment = "Environment" + MetricAttributeRemoteService = "RemoteService" + MetricAttributeRemoteEnvironment = "RemoteEnvironment" + MetricAttributeRemoteOperation = "RemoteOperation" + MetricAttributeRemoteResourceIdentifier = "RemoteResourceIdentifier" + MetricAttributeRemoteResourceType = "RemoteResourceType" +) + +const ( + AttributeEKSClusterName = "EKS.Cluster" + AttributeK8SClusterName = "K8s.Cluster" + AttributeK8SNamespace = "K8s.Namespace" + AttributeEC2AutoScalingGroup = "EC2.AutoScalingGroup" + AttributeEC2InstanceId = "EC2.InstanceId" + AttributeHost = "Host" + AttributePlatformType = "PlatformType" + AttributeTelemetrySDK = "Telemetry.SDK" + AttributeTelemetryAgent = "Telemetry.Agent" + AttributeTelemetrySource = "Telemetry.Source" +) + +const ( + AttributeTmpReserved = "aws.tmp.reserved" +) + +var IndexableMetricAttributes = []string{ + MetricAttributeLocalService, + MetricAttributeLocalOperation, + MetricAttributeEnvironment, + MetricAttributeRemoteService, + MetricAttributeRemoteEnvironment, + MetricAttributeRemoteOperation, + MetricAttributeRemoteResourceIdentifier, + MetricAttributeRemoteResourceType, +} diff --git a/plugins/processors/awsappsignals/config/config.go b/plugins/processors/awsapplicationsignals/config/config.go similarity index 95% rename from plugins/processors/awsappsignals/config/config.go rename to plugins/processors/awsapplicationsignals/config/config.go index f8ba4e16ed..19d5faeddf 100644 --- a/plugins/processors/awsappsignals/config/config.go +++ b/plugins/processors/awsapplicationsignals/config/config.go @@ -8,7 +8,7 @@ import ( "errors" "time" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) type Config struct { @@ -63,6 +63,8 @@ func (cfg *Config) Validate() error { return errors.New("name must not be empty for k8s resolver") } case PlatformEC2, PlatformGeneric: + case PlatformECS: + return errors.New("ecs resolver is not supported") default: return errors.New("unknown resolver") } diff --git a/plugins/processors/awsappsignals/config/config_test.go b/plugins/processors/awsapplicationsignals/config/config_test.go similarity index 100% rename from plugins/processors/awsappsignals/config/config_test.go rename to plugins/processors/awsapplicationsignals/config/config_test.go diff --git a/plugins/processors/awsappsignals/config/resolvers.go b/plugins/processors/awsapplicationsignals/config/resolvers.go similarity index 95% rename from plugins/processors/awsappsignals/config/resolvers.go rename to plugins/processors/awsapplicationsignals/config/resolvers.go index 59fe3f69c2..2ad6e25bf4 100644 --- a/plugins/processors/awsappsignals/config/resolvers.go +++ b/plugins/processors/awsapplicationsignals/config/resolvers.go @@ -12,6 +12,8 @@ const ( PlatformK8s = "k8s" // PlatformEC2 Amazon EC2 platform PlatformEC2 = "ec2" + // PlatformECS Amazon ECS + PlatformECS = "ecs" ) type Resolver struct { diff --git a/plugins/processors/awsappsignals/config/resolvers_test.go b/plugins/processors/awsapplicationsignals/config/resolvers_test.go similarity index 100% rename from plugins/processors/awsappsignals/config/resolvers_test.go rename to plugins/processors/awsapplicationsignals/config/resolvers_test.go diff --git a/plugins/processors/awsappsignals/factory.go b/plugins/processors/awsapplicationsignals/factory.go similarity index 89% rename from plugins/processors/awsappsignals/factory.go rename to plugins/processors/awsapplicationsignals/factory.go index d652c04a6e..18b5cb5a54 100644 --- a/plugins/processors/awsappsignals/factory.go +++ b/plugins/processors/awsapplicationsignals/factory.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "context" @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/collector/processor/processorhelper" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" ) const ( @@ -89,12 +89,12 @@ func createMetricsProcessor( func createProcessor( params processor.CreateSettings, cfg component.Config, -) (*awsappsignalsprocessor, error) { +) (*awsapplicationsignalsprocessor, error) { pCfg, ok := cfg.(*appsignalsconfig.Config) if !ok { - return nil, errors.New("could not initialize awsappsignalsprocessor") + return nil, errors.New("could not initialize awsapplicationsignalsprocessor") } - ap := &awsappsignalsprocessor{logger: params.Logger, config: pCfg} + ap := &awsapplicationsignalsprocessor{logger: params.Logger, config: pCfg} return ap, nil } diff --git a/plugins/processors/awsappsignals/factory_test.go b/plugins/processors/awsapplicationsignals/factory_test.go similarity index 97% rename from plugins/processors/awsappsignals/factory_test.go rename to plugins/processors/awsapplicationsignals/factory_test.go index 25118acb35..29aa9472d3 100644 --- a/plugins/processors/awsappsignals/factory_test.go +++ b/plugins/processors/awsapplicationsignals/factory_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "path/filepath" @@ -12,8 +12,8 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) var expectedRules = []rules.Rule{ diff --git a/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go b/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go new file mode 100644 index 0000000000..75e9a2fca1 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/attributes/attributes.go @@ -0,0 +1,24 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package attributes + +const ( + // aws attributes + AWSSpanKind = "aws.span.kind" + AWSLocalService = "aws.local.service" + AWSLocalEnvironment = "aws.local.environment" + AWSLocalOperation = "aws.local.operation" + AWSRemoteService = "aws.remote.service" + AWSRemoteOperation = "aws.remote.operation" + AWSRemoteEnvironment = "aws.remote.environment" + AWSRemoteTarget = "aws.remote.target" + AWSRemoteResourceIdentifier = "aws.remote.resource.identifier" + AWSRemoteResourceType = "aws.remote.resource.type" + AWSHostedInEnvironment = "aws.hostedin.environment" + + // resource detection processor attributes + ResourceDetectionHostId = "host.id" + ResourceDetectionHostName = "host.name" + ResourceDetectionASG = "ec2.tag.aws:autoscaling:groupName" +) diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch.go similarity index 100% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch.go diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch_test.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch_test.go similarity index 100% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/count_min_sketch_test.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/count_min_sketch_test.go diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter.go similarity index 95% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter.go index 4fad107807..3b2efda288 100644 --- a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter.go +++ b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter.go @@ -7,15 +7,14 @@ import ( "context" "fmt" "sort" - "strings" "sync" "time" "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" ) const ( @@ -29,17 +28,16 @@ const ( ) var awsDeclaredMetricAttributes = []string{ - common.HostedInAttributeClusterName, - common.HostedInAttributeK8SNamespace, - common.HostedInAttributeEnvironment, + common.AttributeEKSClusterName, + common.AttributeK8SNamespace, + common.MetricAttributeEnvironment, common.MetricAttributeLocalService, common.MetricAttributeLocalOperation, common.MetricAttributeRemoteService, common.MetricAttributeRemoteOperation, - common.MetricAttributeRemoteTarget, - common.MetricAttributeRemoteNamespace, - common.HostedInAttributeK8SClusterName, - common.HostedInAttributeEC2Environment, + common.MetricAttributeRemoteResourceIdentifier, + common.MetricAttributeRemoteEnvironment, + common.AttributeK8SClusterName, } type Limiter interface { @@ -338,7 +336,7 @@ func (s *service) admitMetricData(metric *MetricData) bool { func (s *service) rollupMetricData(attributes pcommon.Map) { for _, indexAttr := range awsDeclaredMetricAttributes { - if strings.HasPrefix(indexAttr, "HostedIn.") || (indexAttr == common.MetricAttributeLocalService) || (indexAttr == common.MetricAttributeRemoteService) { + if (indexAttr == common.MetricAttributeEnvironment) || (indexAttr == common.MetricAttributeLocalService) || (indexAttr == common.MetricAttributeRemoteService) { continue } if indexAttr == common.MetricAttributeLocalOperation { diff --git a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go similarity index 87% rename from plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go rename to plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go index ca502cc005..e7b14e4978 100644 --- a/plugins/processors/awsappsignals/internal/cardinalitycontrol/metrics_limiter_test.go +++ b/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol/metrics_limiter_test.go @@ -8,7 +8,6 @@ import ( "fmt" "math/rand" "strconv" - "strings" "testing" "time" @@ -16,19 +15,19 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" - awsappsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + awsapplicationsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" ) var emptyResourceAttributes = pcommon.NewMap() var logger, _ = zap.NewDevelopment() func TestAdmitAndRollup(t *testing.T) { - config := &awsappsignalsconfig.LimiterConfig{ + config := &awsapplicationsignalsconfig.LimiterConfig{ Threshold: 2, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -42,7 +41,9 @@ func TestAdmitAndRollup(t *testing.T) { admittedAttributes[uniqKey.AsString()] = attr } else { for _, indexedAttrKey := range awsDeclaredMetricAttributes { - if strings.HasPrefix(indexedAttrKey, "HostedIn.") || indexedAttrKey == "Service" || indexedAttrKey == "RemoteService" { + if indexedAttrKey == common.MetricAttributeEnvironment || + indexedAttrKey == common.MetricAttributeLocalService || + indexedAttrKey == common.MetricAttributeRemoteService { continue } attrValue, _ := attr.Get(indexedAttrKey) @@ -60,11 +61,11 @@ func TestAdmitAndRollup(t *testing.T) { } func TestAdmitByTopK(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 100, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -90,11 +91,11 @@ func TestAdmitByTopK(t *testing.T) { } func TestAdmitLowCardinalityAttributes(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -110,11 +111,11 @@ func TestAdmitLowCardinalityAttributes(t *testing.T) { } func TestAdmitReservedMetrics(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: false, - RotationInterval: awsappsignalsconfig.DefaultRotationInterval, + RotationInterval: awsapplicationsignalsconfig.DefaultRotationInterval, } config.Validate() @@ -141,7 +142,7 @@ func TestAdmitReservedMetrics(t *testing.T) { func TestClearStaleService(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: false, @@ -167,7 +168,7 @@ func TestClearStaleService(t *testing.T) { } func TestInheritanceAfterRotation(t *testing.T) { - config := awsappsignalsconfig.LimiterConfig{ + config := awsapplicationsignalsconfig.LimiterConfig{ Threshold: 10, Disabled: false, LogDroppedMetrics: true, diff --git a/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go new file mode 100644 index 0000000000..b0966bca2c --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer.go @@ -0,0 +1,234 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package normalizer + +import ( + "fmt" + "strings" + + "go.opentelemetry.io/collector/pdata/pcommon" + deprecatedsemconv "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" + "go.uber.org/zap" + + "github.com/aws/amazon-cloudwatch-agent/internal/version" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" +) + +const ( + // Length limits from Application Signals SLOs + maxEnvironmentLength = 259 + maxServiceNameLength = 255 + + // Length limits from CloudWatch Metrics + defaultMetricAttributeLength = 1024 +) + +type attributesNormalizer struct { + logger *zap.Logger +} + +var attributesRenamingForMetric = map[string]string{ + attr.AWSLocalService: common.MetricAttributeLocalService, + attr.AWSLocalOperation: common.MetricAttributeLocalOperation, + attr.AWSLocalEnvironment: common.MetricAttributeEnvironment, + attr.AWSRemoteService: common.MetricAttributeRemoteService, + attr.AWSRemoteOperation: common.MetricAttributeRemoteOperation, + attr.AWSRemoteEnvironment: common.MetricAttributeRemoteEnvironment, + attr.AWSRemoteTarget: common.MetricAttributeRemoteResourceIdentifier, + attr.AWSRemoteResourceIdentifier: common.MetricAttributeRemoteResourceIdentifier, + attr.AWSRemoteResourceType: common.MetricAttributeRemoteResourceType, +} + +var resourceAttributesRenamingForTrace = map[string]string{ + // these kubernetes resource attributes are set by the openTelemetry operator + // see the code references from upstream: + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 + semconv.AttributeK8SDeploymentName: "K8s.Workload", + semconv.AttributeK8SStatefulSetName: "K8s.Workload", + semconv.AttributeK8SDaemonSetName: "K8s.Workload", + semconv.AttributeK8SJobName: "K8s.Workload", + semconv.AttributeK8SCronJobName: "K8s.Workload", + semconv.AttributeK8SPodName: "K8s.Pod", +} + +var attributesRenamingForTrace = map[string]string{ + attr.AWSRemoteTarget: attr.AWSRemoteResourceIdentifier, +} + +var copyMapForMetric = map[string]string{ + // these kubernetes resource attributes are set by the openTelemtry operator + // see the code referecnes from upstream: + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 + semconv.AttributeK8SDeploymentName: "K8s.Workload", + semconv.AttributeK8SStatefulSetName: "K8s.Workload", + semconv.AttributeK8SDaemonSetName: "K8s.Workload", + semconv.AttributeK8SJobName: "K8s.Workload", + semconv.AttributeK8SCronJobName: "K8s.Workload", + semconv.AttributeK8SPodName: "K8s.Pod", +} + +const ( + instrumentationModeAuto = "Auto" + instrumentationModeManual = "Manual" +) + +func NewAttributesNormalizer(logger *zap.Logger) *attributesNormalizer { + return &attributesNormalizer{ + logger: logger, + } +} + +func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error { + n.copyResourceAttributesToAttributes(attributes, resourceAttributes, isTrace) + truncateAttributesByLength(attributes) + n.renameAttributes(attributes, resourceAttributes, isTrace) + n.normalizeTelemetryAttributes(attributes, resourceAttributes, isTrace) + return nil +} + +func (n *attributesNormalizer) renameAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { + if isTrace { + rename(resourceAttributes, resourceAttributesRenamingForTrace) + rename(attributes, attributesRenamingForTrace) + } else { + rename(attributes, attributesRenamingForMetric) + } +} + +func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { + if isTrace { + return + } + for k, v := range copyMapForMetric { + if resourceAttrValue, ok := resourceAttributes.Get(k); ok { + // print some debug info when an attribute value is overwritten + if originalAttrValue, ok := attributes.Get(k); ok { + n.logger.Debug("attribute value is overwritten", zap.String("attribute", k), zap.String("original", originalAttrValue.AsString()), zap.String("new", resourceAttrValue.AsString())) + } + attributes.PutStr(v, resourceAttrValue.AsString()) + if k == semconv.AttributeK8SPodName { + // only copy "host.id" from resource attributes to "K8s.Node" in attributesif the pod name is set + if host, ok := resourceAttributes.Get("host.id"); ok { + attributes.PutStr("K8s.Node", host.AsString()) + } + } + } + } +} + +func (n *attributesNormalizer) normalizeTelemetryAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { + if isTrace { + return + } + + var ( + sdkName string + sdkVersion string + sdkLang string + ) + var ( + sdkAutoName string + sdkAutoVersion string + ) + sdkName, sdkVersion, sdkLang = "-", "-", "-" + mode := instrumentationModeManual + + resourceAttributes.Range(func(k string, v pcommon.Value) bool { + switch k { + case semconv.AttributeTelemetrySDKName: + sdkName = removeWhitespaces(v.Str()) + case semconv.AttributeTelemetrySDKLanguage: + sdkLang = removeWhitespaces(v.Str()) + case semconv.AttributeTelemetrySDKVersion: + sdkVersion = removeWhitespaces(v.Str()) + } + switch k { + case semconv.AttributeTelemetryDistroName: + sdkAutoName = removeWhitespaces(v.Str()) + case deprecatedsemconv.AttributeTelemetryAutoVersion, semconv.AttributeTelemetryDistroVersion: + sdkAutoVersion = removeWhitespaces(v.Str()) + } + return true + }) + if sdkAutoName != "" { + sdkName = sdkAutoName + mode = instrumentationModeAuto + } + if sdkAutoVersion != "" { + sdkVersion = sdkAutoVersion + mode = instrumentationModeAuto + } + attributes.PutStr(common.AttributeTelemetrySDK, fmt.Sprintf("%s,%s,%s,%s", sdkName, sdkVersion, sdkLang, mode)) + attributes.PutStr(common.AttributeTelemetryAgent, fmt.Sprintf("CWAgent/%s", version.Number())) + + var telemetrySource string + if val, ok := attributes.Get(attr.AWSSpanKind); ok { + switch val.Str() { + case "CLIENT": + telemetrySource = "ClientSpan" + case "SERVER": + telemetrySource = "ServerSpan" + case "PRODUCER": + telemetrySource = "ProducerSpan" + case "CONSUMER": + telemetrySource = "ConsumerSpan" + case "LOCAL_ROOT": + telemetrySource = "LocalRootSpan" + } + attributes.PutStr(common.AttributeTelemetrySource, telemetrySource) + attributes.Remove(attr.AWSSpanKind) + } +} + +func rename(attrs pcommon.Map, renameMap map[string]string) { + for original, replacement := range renameMap { + if value, ok := attrs.Get(original); ok { + attrs.PutStr(replacement, value.AsString()) + attrs.Remove(original) + if original == semconv.AttributeK8SPodName { + // only rename host.id if the pod name is set + if host, ok := attrs.Get("host.id"); ok { + attrs.PutStr("K8s.Node", host.AsString()) + } + } + } + } +} + +func truncateAttributesByLength(attributes pcommon.Map) { + // It's assumed that all attributes are initially inserted as trace attribute, and attributesRenamingForMetric + // contains all attributes that will be used for CloudWatch metric dimension. Therefore, we iterate the keys + // for enforcing the limits on length. + for attrKey := range attributesRenamingForMetric { + switch attrKey { + case attr.AWSLocalEnvironment, attr.AWSRemoteEnvironment: + if val, ok := attributes.Get(attrKey); ok { + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), maxEnvironmentLength)) + } + case attr.AWSLocalService, attr.AWSRemoteService: + if val, ok := attributes.Get(attrKey); ok { + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), maxServiceNameLength)) + } + default: + if val, ok := attributes.Get(attrKey); ok { + attributes.PutStr(attrKey, truncateStringByLength(val.Str(), defaultMetricAttributeLength)) + } + } + } +} + +func truncateStringByLength(val string, length int) string { + if len(val) > length { + return val[:length] + } + return val +} + +func removeWhitespaces(val string) string { + return strings.ReplaceAll(val, " ", "") +} diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go similarity index 68% rename from plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go rename to plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go index ac36c77803..812b5a4142 100644 --- a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go +++ b/plugins/processors/awsapplicationsignals/internal/normalizer/attributesnormalizer_test.go @@ -8,8 +8,11 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" - conventions "go.opentelemetry.io/collector/semconv/v1.6.1" + deprecatedsemconv "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" + + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) func TestRenameAttributes_for_metric(t *testing.T) { @@ -19,7 +22,7 @@ func TestRenameAttributes_for_metric(t *testing.T) { // test for metric // Create a pcommon.Map with some attributes attributes := pcommon.NewMap() - for originalKey, replacementKey := range renameMapForMetric { + for originalKey, replacementKey := range attributesRenamingForMetric { attributes.PutStr(originalKey, replacementKey+"-value") } @@ -28,14 +31,14 @@ func TestRenameAttributes_for_metric(t *testing.T) { normalizer.renameAttributes(attributes, resourceAttributes, false) // Check that the original key has been removed - for originalKey := range renameMapForMetric { + for originalKey := range attributesRenamingForMetric { if _, ok := attributes.Get(originalKey); ok { t.Errorf("originalKey was not removed") } } // Check that the new key has the correct value - for _, replacementKey := range renameMapForMetric { + for _, replacementKey := range attributesRenamingForMetric { if value, ok := attributes.Get(replacementKey); !ok || value.AsString() != replacementKey+"-value" { t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), replacementKey+"-value") } @@ -49,7 +52,7 @@ func TestRenameAttributes_for_trace(t *testing.T) { // test for trace // Create a pcommon.Map with some attributes resourceAttributes := pcommon.NewMap() - for originalKey, replacementKey := range renameMapForTrace { + for originalKey, replacementKey := range resourceAttributesRenamingForTrace { resourceAttributes.PutStr(originalKey, replacementKey+"-value") } resourceAttributes.PutStr("host.id", "i-01ef7d37f42caa168") @@ -59,14 +62,14 @@ func TestRenameAttributes_for_trace(t *testing.T) { normalizer.renameAttributes(attributes, resourceAttributes, true) // Check that the original key has been removed - for originalKey := range renameMapForTrace { + for originalKey := range resourceAttributesRenamingForTrace { if _, ok := resourceAttributes.Get(originalKey); ok { t.Errorf("originalKey was not removed") } } // Check that the new key has the correct value - for _, replacementKey := range renameMapForTrace { + for _, replacementKey := range resourceAttributesRenamingForTrace { if value, ok := resourceAttributes.Get(replacementKey); !ok || value.AsString() != replacementKey+"-value" { t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), replacementKey+"-value") } @@ -106,18 +109,44 @@ func TestCopyResourceAttributesToAttributes(t *testing.T) { } } +func TestTruncateAttributes(t *testing.T) { + attributes := pcommon.NewMap() + + longValue := make([]byte, 300) + for i := 0; i < 300; i++ { + longValue[i] = 'a' + } + longStringValue := string(longValue) + for key, _ := range attributesRenamingForMetric { + attributes.PutStr(key, longStringValue) + } + + truncateAttributesByLength(attributes) + + val, _ := attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, len(val.Str()) == maxEnvironmentLength) + val, _ = attributes.Get(attr.AWSRemoteEnvironment) + assert.True(t, len(val.Str()) == maxEnvironmentLength) + val, _ = attributes.Get(attr.AWSLocalService) + assert.True(t, len(val.Str()) == maxServiceNameLength) + val, _ = attributes.Get(attr.AWSRemoteService) + assert.True(t, len(val.Str()) == maxServiceNameLength) + val, _ = attributes.Get(attr.AWSRemoteResourceIdentifier) + assert.True(t, len(val.Str()) == 300) +} + func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { logger, _ := zap.NewDevelopment() completeResourceAttributes := pcommon.NewMap() - completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKName, "opentelemetry") - completeResourceAttributes.PutStr(conventions.AttributeTelemetryAutoVersion, "0.0.1 auto") - completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKVersion, "0.0.1 test") - completeResourceAttributes.PutStr(conventions.AttributeTelemetrySDKLanguage, "go") + completeResourceAttributes.PutStr(semconv.AttributeTelemetrySDKName, "opentelemetry") + completeResourceAttributes.PutStr(deprecatedsemconv.AttributeTelemetryAutoVersion, "0.0.1 auto") + completeResourceAttributes.PutStr(semconv.AttributeTelemetrySDKVersion, "0.0.1 test") + completeResourceAttributes.PutStr(semconv.AttributeTelemetrySDKLanguage, "go") incompleteResourceAttributes := pcommon.NewMap() - incompleteResourceAttributes.PutStr(conventions.AttributeTelemetrySDKName, "opentelemetry") - incompleteResourceAttributes.PutStr(conventions.AttributeTelemetrySDKVersion, "0.0.1 test") + incompleteResourceAttributes.PutStr(semconv.AttributeTelemetrySDKName, "opentelemetry") + incompleteResourceAttributes.PutStr(semconv.AttributeTelemetrySDKVersion, "0.0.1 test") tests := []struct { name string @@ -160,9 +189,9 @@ func Test_attributesNormalizer_appendNewAttributes(t *testing.T) { n := &attributesNormalizer{ logger: logger, } - n.appendNewAttributes(tt.attributes, tt.resourceAttributes, tt.isTrace) + n.normalizeTelemetryAttributes(tt.attributes, tt.resourceAttributes, tt.isTrace) - if value, ok := tt.attributes.Get("SDK"); !ok { + if value, ok := tt.attributes.Get("Telemetry.SDK"); !ok { if !tt.isTrace { t.Errorf("attribute is not found.") } diff --git a/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go new file mode 100644 index 0000000000..5fcca636d4 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner.go @@ -0,0 +1,42 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package prune + +import ( + "errors" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" +) + +type MetricPruner struct { +} + +func (p *MetricPruner) ShouldBeDropped(attributes pcommon.Map) (bool, error) { + for _, attributeKey := range common.IndexableMetricAttributes { + if val, ok := attributes.Get(attributeKey); ok { + if !isAsciiPrintable(val.Str()) { + return true, errors.New("Metric attribute " + attributeKey + " must contain only ASCII characters.") + } + } + } + return false, nil +} + +func NewPruner() *MetricPruner { + return &MetricPruner{} +} + +func isAsciiPrintable(val string) bool { + nonWhitespaceFound := false + for _, c := range val { + if c < 32 || c > 126 { + return false + } else if !nonWhitespaceFound && c != 32 { + nonWhitespaceFound = true + } + } + return nonWhitespaceFound +} diff --git a/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go new file mode 100644 index 0000000000..4a14c5e9a7 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/prune/metric_pruner_test.go @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package prune + +import ( + "testing" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" +) + +func TestMetricPrunerWithIndexableAttribute(t *testing.T) { + tests := []struct { + name string + val string + want bool + }{ + { + "testShouldDropChineseChar", + "漢", + true, + }, { + "testShouldDropSymbolChar", + "€, £, µ", + true, + }, { + "testShouldDropAllBlackSpace", + " ", + true, + }, + { + "testShouldDropAllTab", + " ", + true, + }, { + "testShouldKeepEnglishWord", + "abcdefg-", + false, + }, + } + + p := &MetricPruner{} + for _, tt := range tests { + attributes := pcommon.NewMap() + attributes.PutStr(common.MetricAttributeLocalService, tt.val) + t.Run(tt.name, func(t *testing.T) { + got, _ := p.ShouldBeDropped(attributes) + if got != tt.want { + t.Errorf("ShouldBeDropped() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMetricPrunerWithNonIndexableAttribute(t *testing.T) { + tests := []struct { + name string + val string + want bool + }{ + { + "testShouldKeepChineseChar", + "漢", + false, + }, { + "testShouldKeepEnglishWord", + "abcdefg-", + false, + }, + } + + p := &MetricPruner{} + for _, tt := range tests { + attributes := pcommon.NewMap() + attributes.PutStr(common.AttributeEC2InstanceId, tt.val) + t.Run(tt.name, func(t *testing.T) { + got, _ := p.ShouldBeDropped(attributes) + if got != tt.want { + t.Errorf("ShouldBeDropped() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go new file mode 100644 index 0000000000..cab4106676 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver.go @@ -0,0 +1,163 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +import ( + "context" + "errors" + "fmt" + "strings" + + "go.opentelemetry.io/collector/pdata/pcommon" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" + "go.uber.org/zap" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" +) + +const ( + AttributeEnvironmentDefault = "default" + + AttributePlatformGeneric = "Generic" + AttributePlatformEC2 = "AWS::EC2" + AttributePlatformEKS = "AWS::EKS" + AttributePlatformK8S = "K8s" +) + +var GenericInheritedAttributes = map[string]string{ + semconv.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, + attr.ResourceDetectionHostName: common.AttributeHost, +} + +// DefaultInheritedAttributes is an allow-list that also renames attributes from the resource detection processor +var DefaultInheritedAttributes = map[string]string{ + semconv.AttributeDeploymentEnvironment: attr.AWSLocalEnvironment, + attr.ResourceDetectionASG: common.AttributeEC2AutoScalingGroup, + attr.ResourceDetectionHostId: common.AttributeEC2InstanceId, + attr.ResourceDetectionHostName: common.AttributeHost, +} + +type subResolver interface { + Process(attributes, resourceAttributes pcommon.Map) error + Stop(ctx context.Context) error +} + +type attributesResolver struct { + subResolvers []subResolver +} + +// create a new attributes resolver +func NewAttributesResolver(resolvers []appsignalsconfig.Resolver, logger *zap.Logger) *attributesResolver { + subResolvers := []subResolver{} + for _, resolver := range resolvers { + switch resolver.Platform { + case appsignalsconfig.PlatformEKS, appsignalsconfig.PlatformK8s: + subResolvers = append(subResolvers, getKubernetesResolver(resolver.Platform, resolver.Name, logger), newKubernetesResourceAttributesResolver(resolver.Platform, resolver.Name)) + case appsignalsconfig.PlatformEC2: + subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformEC2, DefaultInheritedAttributes)) + default: + if ecsutil.GetECSUtilSingleton().IsECS() { + subResolvers = append(subResolvers, newResourceAttributesResolver(appsignalsconfig.PlatformECS, AttributePlatformGeneric, DefaultInheritedAttributes)) + } else { + subResolvers = append(subResolvers, newResourceAttributesResolver(resolver.Platform, AttributePlatformGeneric, GenericInheritedAttributes)) + } + } + } + return &attributesResolver{ + subResolvers: subResolvers, + } +} + +// Process the attributes +func (r *attributesResolver) Process(attributes, resourceAttributes pcommon.Map, _ bool) error { + for _, subResolver := range r.subResolvers { + if err := subResolver.Process(attributes, resourceAttributes); err != nil { + return err + } + } + return nil +} + +func (r *attributesResolver) Stop(ctx context.Context) error { + var errs error + for _, subResolver := range r.subResolvers { + errs = errors.Join(errs, subResolver.Stop(ctx)) + } + return errs +} + +type resourceAttributesResolver struct { + defaultEnvPrefix string + platformType string + attributeMap map[string]string +} + +func newResourceAttributesResolver(defaultEnvPrefix, platformType string, attributeMap map[string]string) *resourceAttributesResolver { + return &resourceAttributesResolver{ + defaultEnvPrefix: defaultEnvPrefix, + platformType: platformType, + attributeMap: attributeMap, + } +} +func (h *resourceAttributesResolver) Process(attributes, resourceAttributes pcommon.Map) error { + for attrKey, mappingKey := range h.attributeMap { + if val, ok := resourceAttributes.Get(attrKey); ok { + attributes.PutStr(mappingKey, val.Str()) + } + } + attributes.PutStr(attr.AWSLocalEnvironment, getLocalEnvironment(attributes, resourceAttributes, h.defaultEnvPrefix)) + attributes.PutStr(common.AttributePlatformType, h.platformType) + return nil +} + +func getLocalEnvironment(attributes, resourceAttributes pcommon.Map, defaultEnvPrefix string) string { + if val, ok := attributes.Get(attr.AWSLocalEnvironment); ok { + return val.Str() + } + if val, found := resourceAttributes.Get(attr.AWSHostedInEnvironment); found { + return val.Str() + } + if defaultEnvPrefix == appsignalsconfig.PlatformECS { + if clusterName, _ := getECSClusterName(resourceAttributes); clusterName != "" { + return getDefaultEnvironment(defaultEnvPrefix, clusterName) + } + if clusterName := ecsutil.GetECSUtilSingleton().Cluster; clusterName != "" { + return getDefaultEnvironment(defaultEnvPrefix, clusterName) + } + } else if defaultEnvPrefix == appsignalsconfig.PlatformEC2 { + if asgAttr, found := resourceAttributes.Get(attr.ResourceDetectionASG); found { + return getDefaultEnvironment(defaultEnvPrefix, asgAttr.Str()) + } + } + return getDefaultEnvironment(defaultEnvPrefix, AttributeEnvironmentDefault) +} + +func getECSClusterName(resourceAttributes pcommon.Map) (string, bool) { + if clusterAttr, ok := resourceAttributes.Get(semconv.AttributeAWSECSClusterARN); ok { + parts := strings.Split(clusterAttr.Str(), "/") + clusterName := parts[len(parts)-1] + return clusterName, true + } else if taskAttr, ok := resourceAttributes.Get(semconv.AttributeAWSECSTaskARN); ok { + parts := strings.SplitAfterN(taskAttr.Str(), ":task/", 2) + if len(parts) == 2 { + taskParts := strings.Split(parts[1], "/") + // cluster name in ARN + if len(taskParts) == 2 { + return taskParts[0], true + } + } + } + return "", false +} + +func getDefaultEnvironment(platformCode, val string) string { + return fmt.Sprintf("%s:%s", platformCode, val) +} + +func (h *resourceAttributesResolver) Stop(ctx context.Context) error { + return nil +} diff --git a/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go new file mode 100644 index 0000000000..cf8fd05f57 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/internal/resolver/attributesresolver_test.go @@ -0,0 +1,247 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.opentelemetry.io/collector/pdata/pcommon" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" + "go.uber.org/zap" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" +) + +type MockSubResolver struct { + mock.Mock +} + +func (m *MockSubResolver) Process(attributes, resourceAttributes pcommon.Map) error { + args := m.Called(attributes, resourceAttributes) + return args.Error(0) +} + +func (m *MockSubResolver) Stop(ctx context.Context) error { + args := m.Called(ctx) + return args.Error(0) +} + +func TestResourceAttributesResolverWithNoConfiguredName(t *testing.T) { + tests := []struct { + name string + platformCode string + platformType string + resolver config.Resolver + }{ + { + "testOnGeneric", + config.PlatformGeneric, + AttributePlatformGeneric, + config.NewGenericResolver(""), + }, + { + "testOnEC2", + config.PlatformEC2, + AttributePlatformEC2, + config.NewEC2Resolver(""), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{tt.resolver}, logger) + resolver := attributesResolver.subResolvers[0] + + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + + resolver.Process(attributes, resourceAttributes) + + attribute, ok := attributes.Get(common.AttributePlatformType) + assert.True(t, ok) + assert.Equal(t, tt.platformType, attribute.Str()) + + attribute, ok = attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, ok) + assert.Equal(t, tt.platformCode+":default", attribute.Str()) + }) + } +} + +func TestResourceAttributesResolverWithECSClusterName(t *testing.T) { + resolver := resourceAttributesResolver{ + defaultEnvPrefix: "ecs", + platformType: "Generic", + attributeMap: DefaultInheritedAttributes, + } + + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/10838bed-421f-43ef-870a-f43feacbbb5b") + + resolver.Process(attributes, resourceAttributes) + + attribute, ok := attributes.Get(common.AttributePlatformType) + assert.True(t, ok) + assert.Equal(t, "Generic", attribute.Str()) + + attribute, ok = attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, ok) + assert.Equal(t, "ecs:my-cluster", attribute.Str()) +} + +func TestResourceAttributesResolverWithOnEC2WithASG(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{config.NewEC2Resolver("")}, logger) + resolver := attributesResolver.subResolvers[0] + + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr(attr.ResourceDetectionASG, "my-asg") + + resolver.Process(attributes, resourceAttributes) + platformAttr, ok := attributes.Get(common.AttributePlatformType) + assert.True(t, ok) + assert.Equal(t, "AWS::EC2", platformAttr.Str()) + envAttr, ok := attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, ok) + assert.Equal(t, "ec2:my-asg", envAttr.Str()) +} + +func TestResourceAttributesResolverWithHostname(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{config.NewGenericResolver("")}, logger) + resolver := attributesResolver.subResolvers[0] + + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr(attr.ResourceDetectionHostName, "hostname") + + resolver.Process(attributes, resourceAttributes) + envAttr, ok := attributes.Get(common.AttributeHost) + assert.True(t, ok) + assert.Equal(t, "hostname", envAttr.AsString()) +} + +func TestResourceAttributesResolverWithCustomEnvironment(t *testing.T) { + tests := []struct { + name string + platformCode string + resolver config.Resolver + }{ + { + "testOnGeneric", + config.PlatformGeneric, + config.NewGenericResolver(""), + }, + { + "testOnEC2", + config.PlatformEC2, + config.NewEC2Resolver(""), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger, _ := zap.NewDevelopment() + attributesResolver := NewAttributesResolver([]config.Resolver{tt.resolver}, logger) + resolver := attributesResolver.subResolvers[0] + + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + // insert default env + resourceAttributes.PutStr(attr.ResourceDetectionASG, "my-asg") + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/10838bed-421f-43ef-870a-f43feacbbb5b") + + // insert custom env + resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "env1") + resolver.Process(attributes, resourceAttributes) + envAttr, ok := attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, ok) + assert.Equal(t, "env1", envAttr.Str()) + + attributes = pcommon.NewMap() + resourceAttributes = pcommon.NewMap() + + resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "error") + resourceAttributes.PutStr(semconv.AttributeDeploymentEnvironment, "env2") + resolver.Process(attributes, resourceAttributes) + envAttr, ok = attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, ok) + assert.Equal(t, "env2", envAttr.Str()) + + attributes = pcommon.NewMap() + resourceAttributes = pcommon.NewMap() + + resourceAttributes.PutStr(semconv.AttributeDeploymentEnvironment, "env3") + resolver.Process(attributes, resourceAttributes) + envAttr, ok = attributes.Get(attr.AWSLocalEnvironment) + assert.True(t, ok) + assert.Equal(t, "env3", envAttr.Str()) + }) + } +} + +func TestAttributesResolver_Process(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + + mockSubResolver1 := new(MockSubResolver) + mockSubResolver1.On("Process", attributes, resourceAttributes).Return(nil) + + mockSubResolver2 := new(MockSubResolver) + mockSubResolver2.On("Process", attributes, resourceAttributes).Return(errors.New("error")) + + r := &attributesResolver{ + subResolvers: []subResolver{mockSubResolver1, mockSubResolver2}, + } + + err := r.Process(attributes, resourceAttributes, true) + assert.Error(t, err) + mockSubResolver1.AssertExpectations(t) + mockSubResolver2.AssertExpectations(t) +} + +func TestAttributesResolver_Stop(t *testing.T) { + ctx := context.Background() + + mockSubResolver1 := new(MockSubResolver) + mockSubResolver1.On("Stop", ctx).Return(nil) + + mockSubResolver2 := new(MockSubResolver) + mockSubResolver2.On("Stop", ctx).Return(errors.New("error")) + + r := &attributesResolver{ + subResolvers: []subResolver{mockSubResolver1, mockSubResolver2}, + } + + err := r.Stop(ctx) + assert.Error(t, err) + mockSubResolver1.AssertExpectations(t) + mockSubResolver2.AssertExpectations(t) +} + +func TestGetClusterName(t *testing.T) { + resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSClusterARN, "arn:aws:ecs:us-west-2:123456789123:cluster/my-cluster") + clusterName, ok := getECSClusterName(resourceAttributes) + assert.True(t, ok) + assert.Equal(t, "my-cluster", clusterName) + + resourceAttributes = pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/10838bed-421f-43ef-870a-f43feacbbb5b") + _, ok = getECSClusterName(resourceAttributes) + assert.False(t, ok) + + resourceAttributes = pcommon.NewMap() + resourceAttributes.PutStr(semconv.AttributeAWSECSTaskARN, "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/10838bed-421f-43ef-870a-f43feacbbb5b") + clusterName, ok = getECSClusterName(resourceAttributes) + assert.True(t, ok) + assert.Equal(t, "my-cluster", clusterName) +} diff --git a/plugins/processors/awsappsignals/internal/resolver/kubernetes.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go similarity index 91% rename from plugins/processors/awsappsignals/internal/resolver/kubernetes.go rename to plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go index a87d8fdc0a..fe5712976b 100644 --- a/plugins/processors/awsappsignals/internal/resolver/kubernetes.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes.go @@ -17,7 +17,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.17.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" @@ -25,8 +25,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" - "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) const ( @@ -48,13 +49,6 @@ const ( jitterKubernetesAPISeconds = 10 ) -var kubernetesHostedInAttributeMap = map[string]string{ - semconv.AttributeK8SNamespaceName: attr.HostedInK8SNamespace, - attr.ResourceDetectionHostId: attr.EC2InstanceId, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, - attr.ResourceDetectionASG: attr.EC2AutoScalingGroupName, -} - var ( // ReplicaSet name = Deployment name + "-" + up to 10 alphanumeric characters string, if the ReplicaSet was created through a deployment // The suffix string of the ReplicaSet name is an int32 number (0 to 4,294,967,295) that is cast to a string and then @@ -74,6 +68,8 @@ var ( type kubernetesResolver struct { logger *zap.Logger clientset kubernetes.Interface + clusterName string + platformCode string ipToPod *sync.Map podToWorkloadAndNamespace *sync.Map ipToServiceAndNamespace *sync.Map @@ -512,7 +508,7 @@ func (m *ServiceToWorkloadMapper) Start(stopCh chan struct{}) { }() } -func getKubernetesResolver(logger *zap.Logger) subResolver { +func getKubernetesResolver(platformCode, clusterName string, logger *zap.Logger) subResolver { once.Do(func() { config, err := clientcmd.BuildConfigFromFlags("", "") if err != nil { @@ -550,6 +546,8 @@ func getKubernetesResolver(logger *zap.Logger) subResolver { instance = &kubernetesResolver{ logger: logger, clientset: clientset, + clusterName: clusterName, + platformCode: platformCode, ipToServiceAndNamespace: serviceWatcher.ipToServiceAndNamespace, serviceAndNamespaceToSelectors: serviceWatcher.serviceAndNamespaceToSelectors, ipToPod: podWatcher.ipToPod, @@ -592,13 +590,14 @@ func (e *kubernetesResolver) GetWorkloadAndNamespaceByIP(ip string) (string, str } func (e *kubernetesResolver) Process(attributes, resourceAttributes pcommon.Map) error { + var namespace string if value, ok := attributes.Get(attr.AWSRemoteService); ok { valueStr := value.AsString() ipStr := "" if ip, _, ok := extractIPPort(valueStr); ok { - if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { + if workload, ns, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { attributes.PutStr(attr.AWSRemoteService, workload) - attributes.PutStr(attr.K8SRemoteNamespace, namespace) + namespace = ns } else { ipStr = ip } @@ -607,9 +606,9 @@ func (e *kubernetesResolver) Process(attributes, resourceAttributes pcommon.Map) } if ipStr != "" { - if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { + if workload, ns, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { attributes.PutStr(attr.AWSRemoteService, workload) - attributes.PutStr(attr.K8SRemoteNamespace, namespace) + namespace = ns } else { e.logger.Debug("failed to Process ip", zap.String("ip", ipStr), zap.Error(err)) attributes.PutStr(attr.AWSRemoteService, "UnknownRemoteService") @@ -617,6 +616,12 @@ func (e *kubernetesResolver) Process(attributes, resourceAttributes pcommon.Map) } } + if _, ok := attributes.Get(attr.AWSRemoteEnvironment); !ok { + if namespace != "" { + attributes.PutStr(attr.AWSRemoteEnvironment, fmt.Sprintf("%s:%s/%s", e.platformCode, e.clusterName, namespace)) + } + } + return nil } @@ -662,30 +667,47 @@ func getHostNetworkPorts(pod *corev1.Pod) []string { return ports } -type kubernetesHostedInAttributeResolver struct { +type kubernetesResourceAttributesResolver struct { + platformCode string clusterName string attributeMap map[string]string } -func newKubernetesHostedInAttributeResolver(clusterName string) *kubernetesHostedInAttributeResolver { - return &kubernetesHostedInAttributeResolver{ +func newKubernetesResourceAttributesResolver(platformCode, clusterName string) *kubernetesResourceAttributesResolver { + return &kubernetesResourceAttributesResolver{ + platformCode: platformCode, clusterName: clusterName, - attributeMap: kubernetesHostedInAttributeMap, + attributeMap: DefaultInheritedAttributes, } } -func (h *kubernetesHostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { +func (h *kubernetesResourceAttributesResolver) Process(attributes, resourceAttributes pcommon.Map) error { for attrKey, mappingKey := range h.attributeMap { if val, ok := resourceAttributes.Get(attrKey); ok { attributes.PutStr(mappingKey, val.AsString()) } } + if h.platformCode == config.PlatformEKS { + attributes.PutStr(common.AttributePlatformType, AttributePlatformEKS) + attributes.PutStr(common.AttributeEKSClusterName, h.clusterName) + } else { + attributes.PutStr(common.AttributePlatformType, AttributePlatformK8S) + attributes.PutStr(common.AttributeK8SClusterName, h.clusterName) + } + var namespace string + if nsAttr, ok := resourceAttributes.Get(semconv.AttributeK8SNamespaceName); ok { + namespace = nsAttr.Str() + } else { + namespace = "UnknownNamespace" + } - if isEks := eksdetector.IsEKS(); isEks.Value { - attributes.PutStr(attr.HostedInClusterNameEKS, h.clusterName) + if val, ok := attributes.Get(attr.AWSLocalEnvironment); !ok { + env := getDefaultEnvironment(h.platformCode, h.clusterName+"/"+namespace) + attributes.PutStr(attr.AWSLocalEnvironment, env) } else { - attributes.PutStr(attr.HostedInClusterNameK8s, h.clusterName) + attributes.PutStr(attr.AWSLocalEnvironment, val.Str()) } + attributes.PutStr(common.AttributeK8SNamespace, namespace) //The application log group in Container Insights is a fixed pattern: // "/aws/containerinsights/{Cluster_Name}/application" // See https://github.com/aws/amazon-cloudwatch-agent-operator/blob/fe144bb02d7b1930715aa3ea32e57a5ff13406aa/helm/templates/fluent-bit-configmap.yaml#L82 @@ -695,6 +717,6 @@ func (h *kubernetesHostedInAttributeResolver) Process(attributes, resourceAttrib return nil } -func (h *kubernetesHostedInAttributeResolver) Stop(ctx context.Context) error { +func (h *kubernetesResourceAttributesResolver) Stop(ctx context.Context) error { return nil } diff --git a/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go similarity index 82% rename from plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go rename to plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go index c21b8ee2ab..20cb17a5cb 100644 --- a/plugins/processors/awsappsignals/internal/resolver/kubernetes_test.go +++ b/plugins/processors/awsapplicationsignals/internal/resolver/kubernetes_test.go @@ -5,6 +5,7 @@ package resolver import ( "context" + "fmt" "strings" "sync" "testing" @@ -13,12 +14,14 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" - semconv "go.opentelemetry.io/collector/semconv/v1.17.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" ) @@ -697,6 +700,7 @@ func TestEksResolver(t *testing.T) { t.Run("Test GetWorkloadAndNamespaceByIP", func(t *testing.T) { resolver := &kubernetesResolver{ logger: logger, + clusterName: "test", ipToPod: &sync.Map{}, podToWorkloadAndNamespace: &sync.Map{}, ipToServiceAndNamespace: &sync.Map{}, @@ -770,6 +774,8 @@ func TestEksResolver(t *testing.T) { logger, _ := zap.NewProduction() resolver := &kubernetesResolver{ logger: logger, + clusterName: "test", + platformCode: config.PlatformEKS, ipToPod: &sync.Map{}, podToWorkloadAndNamespace: &sync.Map{}, ipToServiceAndNamespace: &sync.Map{}, @@ -785,7 +791,7 @@ func TestEksResolver(t *testing.T) { err := resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) assert.Equal(t, "test-deployment", getStrAttr(attributes, attr.AWSRemoteService, t)) - assert.Equal(t, "test-namespace", getStrAttr(attributes, attr.K8SRemoteNamespace, t)) + assert.Equal(t, "eks:test/test-namespace", getStrAttr(attributes, attr.AWSRemoteEnvironment, t)) // Test case 2: "aws.remote.service" contains only IP attributes = pcommon.NewMap() @@ -796,7 +802,7 @@ func TestEksResolver(t *testing.T) { err = resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) assert.Equal(t, "test-deployment-2", getStrAttr(attributes, attr.AWSRemoteService, t)) - assert.Equal(t, "test-namespace-2", getStrAttr(attributes, attr.K8SRemoteNamespace, t)) + assert.Equal(t, "eks:test/test-namespace-2", getStrAttr(attributes, attr.AWSRemoteEnvironment, t)) // Test case 3: "aws.remote.service" contains non-ip string attributes = pcommon.NewMap() @@ -816,7 +822,7 @@ func TestEksResolver(t *testing.T) { }) } -func TestHostedInEksResolver(t *testing.T) { +func TestK8sResourceAttributesResolverOnEKS(t *testing.T) { eksdetector.NewDetector = eksdetector.TestEKSDetector eksdetector.IsEKS = eksdetector.TestIsEKSCacheEKS // helper function to get string values from the attributes @@ -829,27 +835,71 @@ func TestHostedInEksResolver(t *testing.T) { } } - resolver := newKubernetesHostedInAttributeResolver("test-cluster") - - // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr("cloud.provider", "aws") - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("host.id", "instance-id") - resourceAttributes.PutStr("host.name", "hostname") - resourceAttributes.PutStr("ec2.tag.aws:autoscaling:groupName", "asg") - err := resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterNameEKS, t)) - assert.Equal(t, "instance-id", getStrAttr(attributes, attr.EC2InstanceId, t)) - assert.Equal(t, "hostname", getStrAttr(attributes, attr.ResourceDetectionHostName, t)) - assert.Equal(t, "asg", getStrAttr(attributes, attr.EC2AutoScalingGroupName, t)) - assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + resolver := newKubernetesResourceAttributesResolver(config.PlatformEKS, "test-cluster") + + resourceAttributesBase := map[string]string{ + "cloud.provider": "aws", + "k8s.namespace.name": "test-namespace-3", + "host.id": "instance-id", + "host.name": "hostname", + "ec2.tag.aws:autoscaling:groupName": "asg", + } + + tests := []struct { + name string + resourceAttributesOverwrite map[string]string + expectedAttributes map[string]string + }{ + { + "testDefault", + map[string]string{}, + + map[string]string{ + attr.AWSLocalEnvironment: "eks:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeEKSClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", + }, + }, + { + "testOverwrite", + map[string]string{ + semconv.AttributeDeploymentEnvironment: "custom-env", + }, + map[string]string{ + attr.AWSLocalEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeEKSClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + for key, val := range resourceAttributesBase { + resourceAttributes.PutStr(key, val) + } + for key, val := range tt.resourceAttributesOverwrite { + resourceAttributes.PutStr(key, val) + } + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + + for key, val := range tt.expectedAttributes { + assert.Equal(t, val, getStrAttr(attributes, key, t), fmt.Sprintf("expected %s for key %s", val, key)) + } + assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + }) + } } -func TestHostedInNativeK8sEC2Resolver(t *testing.T) { +func TestK8sResourceAttributesResolverOnK8S(t *testing.T) { eksdetector.NewDetector = eksdetector.TestK8sDetector eksdetector.IsEKS = eksdetector.TestIsEKSCacheK8s // helper function to get string values from the attributes @@ -862,27 +912,71 @@ func TestHostedInNativeK8sEC2Resolver(t *testing.T) { } } - resolver := newKubernetesHostedInAttributeResolver("test-cluster") - - // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr("cloud.provider", "aws") - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("host.id", "instance-id") - resourceAttributes.PutStr("host.name", "hostname") - resourceAttributes.PutStr("ec2.tag.aws:autoscaling:groupName", "asg") - err := resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterNameK8s, t)) - assert.Equal(t, "instance-id", getStrAttr(attributes, attr.EC2InstanceId, t)) - assert.Equal(t, "hostname", getStrAttr(attributes, attr.ResourceDetectionHostName, t)) - assert.Equal(t, "asg", getStrAttr(attributes, attr.EC2AutoScalingGroupName, t)) - assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + resolver := newKubernetesResourceAttributesResolver(config.PlatformK8s, "test-cluster") + + resourceAttributesBase := map[string]string{ + "cloud.provider": "aws", + "k8s.namespace.name": "test-namespace-3", + "host.id": "instance-id", + "host.name": "hostname", + "ec2.tag.aws:autoscaling:groupName": "asg", + } + + tests := []struct { + name string + resourceAttributesOverwrite map[string]string + expectedAttributes map[string]string + }{ + { + "testDefaultOnK8s", + map[string]string{}, + + map[string]string{ + attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", + }, + }, + { + "testOverwriteOnK8s", + map[string]string{ + semconv.AttributeDeploymentEnvironment: "custom-env", + }, + map[string]string{ + attr.AWSLocalEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeEC2InstanceId: "instance-id", + common.AttributeHost: "hostname", + common.AttributeEC2AutoScalingGroup: "asg", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + for key, val := range resourceAttributesBase { + resourceAttributes.PutStr(key, val) + } + for key, val := range tt.resourceAttributesOverwrite { + resourceAttributes.PutStr(key, val) + } + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + + for key, val := range tt.expectedAttributes { + assert.Equal(t, val, getStrAttr(attributes, key, t), fmt.Sprintf("expected %s for key %s", val, key)) + } + assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + }) + } } -func TestHostedInNativeK8sOnPremResolver(t *testing.T) { +func TestK8sResourceAttributesResolverOnK8SOnPrem(t *testing.T) { eksdetector.NewDetector = eksdetector.TestK8sDetector // helper function to get string values from the attributes getStrAttr := func(attributes pcommon.Map, key string, t *testing.T) string { @@ -894,27 +988,69 @@ func TestHostedInNativeK8sOnPremResolver(t *testing.T) { } } - resolver := newKubernetesHostedInAttributeResolver("test-cluster") - - // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr("cloud.provider", "aws") - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("host.name", "hostname") - err := resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterNameK8s, t)) - assert.Equal(t, "hostname", getStrAttr(attributes, attr.ResourceDetectionHostName, t)) - assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) - - // EC2 related fields that should not exist for on-prem - _, exists := attributes.Get(attr.EC2AutoScalingGroupName) - assert.False(t, exists) - - _, exists = attributes.Get(attr.EC2InstanceId) - assert.False(t, exists) + resolver := newKubernetesResourceAttributesResolver(config.PlatformK8s, "test-cluster") + + resourceAttributesBase := map[string]string{ + "cloud.provider": "aws", + "k8s.namespace.name": "test-namespace-3", + "host.name": "hostname", + } + + tests := []struct { + name string + resourceAttributesOverwrite map[string]string + expectedAttributes map[string]string + }{ + { + "testDefault", + map[string]string{}, + + map[string]string{ + attr.AWSLocalEnvironment: "k8s:test-cluster/test-namespace-3", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeHost: "hostname", + }, + }, + { + "testOverwrite", + map[string]string{ + semconv.AttributeDeploymentEnvironment: "custom-env", + }, + map[string]string{ + attr.AWSLocalEnvironment: "custom-env", + common.AttributeK8SNamespace: "test-namespace-3", + common.AttributeK8SClusterName: "test-cluster", + common.AttributeHost: "hostname", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + for key, val := range resourceAttributesBase { + resourceAttributes.PutStr(key, val) + } + for key, val := range tt.resourceAttributesOverwrite { + resourceAttributes.PutStr(key, val) + } + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + + for key, val := range tt.expectedAttributes { + assert.Equal(t, val, getStrAttr(attributes, key, t), fmt.Sprintf("expected %s for key %s", val, key)) + } + assert.Equal(t, "/aws/containerinsights/test-cluster/application", getStrAttr(resourceAttributes, semconv.AttributeAWSLogGroupNames, t)) + + // EC2 related fields that should not exist for on-prem + _, exists := attributes.Get(common.AttributeEC2AutoScalingGroup) + assert.False(t, exists) + + _, exists = attributes.Get(common.AttributeEC2InstanceId) + assert.False(t, exists) + }) + } } func TestExtractIPPort(t *testing.T) { diff --git a/plugins/processors/awsappsignals/processor.go b/plugins/processors/awsapplicationsignals/processor.go similarity index 85% rename from plugins/processors/awsappsignals/processor.go rename to plugins/processors/awsapplicationsignals/processor.go index 6d0fcc3108..8fc3e25bb7 100644 --- a/plugins/processors/awsappsignals/processor.go +++ b/plugins/processors/awsapplicationsignals/processor.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "context" @@ -14,17 +14,17 @@ import ( "golang.org/x/text/cases" "golang.org/x/text/language" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/cardinalitycontrol" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/normalizer" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/resolver" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/cardinalitycontrol" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/normalizer" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/prune" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/resolver" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) const ( - failedToProcessAttribute = "failed to process attributes" - failedToProcessAttributeWithCustomRule = "failed to process attributes with custom rule, will drop the metric" - failedToProcessAttributeWithLimiter = "failed to process attributes with limiter, keep the data" + failedToProcessAttribute = "failed to process attributes" + failedToProcessAttributeWithLimiter = "failed to process attributes with limiter, keep the data" ) var metricCaser = cases.Title(language.English) @@ -42,7 +42,7 @@ type stopper interface { Stop(context.Context) error } -type awsappsignalsprocessor struct { +type awsapplicationsignalsprocessor struct { logger *zap.Logger config *appsignalsconfig.Config replaceActions *rules.ReplaceActions @@ -53,7 +53,7 @@ type awsappsignalsprocessor struct { stoppers []stopper } -func (ap *awsappsignalsprocessor) StartMetrics(ctx context.Context, _ component.Host) error { +func (ap *awsapplicationsignalsprocessor) StartMetrics(ctx context.Context, _ component.Host) error { attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) ap.stoppers = []stopper{attributesResolver} attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) @@ -75,14 +75,15 @@ func (ap *awsappsignalsprocessor) StartMetrics(ctx context.Context, _ component. ap.replaceActions = rules.NewReplacer(ap.config.Rules, !limiterConfig.Disabled) + pruner := prune.NewPruner() keeper := rules.NewKeeper(ap.config.Rules, !limiterConfig.Disabled) dropper := rules.NewDropper(ap.config.Rules) - ap.allowlistMutators = []allowListMutator{keeper, dropper} + ap.allowlistMutators = []allowListMutator{pruner, keeper, dropper} return nil } -func (ap *awsappsignalsprocessor) StartTraces(_ context.Context, _ component.Host) error { +func (ap *awsapplicationsignalsprocessor) StartTraces(_ context.Context, _ component.Host) error { attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) customReplacer := rules.NewReplacer(ap.config.Rules, false) @@ -92,7 +93,7 @@ func (ap *awsappsignalsprocessor) StartTraces(_ context.Context, _ component.Hos return nil } -func (ap *awsappsignalsprocessor) Shutdown(ctx context.Context) error { +func (ap *awsapplicationsignalsprocessor) Shutdown(ctx context.Context) error { for _, stopper := range ap.stoppers { err := stopper.Stop(ctx) if err != nil { @@ -102,7 +103,7 @@ func (ap *awsappsignalsprocessor) Shutdown(ctx context.Context) error { return nil } -func (ap *awsappsignalsprocessor) processTraces(_ context.Context, td ptrace.Traces) (ptrace.Traces, error) { +func (ap *awsapplicationsignalsprocessor) processTraces(_ context.Context, td ptrace.Traces) (ptrace.Traces, error) { rss := td.ResourceSpans() for i := 0; i < rss.Len(); i++ { rs := rss.At(i) @@ -125,7 +126,7 @@ func (ap *awsappsignalsprocessor) processTraces(_ context.Context, td ptrace.Tra return td, nil } -func (ap *awsappsignalsprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { +func (ap *awsapplicationsignalsprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { rms := md.ResourceMetrics() for i := 0; i < rms.Len(); i++ { rs := rms.At(i) @@ -146,7 +147,7 @@ func (ap *awsappsignalsprocessor) processMetrics(ctx context.Context, md pmetric // Attributes are provided for each log and trace, but not at the metric level // Need to process attributes for every data point within a metric. -func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { +func (ap *awsapplicationsignalsprocessor) processMetricAttributes(_ context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { // This is a lot of repeated code, but since there is no single parent superclass // between metric data types, we can't use polymorphism. switch m.Type() { @@ -164,7 +165,7 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m p for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -199,7 +200,7 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m p for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -234,7 +235,7 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m p for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -269,7 +270,7 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m p for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true @@ -304,7 +305,7 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(_ context.Context, m p for _, mutator := range ap.allowlistMutators { shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { - ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } if shouldBeDropped { return true diff --git a/plugins/processors/awsappsignals/processor_test.go b/plugins/processors/awsapplicationsignals/processor_test.go similarity index 97% rename from plugins/processors/awsappsignals/processor_test.go rename to plugins/processors/awsapplicationsignals/processor_test.go index 397dae63a2..630272cca4 100644 --- a/plugins/processors/awsappsignals/processor_test.go +++ b/plugins/processors/awsapplicationsignals/processor_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( "context" @@ -12,8 +12,8 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" ) var testRules = []rules.Rule{ @@ -58,7 +58,7 @@ var testRules = []rules.Rule{ func TestProcessMetrics(t *testing.T) { logger, _ := zap.NewDevelopment() - ap := &awsappsignalsprocessor{ + ap := &awsapplicationsignalsprocessor{ logger: logger, config: &config.Config{ Resolvers: []config.Resolver{config.NewGenericResolver("")}, @@ -102,7 +102,7 @@ func TestProcessMetrics(t *testing.T) { func TestProcessMetricsLowercase(t *testing.T) { logger, _ := zap.NewDevelopment() - ap := &awsappsignalsprocessor{ + ap := &awsapplicationsignalsprocessor{ logger: logger, config: &config.Config{ Resolvers: []config.Resolver{config.NewGenericResolver("")}, @@ -129,7 +129,7 @@ func TestProcessMetricsLowercase(t *testing.T) { func TestProcessTraces(t *testing.T) { logger, _ := zap.NewDevelopment() - ap := &awsappsignalsprocessor{ + ap := &awsapplicationsignalsprocessor{ logger: logger, config: &config.Config{ Resolvers: []config.Resolver{config.NewGenericResolver("")}, diff --git a/plugins/processors/awsappsignals/rules/common.go b/plugins/processors/awsapplicationsignals/rules/common.go similarity index 70% rename from plugins/processors/awsappsignals/rules/common.go rename to plugins/processors/awsapplicationsignals/rules/common.go index 4ac6ca8c14..8e432f20d1 100644 --- a/plugins/processors/awsappsignals/rules/common.go +++ b/plugins/processors/awsapplicationsignals/rules/common.go @@ -8,6 +8,9 @@ import ( "github.com/gobwas/glob" "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" ) type AllowListAction string @@ -46,10 +49,14 @@ type ActionItem struct { } var traceKeyMap = map[string]string{ - "Service": "aws.local.service", - "Operation": "aws.local.operation", - "RemoteService": "aws.remote.service", - "RemoteOperation": "aws.remote.operation", + common.MetricAttributeLocalService: attributes.AWSLocalService, + common.MetricAttributeEnvironment: attributes.AWSLocalEnvironment, + common.MetricAttributeLocalOperation: attributes.AWSLocalOperation, + common.MetricAttributeRemoteService: attributes.AWSRemoteService, + common.MetricAttributeRemoteEnvironment: attributes.AWSRemoteEnvironment, + common.MetricAttributeRemoteOperation: attributes.AWSRemoteOperation, + common.MetricAttributeRemoteResourceIdentifier: attributes.AWSRemoteResourceIdentifier, + common.MetricAttributeRemoteResourceType: attributes.AWSRemoteResourceType, } func GetAllowListAction(action string) (AllowListAction, error) { @@ -64,21 +71,17 @@ func GetAllowListAction(action string) (AllowListAction, error) { return "", errors.New("invalid action in rule") } -func getExactKey(metricDimensionKey string, isTrace bool) string { - if !isTrace { - return metricDimensionKey - } - traceDimensionKey, ok := traceKeyMap[metricDimensionKey] - if !ok { - // return original key if there is no matches - return metricDimensionKey +func convertToManagedAttributeKey(attributeKey string, isTrace bool) string { + val, ok := traceKeyMap[attributeKey] + if ok && isTrace { + return val } - return traceDimensionKey + return attributeKey } func matchesSelectors(attributes pcommon.Map, selectorMatchers []SelectorMatcherItem, isTrace bool) bool { for _, item := range selectorMatchers { - exactKey := getExactKey(item.Key, isTrace) + exactKey := convertToManagedAttributeKey(item.Key, isTrace) value, ok := attributes.Get(exactKey) if !ok { return false diff --git a/plugins/processors/awsapplicationsignals/rules/common_test.go b/plugins/processors/awsapplicationsignals/rules/common_test.go new file mode 100644 index 0000000000..40dade4386 --- /dev/null +++ b/plugins/processors/awsapplicationsignals/rules/common_test.go @@ -0,0 +1,46 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package rules + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/internal/attributes" +) + +func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, + isTrace bool) pcommon.Map { + return generateAttributesWithEnv(service, operation, "", remoteService, remoteOperation, "", isTrace) +} + +func generateAttributesWithEnv(service string, operation string, environment string, + remoteService string, remoteOperation string, remoteEnvironment string, + isTrace bool) pcommon.Map { + attributes := pcommon.NewMap() + if isTrace { + attributes.PutStr(attr.AWSLocalService, service) + attributes.PutStr(attr.AWSLocalOperation, operation) + if environment != "" { + attributes.PutStr(attr.AWSLocalEnvironment, environment) + } + attributes.PutStr(attr.AWSRemoteService, remoteService) + attributes.PutStr(attr.AWSRemoteOperation, remoteOperation) + if remoteEnvironment != "" { + attributes.PutStr(attr.AWSRemoteEnvironment, remoteEnvironment) + } + } else { + attributes.PutStr(common.MetricAttributeLocalService, service) + attributes.PutStr(common.MetricAttributeLocalOperation, operation) + if environment != "" { + attributes.PutStr(common.MetricAttributeEnvironment, environment) + } + attributes.PutStr(common.MetricAttributeRemoteService, remoteService) + attributes.PutStr(common.MetricAttributeRemoteOperation, remoteOperation) + if remoteEnvironment != "" { + attributes.PutStr(common.MetricAttributeRemoteEnvironment, remoteEnvironment) + } + } + return attributes +} diff --git a/plugins/processors/awsappsignals/rules/dropper.go b/plugins/processors/awsapplicationsignals/rules/dropper.go similarity index 100% rename from plugins/processors/awsappsignals/rules/dropper.go rename to plugins/processors/awsapplicationsignals/rules/dropper.go diff --git a/plugins/processors/awsappsignals/rules/dropper_test.go b/plugins/processors/awsapplicationsignals/rules/dropper_test.go similarity index 100% rename from plugins/processors/awsappsignals/rules/dropper_test.go rename to plugins/processors/awsapplicationsignals/rules/dropper_test.go diff --git a/plugins/processors/awsappsignals/rules/keeper.go b/plugins/processors/awsapplicationsignals/rules/keeper.go similarity index 97% rename from plugins/processors/awsappsignals/rules/keeper.go rename to plugins/processors/awsapplicationsignals/rules/keeper.go index 50ed315fea..c4b65e999c 100644 --- a/plugins/processors/awsappsignals/rules/keeper.go +++ b/plugins/processors/awsapplicationsignals/rules/keeper.go @@ -6,7 +6,7 @@ package rules import ( "go.opentelemetry.io/collector/pdata/pcommon" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" ) type KeepActions struct { diff --git a/plugins/processors/awsappsignals/rules/keeper_test.go b/plugins/processors/awsapplicationsignals/rules/keeper_test.go similarity index 100% rename from plugins/processors/awsappsignals/rules/keeper_test.go rename to plugins/processors/awsapplicationsignals/rules/keeper_test.go diff --git a/plugins/processors/awsappsignals/rules/replacer.go b/plugins/processors/awsapplicationsignals/rules/replacer.go similarity index 80% rename from plugins/processors/awsappsignals/rules/replacer.go rename to plugins/processors/awsapplicationsignals/rules/replacer.go index de3b392c6b..fdae4ef3e1 100644 --- a/plugins/processors/awsappsignals/rules/replacer.go +++ b/plugins/processors/awsapplicationsignals/rules/replacer.go @@ -6,7 +6,7 @@ package rules import ( "go.opentelemetry.io/collector/pdata/pcommon" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/common" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/common" ) type ReplaceActions struct { @@ -36,18 +36,13 @@ func (r *ReplaceActions) Process(attributes, _ pcommon.Map, isTrace bool) error continue } for _, replacement := range element.Replacements { - targetDimensionKey := getExactKey(replacement.TargetDimension, isTrace) - // don't allow customer add new dimension key - _, isExist := attributes.Get(targetDimensionKey) - if !isExist { - continue - } + targetDimension := replacement.TargetDimension + + attr := convertToManagedAttributeKey(targetDimension, isTrace) // every replacement in one specific dimension only will be performed once - _, ok := finalRules[targetDimensionKey] - if ok { - continue + if _, visited := finalRules[attr]; !visited { + finalRules[attr] = replacement.Value } - finalRules[targetDimensionKey] = replacement.Value } } diff --git a/plugins/processors/awsappsignals/rules/replacer_test.go b/plugins/processors/awsapplicationsignals/rules/replacer_test.go similarity index 77% rename from plugins/processors/awsappsignals/rules/replacer_test.go rename to plugins/processors/awsapplicationsignals/rules/replacer_test.go index d86bc2ab96..d777194550 100644 --- a/plugins/processors/awsappsignals/rules/replacer_test.go +++ b/plugins/processors/awsapplicationsignals/rules/replacer_test.go @@ -119,6 +119,77 @@ func TestReplacerProcess(t *testing.T) { } } +func TestAddManagedDimensionKey(t *testing.T) { + config := []Rule{ + { + Selectors: []Selector{ + { + Dimension: "Service", + Match: "app", + }, + { + Dimension: "RemoteService", + Match: "remote-app", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteEnvironment", + Value: "test", + }, + }, + Action: "replace", + }, + } + + testReplacer := NewReplacer(config, false) + assert.Equal(t, 1, len(testReplacer.Actions)) + + testCases := []TestCaseForReplacer{ + { + name: "testAddMissingRemoteEnvironmentInMetric", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "", false), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", false), + isTrace: false, + }, + { + name: "testAddMissingRemoteEnvironmentInTrace", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "", true), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", true), + isTrace: true, + }, + { + name: "testReplaceRemoteEnvironmentInMetric", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "error", false), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", false), + isTrace: false, + }, + { + name: "testReplaceRemoteEnvironmentInTrace", + input: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "error", true), + output: generateAttributesWithEnv("app", "PUT /api/customer/owners/12345", "test", + "remote-app", "GET", "test", true), + isTrace: true, + }, + } + + testMapPlaceHolder := pcommon.NewMap() + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, testReplacer.Process(tt.input, testMapPlaceHolder, tt.isTrace)) + assert.Equal(t, tt.output, tt.input) + }) + } +} + func TestReplacerProcessWithPriority(t *testing.T) { config := []Rule{ diff --git a/plugins/processors/awsappsignals/testdata/config_eks.yaml b/plugins/processors/awsapplicationsignals/testdata/config_eks.yaml similarity index 100% rename from plugins/processors/awsappsignals/testdata/config_eks.yaml rename to plugins/processors/awsapplicationsignals/testdata/config_eks.yaml diff --git a/plugins/processors/awsappsignals/testdata/config_generic.yaml b/plugins/processors/awsapplicationsignals/testdata/config_generic.yaml similarity index 100% rename from plugins/processors/awsappsignals/testdata/config_generic.yaml rename to plugins/processors/awsapplicationsignals/testdata/config_generic.yaml diff --git a/plugins/processors/awsappsignals/common/types.go b/plugins/processors/awsappsignals/common/types.go deleted file mode 100644 index 993e83b633..0000000000 --- a/plugins/processors/awsappsignals/common/types.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package common - -const ( - AttributeRemoteService = "aws.remote.service" - AttributeHostedInEnvironment = "aws.hostedin.environment" -) - -const ( - MetricAttributeRemoteNamespace = "K8s.RemoteNamespace" - MetricAttributeLocalService = "Service" - MetricAttributeLocalOperation = "Operation" - MetricAttributeRemoteService = "RemoteService" - MetricAttributeRemoteOperation = "RemoteOperation" - MetricAttributeRemoteTarget = "RemoteTarget" -) -const ( - HostedInAttributeClusterName = "HostedIn.EKS.Cluster" - HostedInAttributeK8SNamespace = "HostedIn.K8s.Namespace" - HostedInAttributeEnvironment = "HostedIn.Environment" - HostedInAttributeK8SClusterName = "HostedIn.K8s.Cluster" - HostedInAttributeEC2Environment = "HostedIn.EC2.Environment" -) - -const ( - AttributeTmpReserved = "aws.tmp.reserved" -) diff --git a/plugins/processors/awsappsignals/internal/attributes/attributes.go b/plugins/processors/awsappsignals/internal/attributes/attributes.go deleted file mode 100644 index dc3a1a5194..0000000000 --- a/plugins/processors/awsappsignals/internal/attributes/attributes.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package attributes - -const ( - // aws attributes - AWSLocalService = "aws.local.service" - AWSLocalOperation = "aws.local.operation" - AWSRemoteService = "aws.remote.service" - AWSRemoteOperation = "aws.remote.operation" - AWSRemoteTarget = "aws.remote.target" - AWSHostedInEnvironment = "aws.hostedin.environment" - - // resource detection processor attributes - ResourceDetectionHostId = "host.id" - ResourceDetectionHostName = "host.name" - ResourceDetectionASG = "ec2.tag.aws:autoscaling:groupName" - - // kubernetes resource attributes - K8SDeploymentName = "k8s.deployment.name" - K8SStatefulSetName = "k8s.statefulset.name" - K8SDaemonSetName = "k8s.daemonset.name" - K8SJobName = "k8s.job.name" - K8SCronJobName = "k8s.cronjob.name" - K8SPodName = "k8s.pod.name" - K8SRemoteNamespace = "K8s.RemoteNamespace" - - // ec2 resource attributes - EC2AutoScalingGroupName = "EC2.AutoScalingGroupName" - EC2InstanceId = "EC2.InstanceId" - - // hosted in attribute names - HostedInClusterNameEKS = "HostedIn.EKS.Cluster" - HostedInClusterNameK8s = "HostedIn.K8s.Cluster" - HostedInK8SNamespace = "HostedIn.K8s.Namespace" - HostedInEC2Environment = "HostedIn.EC2.Environment" - HostedInEnvironment = "HostedIn.Environment" - - // sdk attributes - MetricAttributeSDKMetadata = "SDK" -) diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go deleted file mode 100644 index 22559f91a0..0000000000 --- a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package normalizer - -import ( - "fmt" - "strings" - - "go.opentelemetry.io/collector/pdata/pcommon" - conventions "go.opentelemetry.io/collector/semconv/v1.18.0" - "go.uber.org/zap" - - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -type attributesNormalizer struct { - logger *zap.Logger -} - -var renameMapForMetric = map[string]string{ - attr.AWSLocalService: "Service", - attr.AWSLocalOperation: "Operation", - attr.AWSRemoteService: "RemoteService", - attr.AWSRemoteOperation: "RemoteOperation", - attr.AWSRemoteTarget: "RemoteTarget", -} - -var renameMapForTrace = map[string]string{ - // these kubernetes resource attributes are set by the openTelemetry operator - // see the code references from upstream: - // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 - // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 - attr.K8SDeploymentName: "K8s.Workload", - attr.K8SStatefulSetName: "K8s.Workload", - attr.K8SDaemonSetName: "K8s.Workload", - attr.K8SJobName: "K8s.Workload", - attr.K8SCronJobName: "K8s.Workload", - attr.K8SPodName: "K8s.Pod", -} - -var copyMapForMetric = map[string]string{ - // these kubernetes resource attributes are set by the openTelemtry operator - // see the code referecnes from upstream: - // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 - // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 - attr.K8SDeploymentName: "K8s.Workload", - attr.K8SStatefulSetName: "K8s.Workload", - attr.K8SDaemonSetName: "K8s.Workload", - attr.K8SJobName: "K8s.Workload", - attr.K8SCronJobName: "K8s.Workload", - attr.K8SPodName: "K8s.Pod", -} - -const ( - instrumentationModeAuto = "Auto" - instrumentationModeManual = "Manual" -) - -func NewAttributesNormalizer(logger *zap.Logger) *attributesNormalizer { - return &attributesNormalizer{ - logger: logger, - } -} - -func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error { - n.copyResourceAttributesToAttributes(attributes, resourceAttributes, isTrace) - n.renameAttributes(attributes, resourceAttributes, isTrace) - n.appendNewAttributes(attributes, resourceAttributes, isTrace) - return nil -} - -func (n *attributesNormalizer) renameAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { - attrs := attributes - renameMap := renameMapForMetric - if isTrace { - attrs = resourceAttributes - renameMap = renameMapForTrace - } - - rename(attrs, renameMap) -} - -func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { - if isTrace { - return - } - for k, v := range copyMapForMetric { - if resourceAttrValue, ok := resourceAttributes.Get(k); ok { - // print some debug info when an attribute value is overwritten - if originalAttrValue, ok := attributes.Get(k); ok { - n.logger.Debug("attribute value is overwritten", zap.String("attribute", k), zap.String("original", originalAttrValue.AsString()), zap.String("new", resourceAttrValue.AsString())) - } - attributes.PutStr(v, resourceAttrValue.AsString()) - if k == attr.K8SPodName { - // only copy "host.id" from resource attributes to "K8s.Node" in attributesif the pod name is set - if host, ok := resourceAttributes.Get("host.id"); ok { - attributes.PutStr("K8s.Node", host.AsString()) - } - } - } - } -} - -func (n *attributesNormalizer) appendNewAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { - if isTrace { - return - } - - var ( - sdkName string - sdkVersion string - sdkAutoVersion string - sdkLang string - ) - sdkName, sdkVersion, sdkLang = "-", "-", "-" - mode := instrumentationModeManual - - // TODO read telemetry.auto.version from telemetry.distro.* from v1.22 - resourceAttributes.Range(func(k string, v pcommon.Value) bool { - switch k { - case conventions.AttributeTelemetrySDKName: - sdkName = strings.ReplaceAll(v.Str(), " ", "") - case conventions.AttributeTelemetrySDKLanguage: - sdkLang = strings.ReplaceAll(v.Str(), " ", "") - case conventions.AttributeTelemetrySDKVersion: - sdkVersion = strings.ReplaceAll(v.Str(), " ", "") - case conventions.AttributeTelemetryAutoVersion: - sdkAutoVersion = strings.ReplaceAll(v.Str(), " ", "") - } - return true - }) - if sdkAutoVersion != "" { - sdkVersion = sdkAutoVersion - mode = instrumentationModeAuto - } - attributes.PutStr(attr.MetricAttributeSDKMetadata, fmt.Sprintf("%s,%s,%s,%s", sdkName, sdkVersion, sdkLang, mode)) -} - -func rename(attrs pcommon.Map, renameMap map[string]string) { - for original, replacement := range renameMap { - if value, ok := attrs.Get(original); ok { - attrs.PutStr(replacement, value.AsString()) - attrs.Remove(original) - if original == attr.K8SPodName { - // only rename host.id if the pod name is set - if host, ok := attrs.Get("host.id"); ok { - attrs.PutStr("K8s.Node", host.AsString()) - attrs.Remove("host.id") - } - } - } - } -} diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go deleted file mode 100644 index 3531545ec1..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -import ( - "context" - "errors" - - "go.opentelemetry.io/collector/pdata/pcommon" - "go.uber.org/zap" - - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -const AttributePlatformGeneric = "Generic" - -var DefaultHostedInAttributes = map[string]string{ - attr.AWSHostedInEnvironment: attr.HostedInEnvironment, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, -} - -type subResolver interface { - Process(attributes, resourceAttributes pcommon.Map) error - Stop(ctx context.Context) error -} - -type attributesResolver struct { - subResolvers []subResolver -} - -// create a new attributes resolver -func NewAttributesResolver(resolvers []appsignalsconfig.Resolver, logger *zap.Logger) *attributesResolver { - //TODO: Logic for native k8s needs to be implemented - subResolvers := []subResolver{} - for _, resolver := range resolvers { - switch resolver.Platform { - case appsignalsconfig.PlatformEKS, appsignalsconfig.PlatformK8s: - subResolvers = append(subResolvers, getKubernetesResolver(logger), newKubernetesHostedInAttributeResolver(resolver.Name)) - case appsignalsconfig.PlatformEC2: - subResolvers = append(subResolvers, newEC2HostedInAttributeResolver(resolver.Name)) - default: - subResolvers = append(subResolvers, newHostedInAttributeResolver(resolver.Name, DefaultHostedInAttributes)) - } - } - return &attributesResolver{ - subResolvers: subResolvers, - } -} - -// Process the attributes -func (r *attributesResolver) Process(attributes, resourceAttributes pcommon.Map, _ bool) error { - for _, subResolver := range r.subResolvers { - if err := subResolver.Process(attributes, resourceAttributes); err != nil { - return err - } - } - return nil -} - -func (r *attributesResolver) Stop(ctx context.Context) error { - var errs error - for _, subResolver := range r.subResolvers { - if err := subResolver.Stop(ctx); err != nil { - errs = errors.Join(errs, err) - } - } - return errs -} - -type hostedInAttributeResolver struct { - name string - attributeMap map[string]string -} - -func newHostedInAttributeResolver(name string, attributeMap map[string]string) *hostedInAttributeResolver { - if name == "" { - name = AttributePlatformGeneric - } - return &hostedInAttributeResolver{ - name: name, - attributeMap: attributeMap, - } -} -func (h *hostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { - for attrKey, mappingKey := range h.attributeMap { - if val, ok := resourceAttributes.Get(attrKey); ok { - attributes.PutStr(mappingKey, val.AsString()) - } - } - - if _, ok := resourceAttributes.Get(attr.AWSHostedInEnvironment); !ok { - attributes.PutStr(attr.HostedInEnvironment, h.name) - } - - return nil -} - -func (h *hostedInAttributeResolver) Stop(ctx context.Context) error { - return nil -} diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go deleted file mode 100644 index 336ce57ad7..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "go.opentelemetry.io/collector/pdata/pcommon" - - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -type MockSubResolver struct { - mock.Mock -} - -func (m *MockSubResolver) Process(attributes, resourceAttributes pcommon.Map) error { - args := m.Called(attributes, resourceAttributes) - return args.Error(0) -} - -func (m *MockSubResolver) Stop(ctx context.Context) error { - args := m.Called(ctx) - return args.Error(0) -} - -func TestHostedInAttributeResolverWithNoConfiguredName(t *testing.T) { - resolver := newHostedInAttributeResolver("", DefaultHostedInAttributes) - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEnvironment) - assert.True(t, ok) - assert.Equal(t, "Generic", envAttr.AsString()) -} - -func TestHostedInAttributeResolverWithConfiguredName(t *testing.T) { - resolver := newHostedInAttributeResolver("test", DefaultHostedInAttributes) - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEnvironment) - assert.True(t, ok) - assert.Equal(t, "test", envAttr.AsString()) -} - -func TestHostedInAttributeResolverWithConflictedName(t *testing.T) { - resolver := newHostedInAttributeResolver("test", DefaultHostedInAttributes) - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, "self-defined") - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEnvironment) - assert.True(t, ok) - assert.Equal(t, "self-defined", envAttr.AsString()) -} - -func TestHostedInAttributeResolverWithHostname(t *testing.T) { - resolver := newHostedInAttributeResolver("test", DefaultHostedInAttributes) - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionHostName, "hostname") - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.ResourceDetectionHostName) - assert.True(t, ok) - assert.Equal(t, "hostname", envAttr.AsString()) -} - -func TestAttributesResolver_Process(t *testing.T) { - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - mockSubResolver1 := new(MockSubResolver) - mockSubResolver1.On("Process", attributes, resourceAttributes).Return(nil) - - mockSubResolver2 := new(MockSubResolver) - mockSubResolver2.On("Process", attributes, resourceAttributes).Return(errors.New("error")) - - r := &attributesResolver{ - subResolvers: []subResolver{mockSubResolver1, mockSubResolver2}, - } - - err := r.Process(attributes, resourceAttributes, true) - assert.Error(t, err) - mockSubResolver1.AssertExpectations(t) - mockSubResolver2.AssertExpectations(t) -} - -func TestAttributesResolver_Stop(t *testing.T) { - ctx := context.Background() - - mockSubResolver1 := new(MockSubResolver) - mockSubResolver1.On("Stop", ctx).Return(nil) - - mockSubResolver2 := new(MockSubResolver) - mockSubResolver2.On("Stop", ctx).Return(errors.New("error")) - - r := &attributesResolver{ - subResolvers: []subResolver{mockSubResolver1, mockSubResolver2}, - } - - err := r.Stop(ctx) - assert.Error(t, err) - mockSubResolver1.AssertExpectations(t) - mockSubResolver2.AssertExpectations(t) -} diff --git a/plugins/processors/awsappsignals/internal/resolver/ec2.go b/plugins/processors/awsappsignals/internal/resolver/ec2.go deleted file mode 100644 index f856f4b5d4..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/ec2.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -import ( - "context" - - "go.opentelemetry.io/collector/pdata/pcommon" - - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -const AttributePlatformEC2 = "EC2" - -// EC2HostedInAttributes is an allow-list that also renames attributes from the resource detection processor -var EC2HostedInAttributes = map[string]string{ - attr.ResourceDetectionHostId: attr.EC2InstanceId, - attr.ResourceDetectionHostName: attr.ResourceDetectionHostName, - attr.ResourceDetectionASG: attr.EC2AutoScalingGroupName, -} - -type ec2HostedInAttributeResolver struct { - name string - attributeMap map[string]string -} - -func newEC2HostedInAttributeResolver(name string) *ec2HostedInAttributeResolver { - if name == "" { - name = AttributePlatformEC2 - } - return &ec2HostedInAttributeResolver{ - name: name, - attributeMap: EC2HostedInAttributes, - } -} -func (h *ec2HostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { - for attrKey, mappingKey := range h.attributeMap { - if val, ok := resourceAttributes.Get(attrKey); ok { - attributes.PutStr(mappingKey, val.AsString()) - } - } - - // If aws.hostedin.environment is populated, override HostedIn.EC2.Environment value - // Otherwise, keep ASG name if it exists - if val, ok := resourceAttributes.Get(attr.AWSHostedInEnvironment); ok { - attributes.PutStr(attr.HostedInEC2Environment, val.AsString()) - } else if val, ok := resourceAttributes.Get(attr.ResourceDetectionASG); ok { - attributes.PutStr(attr.HostedInEC2Environment, val.AsString()) - } else { - attributes.PutStr(attr.HostedInEC2Environment, h.name) - } - - return nil -} - -func (h *ec2HostedInAttributeResolver) Stop(ctx context.Context) error { - return nil -} diff --git a/plugins/processors/awsappsignals/internal/resolver/ec2_test.go b/plugins/processors/awsappsignals/internal/resolver/ec2_test.go deleted file mode 100644 index d65c80c897..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/ec2_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "go.opentelemetry.io/collector/pdata/pcommon" - - attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" -) - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_NoASG_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, AttributePlatformEC2, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_ASGExists_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - asgName := "ASG" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionASG, asgName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, asgName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_NoASG_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, "test", envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_ASGExists_NoEnv(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - asgName := "ASG" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionASG, asgName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, asgName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_NoASG_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_NoASG_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithNoConfiguredName_ASGExists_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - asgName := "ASG" - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.EC2AutoScalingGroupName, asgName) - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithConfiguredName_ASGExists_EnvExists(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("test") - - asgName := "ASG" - envName := "my-env" - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.EC2AutoScalingGroupName, asgName) - resourceAttributes.PutStr(attr.AWSHostedInEnvironment, envName) - - resolver.Process(attributes, resourceAttributes) - envAttr, ok := attributes.Get(attr.HostedInEC2Environment) - assert.True(t, ok) - assert.Equal(t, envName, envAttr.AsString()) -} - -func TestEC2HostedInAttributeResolverWithResourceDetectionAttributes(t *testing.T) { - resolver := newEC2HostedInAttributeResolver("") - - attributes := pcommon.NewMap() - resourceAttributes := pcommon.NewMap() - resourceAttributes.PutStr(attr.ResourceDetectionHostId, "hostid") - resourceAttributes.PutStr(attr.ResourceDetectionHostName, "hostname") - resourceAttributes.PutStr(attr.ResourceDetectionASG, "asg") - - resolver.Process(attributes, resourceAttributes) - expectedInstanceId, ok := attributes.Get(attr.EC2InstanceId) - assert.True(t, ok) - assert.Equal(t, "hostid", expectedInstanceId.AsString()) - - expectedHostName, ok := attributes.Get(attr.ResourceDetectionHostName) - assert.True(t, ok) - assert.Equal(t, "hostname", expectedHostName.AsString()) - - expectedASG, ok := attributes.Get(attr.EC2AutoScalingGroupName) - assert.True(t, ok) - assert.Equal(t, "asg", expectedASG.AsString()) -} diff --git a/plugins/processors/awsappsignals/rules/common_test.go b/plugins/processors/awsappsignals/rules/common_test.go deleted file mode 100644 index e2ab0fe9a1..0000000000 --- a/plugins/processors/awsappsignals/rules/common_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package rules - -import "go.opentelemetry.io/collector/pdata/pcommon" - -func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, - isTrace bool) pcommon.Map { - attributes := pcommon.NewMap() - if isTrace { - attributes.PutStr("aws.local.service", service) - attributes.PutStr("aws.local.operation", operation) - attributes.PutStr("aws.remote.service", remoteService) - attributes.PutStr("aws.remote.operation", remoteOperation) - } else { - attributes.PutStr("Service", service) - attributes.PutStr("Operation", operation) - attributes.PutStr("RemoteService", remoteService) - attributes.PutStr("RemoteOperation", remoteOperation) - } - return attributes -} diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 2d575aed4e..d2c314d5a6 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -27,7 +27,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/plugins/outputs/cloudwatch" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/gpuattributes" ) @@ -47,7 +47,7 @@ func Factories() (otelcol.Factories, error) { } if factories.Processors, err = processor.MakeFactoryMap( - awsappsignals.NewFactory(), + awsapplicationsignals.NewFactory(), batchprocessor.NewFactory(), cumulativetodeltaprocessor.NewFactory(), ec2tagger.NewFactory(), diff --git a/translator/config/mode.go b/translator/config/mode.go index a5cb44f492..5f8c8f93ed 100644 --- a/translator/config/mode.go +++ b/translator/config/mode.go @@ -10,6 +10,10 @@ const ( ModeWithIRSA = "withIRSA" ) +const ( + ModeECS = "ECS" +) + const ( ModeEKS = "EKS" ModeK8sEC2 = "K8sEC2" diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 3e9061946c..fb91c0a6cf 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -9,109 +9,102 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/eks + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.EKS.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index 23342166e1..a49ebeeb37 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -9,109 +9,102 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/k8s + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.K8s.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.K8s.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces @@ -302,8 +294,8 @@ processors: log_dropped_metrics: true rotation_interval: 10m0s resolvers: - - name: TestCluster - platform: k8s + - name: TestCluster + platform: k8s batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index cf66cc4d5e..6a4d070a80 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -9,109 +9,102 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/eks + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.EKS.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 45822ef84e..a8c3e656d9 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -9,109 +9,102 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: false - log_group_name: /aws/appsignals/eks + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment - Operation + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace + - - Environment + - RemoteEnvironment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.EKS.Cluster - - HostedIn.K8s.Namespace + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -242,13 +235,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - HostedIn.K8s.Namespace - - K8s.RemoteNamespace - - aws.remote.target - - HostedIn.Environment - - HostedIn.EKS.Cluster + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: false max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index f065a5b739..dd5c0e8d4d 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -9,68 +9,72 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: true - log_group_name: /aws/appsignals/generic + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.Environment + - - Environment - Operation - Service - - - HostedIn.Environment + - - Environment - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -94,10 +98,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - aws.remote.target - - HostedIn.Environment + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: true max_retries: 2 middleware: agenthealth/traces diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml index 3901d23a85..11c0eefb2b 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_fallback_config.yaml @@ -9,68 +9,72 @@ exporters: enhanced_container_insights: false imds_retries: 1 local_mode: true - log_group_name: /aws/appsignals/generic + log_group_name: /aws/application-signals/data log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - HostedIn.Environment + - - Environment - Operation - Service - - - HostedIn.Environment + - - Environment - Service label_matchers: - label_names: - - aws.span.kind - regex: ^(SERVER|LOCAL_ROOT)$ + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - Operation - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteService - Service - - - HostedIn.Environment + - - Environment - RemoteOperation + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - HostedIn.Environment + - - Environment - RemoteOperation - RemoteService - Service - - - HostedIn.Environment + - - Environment + - RemoteResourceIdentifier + - RemoteResourceType - RemoteService - - RemoteTarget - Service - - - RemoteService - - RemoteTarget + - - RemoteResourceIdentifier + - RemoteResourceType + - RemoteService - - RemoteService label_matchers: - label_names: - - aws.span.kind - regex: ^(CLIENT|PRODUCER|CONSUMER)$ + - Telemetry.Source + regex: ^(ClientSpan|ProducerSpan|ConsumerSpan)$ separator: ; metric_name_selectors: - Latency - Fault - Error middleware: agenthealth/logs - namespace: AppSignals + namespace: ApplicationSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -94,10 +98,12 @@ exporters: indexed_attributes: - aws.local.service - aws.local.operation + - aws.local.environment - aws.remote.service - aws.remote.operation - - aws.remote.target - - HostedIn.Environment + - aws.remote.environment + - aws.remote.resource.identifier + - aws.remote.resource.type local_mode: true max_retries: 2 middleware: agenthealth/traces diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml deleted file mode 100644 index a128be9a5a..0000000000 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_ec2.yaml +++ /dev/null @@ -1,33 +0,0 @@ -log_group_name: "/aws/appsignals/ec2" -namespace: "AppSignals" -middleware: agenthealth/logs -dimension_rollup_option: "NoDimensionRollup" -metric_declarations: - - dimensions: - - [HostedIn.EC2.Environment, Service, Operation] - - [HostedIn.EC2.Environment, Service] - label_matchers: - - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' - metric_name_selectors: - - Latency - - Fault - - Error - - dimensions: - - [HostedIn.EC2.Environment, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EC2.Environment, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.EC2.Environment, Service, RemoteService] - - [HostedIn.EC2.Environment, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EC2.Environment, Service, RemoteService, RemoteOperation] - - [HostedIn.EC2.Environment, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] - - [RemoteService] - label_matchers: - - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' - metric_name_selectors: - - Latency - - Fault - - Error \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml index 7171c91dac..036c519717 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml @@ -1,37 +1,37 @@ -log_group_name: "/aws/appsignals/eks" -namespace: "AppSignals" +log_group_name: "/aws/application-signals/data" +namespace: "ApplicationSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service] + - [Environment, Service, Operation] + - [Environment, Service] label_matchers: - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' + - Telemetry.Source + regex: '^(ServerSpan|LocalRootSpan)$' metric_name_selectors: - Latency - Fault - Error - dimensions: - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation] - - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteEnvironment] + - [Environment, Service, RemoteService] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteResourceIdentifier, RemoteResourceType] + - [RemoteService, RemoteResourceIdentifier, RemoteResourceType] - [RemoteService] label_matchers: - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' + - Telemetry.Source + regex: '^(ClientSpan|ProducerSpan|ConsumerSpan)$' metric_name_selectors: - Latency - Fault diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml index 942faf6d81..735f6df7da 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml @@ -1,32 +1,32 @@ -log_group_name: "/aws/appsignals/generic" -namespace: "AppSignals" +log_group_name: "/aws/application-signals/data" +namespace: "ApplicationSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: - - [HostedIn.Environment, Service, Operation] - - [HostedIn.Environment, Service] + - [Environment, Service, Operation] + - [Environment, Service] label_matchers: - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' + - Telemetry.Source + regex: '^(ServerSpan|LocalRootSpan)$' metric_name_selectors: - Latency - Fault - Error - dimensions: - - [HostedIn.Environment, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.Environment, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.Environment, Service, RemoteService] - - [HostedIn.Environment, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.Environment, Service, RemoteService, RemoteOperation] - - [HostedIn.Environment, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService] + - [Environment, Service, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteResourceIdentifier, RemoteResourceType] + - [RemoteService, RemoteResourceIdentifier, RemoteResourceType] - [RemoteService] label_matchers: - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' + - Telemetry.Source + regex: '^(ClientSpan|ProducerSpan|ConsumerSpan)$' metric_name_selectors: - Latency - Fault diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml index d4bf0794bd..ce85e50f78 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_k8s.yaml @@ -1,37 +1,37 @@ -log_group_name: "/aws/appsignals/k8s" -namespace: "AppSignals" +log_group_name: "/aws/application-signals/data" +namespace: "ApplicationSignals" middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service] + - [Environment, Service, Operation] + - [Environment, Service] label_matchers: - label_names: - - aws.span.kind - regex: '^(SERVER|LOCAL_ROOT)$' + - Telemetry.Source + regex: ^(ServerSpan|LocalRootSpan)$ metric_name_selectors: - Latency - Fault - Error - dimensions: - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation] - - [HostedIn.K8s.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteTarget] - - [RemoteService, RemoteTarget] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, Operation, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, Operation, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteEnvironment] + - [Environment, Service, RemoteService] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation, RemoteEnvironment] + - [Environment, Service, RemoteService, RemoteOperation, RemoteResourceIdentifier, RemoteResourceType] + - [Environment, Service, RemoteService, RemoteOperation] + - [Environment, Service, RemoteService, RemoteResourceIdentifier, RemoteResourceType] + - [RemoteService, RemoteResourceIdentifier, RemoteResourceType] - [RemoteService] label_matchers: - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER|CONSUMER)$' + - Telemetry.Source + regex: '^(ClientSpan|ProducerSpan|ConsumerSpan)$' metric_name_selectors: - Latency - Fault diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 3830742569..bc130f3d54 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -22,6 +22,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) //go:embed awsemf_default_ecs.yaml @@ -42,9 +43,6 @@ var appSignalsConfigK8s string //go:embed appsignals_config_generic.yaml var appSignalsConfigGeneric string -//go:embed appsignals_config_ec2.yaml -var appSignalsConfigEC2 string - var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) @@ -142,14 +140,27 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { func getAppSignalsConfig() string { ctx := context.CurrentContext() - kubernetesMode := ctx.KubernetesMode() - if kubernetesMode == config.ModeEKS { + mode := ctx.KubernetesMode() + if mode == "" { + mode = ctx.Mode() + } + if mode == config.ModeEC2 { + if ecsutil.GetECSUtilSingleton().IsECS() { + mode = config.ModeECS + } + } + + switch mode { + case config.ModeEKS: return appSignalsConfigEks - } else if kubernetesMode == config.ModeK8sEC2 || kubernetesMode == config.ModeK8sOnPrem { + case config.ModeK8sEC2, config.ModeK8sOnPrem: return appSignalsConfigK8s + case config.ModeEC2, config.ModeECS: + return appSignalsConfigGeneric + default: + return appSignalsConfigGeneric } - return appSignalsConfigGeneric } func (t *translator) isAppSignals(conf *confmap.Conf) bool { diff --git a/translator/translate/otel/exporter/awsxray/translator.go b/translator/translate/otel/exporter/awsxray/translator.go index 825b300b3f..3754080f86 100644 --- a/translator/translate/otel/exporter/awsxray/translator.go +++ b/translator/translate/otel/exporter/awsxray/translator.go @@ -35,26 +35,10 @@ type translator struct { var _ common.Translator[component.Config] = (*translator)(nil) var ( - indexedAttributesEKS = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", - "HostedIn.Environment", "HostedIn.EKS.Cluster", - } - - indexedAttributesK8s = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", - "HostedIn.Environment", "HostedIn.K8s.Cluster", - } - - indexedAttributesEC2 = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "HostedIn.EC2.Environment", "aws.remote.target", - } - - indexedAttributesGeneric = []string{ - "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", "aws.remote.target", - "HostedIn.Environment", + indexedAttributes = []string{ + "aws.local.service", "aws.local.operation", "aws.local.environment", + "aws.remote.service", "aws.remote.operation", "aws.remote.environment", + "aws.remote.resource.identifier", "aws.remote.resource.type", } ) @@ -80,14 +64,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*awsxrayexporter.Config) if isAppSignals(conf) { - ctx := context.CurrentContext() - if ctx.KubernetesMode() == config.ModeEKS { - cfg.IndexedAttributes = indexedAttributesEKS - } else if ctx.KubernetesMode() == config.ModeK8sEC2 || ctx.KubernetesMode() == config.ModeK8sOnPrem { - cfg.IndexedAttributes = indexedAttributesK8s - } else { - cfg.IndexedAttributes = indexedAttributesGeneric - } + cfg.IndexedAttributes = indexedAttributes } c := confmap.NewFromStringMap(map[string]interface{}{ diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index dd6ff49227..e59f1c5672 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -74,13 +74,12 @@ func TestTranslator(t *testing.T) { "indexed_attributes": []string{ "aws.local.service", "aws.local.operation", + "aws.local.environment", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", - "K8s.RemoteNamespace", - "aws.remote.target", - "HostedIn.Environment", - "HostedIn.EKS.Cluster", + "aws.remote.environment", + "aws.remote.resource.identifier", + "aws.remote.resource.type", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", @@ -106,13 +105,12 @@ func TestTranslator(t *testing.T) { "indexed_attributes": []string{ "aws.local.service", "aws.local.operation", + "aws.local.environment", "aws.remote.service", "aws.remote.operation", - "HostedIn.K8s.Namespace", - "K8s.RemoteNamespace", - "aws.remote.target", - "HostedIn.Environment", - "HostedIn.K8s.Cluster", + "aws.remote.environment", + "aws.remote.resource.identifier", + "aws.remote.resource.type", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", @@ -138,10 +136,12 @@ func TestTranslator(t *testing.T) { "indexed_attributes": []string{ "aws.local.service", "aws.local.operation", + "aws.local.environment", "aws.remote.service", "aws.remote.operation", - "aws.remote.target", - "HostedIn.Environment", + "aws.remote.environment", + "aws.remote.resource.identifier", + "aws.remote.resource.type", }, "certificate_file_path": "/ca/bundle", "region": "us-east-1", diff --git a/translator/translate/otel/pipeline/appsignals/translator.go b/translator/translate/otel/pipeline/applicationsignals/translator.go similarity index 93% rename from translator/translate/otel/pipeline/appsignals/translator.go rename to translator/translate/otel/pipeline/applicationsignals/translator.go index db2b2ac25d..803afbd002 100644 --- a/translator/translate/otel/pipeline/appsignals/translator.go +++ b/translator/translate/otel/pipeline/applicationsignals/translator.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package appsignals +package applicationsignals import ( "fmt" @@ -14,7 +14,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsxray" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/awsproxy" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsappsignals" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsapplicationsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" ) @@ -52,7 +52,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators.Processors.Set(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType))) - translators.Processors.Set(awsappsignals.NewTranslator(awsappsignals.WithDataType(t.dataType))) + translators.Processors.Set(awsapplicationsignals.NewTranslator(awsapplicationsignals.WithDataType(t.dataType))) if t.dataType == component.DataTypeTraces { translators.Exporters.Set(awsxray.NewTranslatorWithName(common.AppSignals)) diff --git a/translator/translate/otel/pipeline/appsignals/translator_test.go b/translator/translate/otel/pipeline/applicationsignals/translator_test.go similarity index 99% rename from translator/translate/otel/pipeline/appsignals/translator_test.go rename to translator/translate/otel/pipeline/applicationsignals/translator_test.go index dc215e406e..575f726c95 100644 --- a/translator/translate/otel/pipeline/appsignals/translator_test.go +++ b/translator/translate/otel/pipeline/applicationsignals/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package appsignals +package applicationsignals import ( "fmt" diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_ec2.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_ec2.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_ec2.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_ec2.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_eks.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_eks.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_eks.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_eks.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_generic.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_generic.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_generic.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_generic.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/config_k8s.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/config_k8s.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/config_k8s.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/config_k8s.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json b/translator/translate/otel/processor/awsapplicationsignals/testdata/invalidRulesConfig.json similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json rename to translator/translate/otel/processor/awsapplicationsignals/testdata/invalidRulesConfig.json diff --git a/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json b/translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfig.json similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json rename to translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfig.json diff --git a/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigEKS.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigEKS.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigEKS.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigEKS.yaml diff --git a/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigGeneric.yaml b/translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigGeneric.yaml similarity index 100% rename from translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigGeneric.yaml rename to translator/translate/otel/processor/awsapplicationsignals/testdata/validRulesConfigGeneric.yaml diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsapplicationsignals/translator.go similarity index 89% rename from translator/translate/otel/processor/awsappsignals/translator.go rename to translator/translate/otel/processor/awsapplicationsignals/translator.go index 689a807aab..49c669efc9 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsapplicationsignals/translator.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( _ "embed" @@ -12,13 +12,14 @@ import ( "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/processor" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" - appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" + appsignalsconfig "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/rules" "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) type translator struct { @@ -48,7 +49,7 @@ func WithDataType(dataType component.DataType) Option { var _ common.Translator[component.Config] = (*translator)(nil) func NewTranslator(opts ...Option) common.Translator[component.Config] { - t := &translator{factory: awsappsignals.NewFactory()} + t := &translator{factory: awsapplicationsignals.NewFactory()} for _, opt := range opts { opt.apply(t) } @@ -75,16 +76,33 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } } - kubernetesMode := context.CurrentContext().KubernetesMode() - if kubernetesMode == config.ModeEKS { + mode := context.CurrentContext().KubernetesMode() + if mode == "" { + mode = context.CurrentContext().Mode() + } + if mode == config.ModeEC2 { + if ecsutil.GetECSUtilSingleton().IsECS() { + mode = config.ModeECS + } + } + switch mode { + case config.ModeEKS: cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewEKSResolver(hostedIn), } - } else if kubernetesMode == config.ModeK8sEC2 || kubernetesMode == config.ModeK8sOnPrem { + case config.ModeK8sEC2, config.ModeK8sOnPrem: cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewK8sResolver(hostedIn), } - } else { + case config.ModeEC2: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewEC2Resolver(hostedIn), + } + case config.ModeECS: + cfg.Resolvers = []appsignalsconfig.Resolver{ + appsignalsconfig.NewGenericResolver(hostedIn), + } + default: cfg.Resolvers = []appsignalsconfig.Resolver{ appsignalsconfig.NewGenericResolver(hostedIn), } diff --git a/translator/translate/otel/processor/awsappsignals/translator_test.go b/translator/translate/otel/processor/awsapplicationsignals/translator_test.go similarity index 91% rename from translator/translate/otel/processor/awsappsignals/translator_test.go rename to translator/translate/otel/processor/awsapplicationsignals/translator_test.go index cca81d4f1b..9f222d0116 100644 --- a/translator/translate/otel/processor/awsappsignals/translator_test.go +++ b/translator/translate/otel/processor/awsapplicationsignals/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsappsignals +package awsapplicationsignals import ( _ "embed" @@ -14,8 +14,8 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/config" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals/config" translatorConfig "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" @@ -26,7 +26,7 @@ var ( validAppSignalsYamlEKS string //go:embed testdata/config_k8s.yaml validAppSignalsYamlK8s string - //go:embed testdata/config_generic.yaml + //go:embed testdata/config_ec2.yaml validAppSignalsYamlEC2 string //go:embed testdata/config_generic.yaml validAppSignalsYamlGeneric string @@ -54,8 +54,8 @@ func TestTranslate(t *testing.T) { kubernetesMode string mode string }{ - //The config for the awsappsignals processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/appsignals/auto-instrumentation-new.yaml#L20 - //The awsappsignals processor config does not have a platform field, instead it gets added to resolvers when marshalled + //The config for the awsapplicationsignals processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/appsignals/auto-instrumentation-new.yaml#L20 + //The awsapplicationsignals processor config does not have a platform field, instead it gets added to resolvers when marshalled "WithAppSignalsEnabledEKS": { input: map[string]interface{}{ "logs": map[string]interface{}{ @@ -118,7 +118,7 @@ func TestTranslate(t *testing.T) { "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ "application_signals": map[string]interface{}{ - "hosted_in": "", + "hosted_in": "test", }, }, }}, @@ -169,7 +169,7 @@ func TestTranslate(t *testing.T) { "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ "app_signals": map[string]interface{}{ - "hosted_in": "", + "hosted_in": "test", }, }, }}, @@ -177,7 +177,7 @@ func TestTranslate(t *testing.T) { mode: translatorConfig.ModeEC2, }, } - factory := awsappsignals.NewFactory() + factory := awsapplicationsignals.NewFactory() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { if testCase.isKubernetes { diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 4cdc8540c1..79649dfa45 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -22,7 +22,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/appsignals" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/applicationsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsights" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/emf_logs" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/host" @@ -67,8 +67,8 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { } }) translators := common.NewTranslatorMap( - appsignals.NewTranslator(component.DataTypeTraces), - appsignals.NewTranslator(component.DataTypeMetrics), + applicationsignals.NewTranslator(component.DataTypeTraces), + applicationsignals.NewTranslator(component.DataTypeMetrics), host.NewTranslator(common.PipelineNameHost, hostReceivers), host.NewTranslator(common.PipelineNameHostDeltaMetrics, deltaMetricsReceivers), containerinsights.NewTranslator(),