From 73f4e3501b6f3761eb6e5a9f5797918e290325f0 Mon Sep 17 00:00:00 2001 From: Mahad Janjua Date: Wed, 4 Oct 2023 16:28:10 -0700 Subject: [PATCH] [APM] High Cardinality: Integration testing The implemented tests ensure that a user can change the configuration of CW agent to drop, keep, or replace certain metrics. --- generator/test_case_generator.go | 15 ++ test/apm/apm_test.go | 7 +- test/apm/default_test.go | 3 +- test/apm/high_cardinality_drop/apm_test.go | 80 +++++++ .../high_cardinality_drop_test.go | 69 ++++++ .../resources/client_producer.json | 199 ++++++++++++++++ .../resources/config.json | 31 +++ .../resources/server_consumer.json | 219 ++++++++++++++++++ test/apm/high_cardinality_keep/apm_test.go | 80 +++++++ .../high_cardinality_keep_test.go | 70 ++++++ .../resources/client_producer.json | 199 ++++++++++++++++ .../resources/config.json | 31 +++ .../resources/server_consumer.json | 219 ++++++++++++++++++ test/apm/high_cardinality_replace/apm_test.go | 89 +++++++ .../high_cardinality_replace_test.go | 76 ++++++ .../resources/client_producer.json | 199 ++++++++++++++++ .../resources/config.json | 37 +++ .../resources/server_consumer.json | 219 ++++++++++++++++++ test/metric/apm_util.go | 41 +++- 19 files changed, 1874 insertions(+), 9 deletions(-) create mode 100644 test/apm/high_cardinality_drop/apm_test.go create mode 100644 test/apm/high_cardinality_drop/high_cardinality_drop_test.go create mode 100644 test/apm/high_cardinality_drop/resources/client_producer.json create mode 100644 test/apm/high_cardinality_drop/resources/config.json create mode 100644 test/apm/high_cardinality_drop/resources/server_consumer.json create mode 100644 test/apm/high_cardinality_keep/apm_test.go create mode 100644 test/apm/high_cardinality_keep/high_cardinality_keep_test.go create mode 100644 test/apm/high_cardinality_keep/resources/client_producer.json create mode 100644 test/apm/high_cardinality_keep/resources/config.json create mode 100644 test/apm/high_cardinality_keep/resources/server_consumer.json create mode 100644 test/apm/high_cardinality_replace/apm_test.go create mode 100644 test/apm/high_cardinality_replace/high_cardinality_replace_test.go create mode 100644 test/apm/high_cardinality_replace/resources/client_producer.json create mode 100644 test/apm/high_cardinality_replace/resources/config.json create mode 100644 test/apm/high_cardinality_replace/resources/server_consumer.json diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index af13f06cc..309e07005 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -165,6 +165,21 @@ var testTypeToTestConfig = map[string][]testConfig{ {testDir: "./test/apm", terraformDir: "terraform/eks/daemon/apm", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, + { + testDir: "./test/apm/high_cardinality_drop", + terraformDir: "terraform/eks/daemon/apm", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/apm/high_cardinality_keep", + terraformDir: "terraform/eks/daemon/apm", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/apm/high_cardinality_replace", + terraformDir: "terraform/eks/daemon/apm", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, }, "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, diff --git a/test/apm/apm_test.go b/test/apm/apm_test.go index 5e3f34f14..7c3a96dbc 100644 --- a/test/apm/apm_test.go +++ b/test/apm/apm_test.go @@ -7,10 +7,11 @@ package apm import ( "fmt" - "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" "log" "testing" + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + "github.com/stretchr/testify/suite" "github.com/aws/amazon-cloudwatch-agent-test/environment" @@ -52,11 +53,11 @@ func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { eksTestRunners = []*test_runner.EKSTestRunner{ { - Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "EKS.Cluster"}, + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "HostedIn.EKS.Cluster"}, Env: *env, }, { - Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "EKS.Cluster"}, + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "HostedIn.EKS.Cluster"}, Env: *env, }, } diff --git a/test/apm/default_test.go b/test/apm/default_test.go index 2b4b46633..23305b628 100644 --- a/test/apm/default_test.go +++ b/test/apm/default_test.go @@ -6,11 +6,12 @@ package apm import ( + "time" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" - "time" ) const testRetryCount = 3 diff --git a/test/apm/high_cardinality_drop/apm_test.go b/test/apm/high_cardinality_drop/apm_test.go new file mode 100644 index 000000000..0fa59ba1a --- /dev/null +++ b/test/apm/high_cardinality_drop/apm_test.go @@ -0,0 +1,80 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "fmt" + "log" + "testing" + + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +type APMTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *APMTestSuite) SetupSuite() { + fmt.Println(">>>> Starting APMTestSuite") +} + +func (suite *APMTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished APMTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, "APM-High-Cardinality-Drop-Metrics", "HostedIn.EKS.Cluster"}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *APMTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "APM Test Suite Failed") +} + +func (suite *APMTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAPMSuite(t *testing.T) { + suite.Run(t, new(APMTestSuite)) +} diff --git a/test/apm/high_cardinality_drop/high_cardinality_drop_test.go b/test/apm/high_cardinality_drop/high_cardinality_drop_test.go new file mode 100644 index 000000000..d04ec3daf --- /dev/null +++ b/test/apm/high_cardinality_drop/high_cardinality_drop_test.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const testRetryCount = 3 +const namespace = "AWS/APM" + +type APMRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *APMRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructions() + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAPMMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(15 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *APMRunner) GetTestName() string { + return t.testName +} + +func (t *APMRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *APMRunner) GetMeasuredMetrics() []string { + return metric.APMMetricNames +} + +func (e *APMRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructions() []dimension.Instruction { + return metric.ServerConsumerInstructions +} + +var _ test_runner.ITestRunner = (*APMRunner)(nil) diff --git a/test/apm/high_cardinality_drop/resources/client_producer.json b/test/apm/high_cardinality_drop/resources/client_producer.json new file mode 100644 index 000000000..dc473cbc8 --- /dev/null +++ b/test/apm/high_cardinality_drop/resources/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/high_cardinality_drop/resources/config.json b/test/apm/high_cardinality_drop/resources/config.json new file mode 100644 index 000000000..de049ca10 --- /dev/null +++ b/test/apm/high_cardinality_drop/resources/config.json @@ -0,0 +1,31 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "apm": { + "enabled": true, + "rules": [ + { + "selectors": [ + { + "dimensions": "RemoteTarget", + "match": "remote-target" + } + ], + "action": "drop", + "rule_name": "drop" + } + ] + } + } + }, + "traces": { + "traces_collected": { + "apm": { + "enabled": true + } + } + } +} \ No newline at end of file diff --git a/test/apm/high_cardinality_drop/resources/server_consumer.json b/test/apm/high_cardinality_drop/resources/server_consumer.json new file mode 100644 index 000000000..8e3e263de --- /dev/null +++ b/test/apm/high_cardinality_drop/resources/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CONSUMER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/high_cardinality_keep/apm_test.go b/test/apm/high_cardinality_keep/apm_test.go new file mode 100644 index 000000000..fb5a69b7a --- /dev/null +++ b/test/apm/high_cardinality_keep/apm_test.go @@ -0,0 +1,80 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "fmt" + "log" + "testing" + + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +type APMTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *APMTestSuite) SetupSuite() { + fmt.Println(">>>> Starting APMTestSuite") +} + +func (suite *APMTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished APMTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, "APM-High-Cardinality-Keep-Metrics", "HostedIn.EKS.Cluster"}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *APMTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "APM Test Suite Failed") +} + +func (suite *APMTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAPMSuite(t *testing.T) { + suite.Run(t, new(APMTestSuite)) +} diff --git a/test/apm/high_cardinality_keep/high_cardinality_keep_test.go b/test/apm/high_cardinality_keep/high_cardinality_keep_test.go new file mode 100644 index 000000000..2f261aa43 --- /dev/null +++ b/test/apm/high_cardinality_keep/high_cardinality_keep_test.go @@ -0,0 +1,70 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const testRetryCount = 3 +const namespace = "AWS/APM" + +type APMRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *APMRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructions() + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAPMMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(15 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *APMRunner) GetTestName() string { + return t.testName +} + +func (t *APMRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *APMRunner) GetMeasuredMetrics() []string { + return metric.APMMetricNames +} + +func (e *APMRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructions() []dimension.Instruction { + // We choose to keep this metric for this test case + return metric.ClientProducerInstructions +} + +var _ test_runner.ITestRunner = (*APMRunner)(nil) diff --git a/test/apm/high_cardinality_keep/resources/client_producer.json b/test/apm/high_cardinality_keep/resources/client_producer.json new file mode 100644 index 000000000..dc473cbc8 --- /dev/null +++ b/test/apm/high_cardinality_keep/resources/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/high_cardinality_keep/resources/config.json b/test/apm/high_cardinality_keep/resources/config.json new file mode 100644 index 000000000..1aa0423f1 --- /dev/null +++ b/test/apm/high_cardinality_keep/resources/config.json @@ -0,0 +1,31 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "apm": { + "enabled": true, + "rules": [ + { + "selectors": [ + { + "dimensions": "RemoteTarget", + "match": "remote-target" + } + ], + "action": "keep", + "rule_name": "keep" + } + ] + } + } + }, + "traces": { + "traces_collected": { + "apm": { + "enabled": true + } + } + } +} \ No newline at end of file diff --git a/test/apm/high_cardinality_keep/resources/server_consumer.json b/test/apm/high_cardinality_keep/resources/server_consumer.json new file mode 100644 index 000000000..8e3e263de --- /dev/null +++ b/test/apm/high_cardinality_keep/resources/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CONSUMER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/high_cardinality_replace/apm_test.go b/test/apm/high_cardinality_replace/apm_test.go new file mode 100644 index 000000000..41c2e3758 --- /dev/null +++ b/test/apm/high_cardinality_replace/apm_test.go @@ -0,0 +1,89 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "fmt" + "log" + "testing" + + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const ( + APMServerConsumerTestName = "APM-High-Cardinality-Server-Consumer" + APMClientProducerTestName = "APM-High-Cardinality-Client-Producer-Replaced" +) + +type APMTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *APMTestSuite) SetupSuite() { + fmt.Println(">>>> Starting APMTestSuite") +} + +func (suite *APMTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished APMTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *APMTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "APM Test Suite Failed") +} + +func (suite *APMTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAPMSuite(t *testing.T) { + suite.Run(t, new(APMTestSuite)) +} diff --git a/test/apm/high_cardinality_replace/high_cardinality_replace_test.go b/test/apm/high_cardinality_replace/high_cardinality_replace_test.go new file mode 100644 index 000000000..e1e7bee68 --- /dev/null +++ b/test/apm/high_cardinality_replace/high_cardinality_replace_test.go @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const testRetryCount = 3 +const namespace = "AWS/APM" + +type APMRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *APMRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructionsFromTestName(t.testName) + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAPMMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(15 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *APMRunner) GetTestName() string { + return t.testName +} + +func (t *APMRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *APMRunner) GetMeasuredMetrics() []string { + return metric.APMMetricNames +} + +func (e *APMRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructionsFromTestName(testName string) []dimension.Instruction { + switch testName{ + case APMClientProducerTestName: + return metric.ClientProducerInstructions_ReplacedRemoteTarget + case APMServerConsumerTestName: + return metric.ServerConsumerInstructions + default: + return nil + } +} + +var _ test_runner.ITestRunner = (*APMRunner)(nil) diff --git a/test/apm/high_cardinality_replace/resources/client_producer.json b/test/apm/high_cardinality_replace/resources/client_producer.json new file mode 100644 index 000000000..dc473cbc8 --- /dev/null +++ b/test/apm/high_cardinality_replace/resources/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/high_cardinality_replace/resources/config.json b/test/apm/high_cardinality_replace/resources/config.json new file mode 100644 index 000000000..026d98f76 --- /dev/null +++ b/test/apm/high_cardinality_replace/resources/config.json @@ -0,0 +1,37 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "apm": { + "enabled": true, + "rules": [ + { + "selectors": [ + { + "dimensions": "RemoteTarget", + "match": "remote-target" + } + ], + "replacements":[ + { + "target_dimension":"RemoteTarget", + "value":"replaced" + } + ], + "action": "replace", + "rule_name": "replace1" + } + ] + } + } + }, + "traces": { + "traces_collected": { + "apm": { + "enabled": true + } + } + } +} \ No newline at end of file diff --git a/test/apm/high_cardinality_replace/resources/server_consumer.json b/test/apm/high_cardinality_replace/resources/server_consumer.json new file mode 100644 index 000000000..8e3e263de --- /dev/null +++ b/test/apm/high_cardinality_replace/resources/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CONSUMER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go index 4d41aa8f2..121936132 100644 --- a/test/metric/apm_util.go +++ b/test/metric/apm_util.go @@ -20,11 +20,11 @@ var ( ServerConsumerInstructions = []dimension.Instruction{ { - Key: "EKS.Cluster", + Key: "HostedIn.EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "K8s.Namespace", + Key: "HostedIn.K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { @@ -39,11 +39,11 @@ var ( ClientProducerInstructions = []dimension.Instruction{ { - Key: "EKS.Cluster", + Key: "HostedIn.EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "K8s.Namespace", + Key: "HostedIn.K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { @@ -67,6 +67,37 @@ var ( Value: dimension.ExpectedDimensionValue{Value: aws.String("remote-target")}, }, } + + ClientProducerInstructions_ReplacedRemoteTarget = []dimension.Instruction{ + { + Key: "HostedIn.EKS.Cluster", + Value: dimension.UnknownDimensionValue(), + }, + { + Key: "HostedIn.K8s.Namespace", + Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, + }, + { + Key: "Service", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name")}, + }, + { + Key: "RemoteService", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name-remote")}, + }, + { + Key: "Operation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("operation")}, + }, + { + Key: "RemoteOperation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("remote-operation")}, + }, + { + Key: "RemoteTarget", + Value: dimension.ExpectedDimensionValue{Value: aws.String("replaced")}, + }, + } ) func ValidateAPMMetric(dimFactory dimension.Factory, namespace string, metricName string, instructions []dimension.Instruction) status.TestResult { @@ -92,4 +123,4 @@ func ValidateAPMMetric(dimFactory dimension.Factory, namespace string, metricNam testResult.Status = status.SUCCESSFUL return testResult -} \ No newline at end of file +}