From 5aff0ecbdd199961b8aad1814faf37bddc671114 Mon Sep 17 00:00:00 2001 From: Mahad Janjua Date: Tue, 31 Oct 2023 11:33:00 -0700 Subject: [PATCH] App Signals: High Cardinality User Configuration Testing --- generator/test_case_generator.go | 15 ++ .../high_cardinality_drop/app_signals_test.go | 93 ++++++++ .../high_cardinality_drop_metrics_test.go | 69 ++++++ .../resources/config.json | 31 +++ .../resources/metrics/client_producer.json | 199 ++++++++++++++++ .../resources/metrics/server_consumer.json | 219 ++++++++++++++++++ .../high_cardinality_keep/app_signals_test.go | 93 ++++++++ .../high_cardinality_keep_metrics_test.go | 69 ++++++ .../resources/config.json | 31 +++ .../resources/metrics/client_producer.json | 199 ++++++++++++++++ .../resources/metrics/server_consumer.json | 219 ++++++++++++++++++ .../app_signals_test.go | 93 ++++++++ .../high_cardinality_replace_metrics_test.go | 69 ++++++ .../resources/config.json | 37 +++ .../resources/metrics/client_producer.json | 199 ++++++++++++++++ .../resources/metrics/server_consumer.json | 219 ++++++++++++++++++ test/metric/app_signals_util.go | 31 +++ 17 files changed, 1885 insertions(+) create mode 100644 test/app_signals/high_cardinality_drop/app_signals_test.go create mode 100644 test/app_signals/high_cardinality_drop/high_cardinality_drop_metrics_test.go create mode 100644 test/app_signals/high_cardinality_drop/resources/config.json create mode 100644 test/app_signals/high_cardinality_drop/resources/metrics/client_producer.json create mode 100644 test/app_signals/high_cardinality_drop/resources/metrics/server_consumer.json create mode 100644 test/app_signals/high_cardinality_keep/app_signals_test.go create mode 100644 test/app_signals/high_cardinality_keep/high_cardinality_keep_metrics_test.go create mode 100644 test/app_signals/high_cardinality_keep/resources/config.json create mode 100644 test/app_signals/high_cardinality_keep/resources/metrics/client_producer.json create mode 100644 test/app_signals/high_cardinality_keep/resources/metrics/server_consumer.json create mode 100644 test/app_signals/high_cardinality_replace/app_signals_test.go create mode 100644 test/app_signals/high_cardinality_replace/high_cardinality_replace_metrics_test.go create mode 100644 test/app_signals/high_cardinality_replace/resources/config.json create mode 100644 test/app_signals/high_cardinality_replace/resources/metrics/client_producer.json create mode 100644 test/app_signals/high_cardinality_replace/resources/metrics/server_consumer.json diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 655d1c8ee..84699f108 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -190,6 +190,21 @@ var testTypeToTestConfig = map[string][]testConfig{ {testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, + { + testDir: "./test/app_signals/high_cardinality_drop", + terraformDir: "terraform/eks/daemon/app_signals", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/app_signals/high_cardinality_keep", + terraformDir: "terraform/eks/daemon/app_signals", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/app_signals/high_cardinality_replace", + terraformDir: "terraform/eks/daemon/app_signals", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, }, "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, diff --git a/test/app_signals/high_cardinality_drop/app_signals_test.go b/test/app_signals/high_cardinality_drop/app_signals_test.go new file mode 100644 index 000000000..01e9976b8 --- /dev/null +++ b/test/app_signals/high_cardinality_drop/app_signals_test.go @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package app_signals + +import ( + "fmt" + "log" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const ( + AppSignalsServerConsumerTestName = "AppSignals-Server-Consumer" + AppSignalsClientProducerTestName = "AppSignals-Client-Producer" + AppSignalsTracesTestName = "AppSignals-Traces" +) + +type AppSignalsTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *AppSignalsTestSuite) SetupSuite() { + fmt.Println(">>>> Starting AppSignalsTestSuite") +} + +func (suite *AppSignalsTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished AppSignalsTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsServerConsumerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsClientProducerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &AppSignalsTracesRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsTracesTestName, env.EKSClusterName}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *AppSignalsTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "AppSignals Test Suite Failed") +} + +func (suite *AppSignalsTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAppSignalsSuite(t *testing.T) { + suite.Run(t, new(AppSignalsTestSuite)) +} diff --git a/test/app_signals/high_cardinality_drop/high_cardinality_drop_metrics_test.go b/test/app_signals/high_cardinality_drop/high_cardinality_drop_metrics_test.go new file mode 100644 index 000000000..d2f43d415 --- /dev/null +++ b/test/app_signals/high_cardinality_drop/high_cardinality_drop_metrics_test.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package app_signals + +import ( + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const testRetryCount = 6 +const namespace = "AppSignals" + +type AppSignalsMetricsRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *AppSignalsMetricsRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructionsFromTestName(t.testName) + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAppSignalsMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(30 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *AppSignalsMetricsRunner) GetTestName() string { + return t.testName +} + +func (t *AppSignalsMetricsRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *AppSignalsMetricsRunner) GetMeasuredMetrics() []string { + return metric.AppSignalsMetricNames +} + +func (e *AppSignalsMetricsRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructionsFromTestName(testName string) []dimension.Instruction { + return metric.ServerConsumerInstructions +} + +var _ test_runner.ITestRunner = (*AppSignalsMetricsRunner)(nil) diff --git a/test/app_signals/high_cardinality_drop/resources/config.json b/test/app_signals/high_cardinality_drop/resources/config.json new file mode 100644 index 000000000..c05d464ec --- /dev/null +++ b/test/app_signals/high_cardinality_drop/resources/config.json @@ -0,0 +1,31 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "app_signals": { + "enabled": true, + "rules": [ + { + "selectors": [ + { + "dimensions": "RemoteTarget", + "match": "remote-target" + } + ], + "action": "drop", + "rule_name": "drop" + } + ] + } + } + }, + "traces": { + "traces_collected": { + "app_signals": { + "enabled": true + } + } + } + } \ No newline at end of file diff --git a/test/app_signals/high_cardinality_drop/resources/metrics/client_producer.json b/test/app_signals/high_cardinality_drop/resources/metrics/client_producer.json new file mode 100644 index 000000000..8a39c60d6 --- /dev/null +++ b/test/app_signals/high_cardinality_drop/resources/metrics/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/app_signals/high_cardinality_drop/resources/metrics/server_consumer.json b/test/app_signals/high_cardinality_drop/resources/metrics/server_consumer.json new file mode 100644 index 000000000..6748e6fd0 --- /dev/null +++ b/test/app_signals/high_cardinality_drop/resources/metrics/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/app_signals/high_cardinality_keep/app_signals_test.go b/test/app_signals/high_cardinality_keep/app_signals_test.go new file mode 100644 index 000000000..01e9976b8 --- /dev/null +++ b/test/app_signals/high_cardinality_keep/app_signals_test.go @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package app_signals + +import ( + "fmt" + "log" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const ( + AppSignalsServerConsumerTestName = "AppSignals-Server-Consumer" + AppSignalsClientProducerTestName = "AppSignals-Client-Producer" + AppSignalsTracesTestName = "AppSignals-Traces" +) + +type AppSignalsTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *AppSignalsTestSuite) SetupSuite() { + fmt.Println(">>>> Starting AppSignalsTestSuite") +} + +func (suite *AppSignalsTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished AppSignalsTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsServerConsumerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsClientProducerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &AppSignalsTracesRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsTracesTestName, env.EKSClusterName}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *AppSignalsTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "AppSignals Test Suite Failed") +} + +func (suite *AppSignalsTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAppSignalsSuite(t *testing.T) { + suite.Run(t, new(AppSignalsTestSuite)) +} diff --git a/test/app_signals/high_cardinality_keep/high_cardinality_keep_metrics_test.go b/test/app_signals/high_cardinality_keep/high_cardinality_keep_metrics_test.go new file mode 100644 index 000000000..85a7d8f31 --- /dev/null +++ b/test/app_signals/high_cardinality_keep/high_cardinality_keep_metrics_test.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package app_signals + +import ( + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const testRetryCount = 6 +const namespace = "AppSignals" + +type AppSignalsMetricsRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *AppSignalsMetricsRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructionsFromTestName(t.testName) + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAppSignalsMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(30 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *AppSignalsMetricsRunner) GetTestName() string { + return t.testName +} + +func (t *AppSignalsMetricsRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *AppSignalsMetricsRunner) GetMeasuredMetrics() []string { + return metric.AppSignalsMetricNames +} + +func (e *AppSignalsMetricsRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructionsFromTestName(testName string) []dimension.Instruction { + return metric.ClientProducerInstructions +} + +var _ test_runner.ITestRunner = (*AppSignalsMetricsRunner)(nil) diff --git a/test/app_signals/high_cardinality_keep/resources/config.json b/test/app_signals/high_cardinality_keep/resources/config.json new file mode 100644 index 000000000..729bc9191 --- /dev/null +++ b/test/app_signals/high_cardinality_keep/resources/config.json @@ -0,0 +1,31 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "app_signals": { + "enabled": true, + "rules": [ + { + "selectors": [ + { + "dimensions": "RemoteTarget", + "match": "remote-target" + } + ], + "action": "keep", + "rule_name": "keep" + } + ] + } + } + }, + "traces": { + "traces_collected": { + "app_signals": { + "enabled": true + } + } + } + } \ No newline at end of file diff --git a/test/app_signals/high_cardinality_keep/resources/metrics/client_producer.json b/test/app_signals/high_cardinality_keep/resources/metrics/client_producer.json new file mode 100644 index 000000000..8a39c60d6 --- /dev/null +++ b/test/app_signals/high_cardinality_keep/resources/metrics/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/app_signals/high_cardinality_keep/resources/metrics/server_consumer.json b/test/app_signals/high_cardinality_keep/resources/metrics/server_consumer.json new file mode 100644 index 000000000..6748e6fd0 --- /dev/null +++ b/test/app_signals/high_cardinality_keep/resources/metrics/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/app_signals/high_cardinality_replace/app_signals_test.go b/test/app_signals/high_cardinality_replace/app_signals_test.go new file mode 100644 index 000000000..01e9976b8 --- /dev/null +++ b/test/app_signals/high_cardinality_replace/app_signals_test.go @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package app_signals + +import ( + "fmt" + "log" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const ( + AppSignalsServerConsumerTestName = "AppSignals-Server-Consumer" + AppSignalsClientProducerTestName = "AppSignals-Client-Producer" + AppSignalsTracesTestName = "AppSignals-Traces" +) + +type AppSignalsTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *AppSignalsTestSuite) SetupSuite() { + fmt.Println(">>>> Starting AppSignalsTestSuite") +} + +func (suite *AppSignalsTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished AppSignalsTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsServerConsumerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsClientProducerTestName, "HostedIn.EKS.Cluster"}, + Env: *env, + }, + { + Runner: &AppSignalsTracesRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsTracesTestName, env.EKSClusterName}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *AppSignalsTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "AppSignals Test Suite Failed") +} + +func (suite *AppSignalsTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAppSignalsSuite(t *testing.T) { + suite.Run(t, new(AppSignalsTestSuite)) +} diff --git a/test/app_signals/high_cardinality_replace/high_cardinality_replace_metrics_test.go b/test/app_signals/high_cardinality_replace/high_cardinality_replace_metrics_test.go new file mode 100644 index 000000000..366a79e70 --- /dev/null +++ b/test/app_signals/high_cardinality_replace/high_cardinality_replace_metrics_test.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package app_signals + +import ( + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const testRetryCount = 6 +const namespace = "AppSignals" + +type AppSignalsMetricsRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *AppSignalsMetricsRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructionsFromTestName(t.testName) + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAppSignalsMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(30 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *AppSignalsMetricsRunner) GetTestName() string { + return t.testName +} + +func (t *AppSignalsMetricsRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *AppSignalsMetricsRunner) GetMeasuredMetrics() []string { + return metric.AppSignalsMetricNames +} + +func (e *AppSignalsMetricsRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructionsFromTestName(testName string) []dimension.Instruction { + return metric.ClientProducerReplacedInstructions +} + +var _ test_runner.ITestRunner = (*AppSignalsMetricsRunner)(nil) diff --git a/test/app_signals/high_cardinality_replace/resources/config.json b/test/app_signals/high_cardinality_replace/resources/config.json new file mode 100644 index 000000000..1e2f396c2 --- /dev/null +++ b/test/app_signals/high_cardinality_replace/resources/config.json @@ -0,0 +1,37 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "app_signals": { + "enabled": true, + "rules": [ + { + "selectors": [ + { + "dimensions": "RemoteTarget", + "match": "remote-target" + } + ], + "replacements":[ + { + "target_dimension":"RemoteTarget", + "value":"replaced" + } + ], + "action": "replace", + "rule_name": "replace1" + } + ] + } + } + }, + "traces": { + "traces_collected": { + "app_signals": { + "enabled": true + } + } + } + } \ No newline at end of file diff --git a/test/app_signals/high_cardinality_replace/resources/metrics/client_producer.json b/test/app_signals/high_cardinality_replace/resources/metrics/client_producer.json new file mode 100644 index 000000000..8a39c60d6 --- /dev/null +++ b/test/app_signals/high_cardinality_replace/resources/metrics/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/app_signals/high_cardinality_replace/resources/metrics/server_consumer.json b/test/app_signals/high_cardinality_replace/resources/metrics/server_consumer.json new file mode 100644 index 000000000..6748e6fd0 --- /dev/null +++ b/test/app_signals/high_cardinality_replace/resources/metrics/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/metric/app_signals_util.go b/test/metric/app_signals_util.go index 781209ab8..4e070e40d 100644 --- a/test/metric/app_signals_util.go +++ b/test/metric/app_signals_util.go @@ -68,6 +68,37 @@ var ( Value: dimension.ExpectedDimensionValue{Value: aws.String("remote-target")}, }, } + + ClientProducerReplacedInstructions = []dimension.Instruction{ + { + Key: "HostedIn.EKS.Cluster", + Value: dimension.UnknownDimensionValue(), + }, + { + Key: "HostedIn.K8s.Namespace", + Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, + }, + { + Key: "Service", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name")}, + }, + { + Key: "RemoteService", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name-remote")}, + }, + { + Key: "Operation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("operation")}, + }, + { + Key: "RemoteOperation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("remote-operation")}, + }, + { + Key: "RemoteTarget", + Value: dimension.ExpectedDimensionValue{Value: aws.String("replaced")}, + }, + } ) func ValidateAppSignalsMetric(dimFactory dimension.Factory, namespace string, metricName string, instructions []dimension.Instruction) status.TestResult {