From 6916897f4aa22f083e8440a64bb8901ec072a6cc Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 27 Jul 2023 10:03:36 -0400 Subject: [PATCH 01/29] Add APM metric testing --- generator/test_case_generator.go | 3 + terraform/eks/daemon/apm/main.tf | 495 +++++++++++++++++++ terraform/eks/daemon/apm/providers.tf | 17 + terraform/eks/daemon/apm/variables.tf | 37 ++ test/apm/apm_test.go | 88 ++++ test/apm/default_test.go | 75 +++ test/apm/resources/client_producer.json | 199 ++++++++ test/apm/resources/config.json | 19 + test/apm/resources/server_consumer.json | 219 ++++++++ test/metric/apm_util.go | 95 ++++ test/metric/dimension/instanceid_provider.go | 8 + test/metric/stat.go | 1 + 12 files changed, 1256 insertions(+) create mode 100644 terraform/eks/daemon/apm/main.tf create mode 100644 terraform/eks/daemon/apm/providers.tf create mode 100644 terraform/eks/daemon/apm/variables.tf create mode 100644 test/apm/apm_test.go create mode 100644 test/apm/default_test.go create mode 100644 test/apm/resources/client_producer.json create mode 100644 test/apm/resources/config.json create mode 100644 test/apm/resources/server_consumer.json create mode 100644 test/metric/apm_util.go diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index fe240389d..a219652c8 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -163,6 +163,9 @@ var testTypeToTestConfig = map[string][]testConfig{ targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, + {testDir: "./test/apm", terraformDir: "terraform/eks/daemon/apm", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, }, "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, diff --git a/terraform/eks/daemon/apm/main.tf b/terraform/eks/daemon/apm/main.tf new file mode 100644 index 000000000..6167d48bb --- /dev/null +++ b/terraform/eks/daemon/apm/main.tf @@ -0,0 +1,495 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +module "common" { + source = "../../../common" + cwagent_image_repo = var.cwagent_image_repo + cwagent_image_tag = var.cwagent_image_tag +} + +module "basic_components" { + source = "../../../basic_components" + + region = var.region +} + +data "aws_eks_cluster_auth" "this" { + name = aws_eks_cluster.this.name +} + +resource "aws_eks_cluster" "this" { + name = "cwagent-eks-integ-${module.common.testing_id}" + role_arn = module.basic_components.role_arn + version = var.k8s_version + enabled_cluster_log_types = [ + "api", + "audit", + "authenticator", + "controllerManager", + "scheduler" + ] + vpc_config { + subnet_ids = module.basic_components.public_subnet_ids + security_group_ids = [module.basic_components.security_group] + } +} + +# EKS Node Groups +resource "aws_eks_node_group" "this" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "cwagent-eks-integ-node" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = "AL2_x86_64" + capacity_type = "ON_DEMAND" + disk_size = 20 + instance_types = ["t3.medium"] + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy + ] +} + +# EKS Node IAM Role +resource "aws_iam_role" "node_role" { + name = "cwagent-eks-Worker-Role-${module.common.testing_id}" + + assume_role_policy = < server_consumer.json; curl -H 'Content-Type: application/json' -d @server_consumer.json -i http://127.0.0.1:4318/v1/metrics --verbose --http0.9; echo '${data.template_file.client_producer.rendered}' | sed -e \"s/START_TIME/$(date +%s%N)/\" > client_producer.json; curl -H 'Content-Type: application/json' -d @client_producer.json -i http://127.0.0.1:4318/v1/metrics --verbose --http0.9; sleep 1; done" + ] + env { + name = "HOST_IP" + value_from { + field_ref { + field_path = "status.hostIP" + } + } + } + env { + name = "HOST_NAME" + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + env { + name = "K8S_NAMESPACE" + value_from { + field_ref { + field_path = "metadata.namespace" + } + } + } + volume_mount { + mount_path = "/etc/cwagentconfig" + name = "cwagentconfig" + } + } + service_account_name = "cloudwatch-agent" + termination_grace_period_seconds = 60 + } + } + } +} + +########################################## +# Template Files +########################################## +locals { + cwagent_config = "../../../../${var.test_dir}/resources/config.json" + server_consumer = "../../../../${var.test_dir}/resources/server_consumer.json" + client_producer = "../../../../${var.test_dir}/resources/client_producer.json" +} + +data "template_file" "cwagent_config" { + template = file(local.cwagent_config) + vars = { + } +} + +resource "kubernetes_config_map" "cwagentconfig" { + depends_on = [ + kubernetes_namespace.namespace, + kubernetes_service_account.cwagentservice + ] + metadata { + name = "cwagentconfig" + namespace = "amazon-cloudwatch" + } + data = { + "cwagentconfig.json" : data.template_file.cwagent_config.rendered + } +} + +data "template_file" "server_consumer" { + template = file(local.server_consumer) + vars = { + } +} + +data "template_file" "client_producer" { + template = file(local.client_producer) + vars = { + } +} + +resource "kubernetes_service_account" "cwagentservice" { + depends_on = [kubernetes_namespace.namespace] + metadata { + name = "cloudwatch-agent" + namespace = "amazon-cloudwatch" + } +} + +resource "kubernetes_cluster_role" "clusterrole" { + depends_on = [kubernetes_namespace.namespace] + metadata { + name = "cloudwatch-agent-role" + } + rule { + verbs = ["list", "watch"] + resources = ["pods", "nodes", "endpoints", "services"] + api_groups = [""] + } + rule { + verbs = ["list", "watch"] + resources = ["replicasets"] + api_groups = ["apps"] + } + rule { + verbs = ["list", "watch"] + resources = ["jobs"] + api_groups = ["batch"] + } + rule { + verbs = ["get"] + resources = ["nodes/proxy", "configmaps"] + api_groups = [""] + } + rule { + verbs = ["create"] + resources = ["nodes/stats", "configmaps", "events"] + api_groups = [""] + } + rule { + verbs = ["get", "update"] + resource_names = ["cwagent-clusterleader"] + resources = ["configmaps"] + api_groups = [""] + } +} + +resource "kubernetes_cluster_role_binding" "rolebinding" { + depends_on = [kubernetes_namespace.namespace] + metadata { + name = "cloudwatch-agent-role-binding" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cloudwatch-agent-role" + } + subject { + kind = "ServiceAccount" + name = "cloudwatch-agent" + namespace = "amazon-cloudwatch" + } +} + +resource "null_resource" "validator" { + depends_on = [ + aws_eks_node_group.this, + kubernetes_daemonset.service, + kubernetes_cluster_role_binding.rolebinding, + kubernetes_service_account.cwagentservice, + ] + provisioner "local-exec" { + command = <<-EOT + echo "Validating EKS metrics/logs for AMF" + cd ../../../.. + go test ${var.test_dir} -eksClusterName=${aws_eks_cluster.this.name} -computeType=EKS -v -eksDeploymentStrategy=DAEMON + EOT + } +} diff --git a/terraform/eks/daemon/apm/providers.tf b/terraform/eks/daemon/apm/providers.tf new file mode 100644 index 000000000..9bd2885f5 --- /dev/null +++ b/terraform/eks/daemon/apm/providers.tf @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +provider "aws" { + region = var.region +} + +provider "kubernetes" { + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.this.token +} \ No newline at end of file diff --git a/terraform/eks/daemon/apm/variables.tf b/terraform/eks/daemon/apm/variables.tf new file mode 100644 index 000000000..aacb0eaa1 --- /dev/null +++ b/terraform/eks/daemon/apm/variables.tf @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +variable "region" { + type = string + default = "us-west-2" +} + +variable "test_dir" { + type = string + default = "./test/apm" +} + +variable "cwagent_image_repo" { + type = string + default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-release" +} + +variable "cwagent_image_tag" { + type = string + default = "latest" +} + +variable "k8s_version" { + type = string + default = "1.24" +} + +variable "ami_type" { + type = string + default = "AL2_x86_64" +} + +variable "instance_type" { + type = string + default = "t3a.medium" +} \ No newline at end of file diff --git a/test/apm/apm_test.go b/test/apm/apm_test.go new file mode 100644 index 000000000..5e3f34f14 --- /dev/null +++ b/test/apm/apm_test.go @@ -0,0 +1,88 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "fmt" + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" + "log" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +const ( + APMServerConsumerTestName = "APM-Server-Consumer" + APMClientProducerTestName = "APM-Client-Producer" +) + +type APMTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *APMTestSuite) SetupSuite() { + fmt.Println(">>>> Starting APMTestSuite") +} + +func (suite *APMTestSuite) TearDownSuite() { + suite.Result.Print() + fmt.Println(">>>> Finished APMTestSuite") +} + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +var ( + eksTestRunners []*test_runner.EKSTestRunner +) + +func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + + eksTestRunners = []*test_runner.EKSTestRunner{ + { + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "EKS.Cluster"}, + Env: *env, + }, + { + Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "EKS.Cluster"}, + Env: *env, + }, + } + } + return eksTestRunners +} + +func (suite *APMTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + switch env.ComputeType { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: + return + } + + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "APM Test Suite Failed") +} + +func (suite *APMTestSuite) AddToSuiteResult(r status.TestGroupResult) { + suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) +} + +func TestAPMSuite(t *testing.T) { + suite.Run(t, new(APMTestSuite)) +} diff --git a/test/apm/default_test.go b/test/apm/default_test.go new file mode 100644 index 000000000..2b4b46633 --- /dev/null +++ b/test/apm/default_test.go @@ -0,0 +1,75 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" + "time" +) + +const testRetryCount = 3 +const namespace = "AWS/APM" + +type APMRunner struct { + test_runner.BaseTestRunner + testName string + dimensionKey string +} + +func (t *APMRunner) Validate() status.TestGroupResult { + metricsToFetch := t.GetMeasuredMetrics() + testResults := make([]status.TestResult, len(metricsToFetch)) + instructions := GetInstructionsFromTestName(t.testName) + + for i, metricName := range metricsToFetch { + var testResult status.TestResult + for j := 0; j < testRetryCount; j++ { + testResult = metric.ValidateAPMMetric(t.DimensionFactory, namespace, metricName, instructions) + if testResult.Status == status.SUCCESSFUL { + break + } + time.Sleep(15 * time.Second) + } + testResults[i] = testResult + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *APMRunner) GetTestName() string { + return t.testName +} + +func (t *APMRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *APMRunner) GetMeasuredMetrics() []string { + return metric.APMMetricNames +} + +func (e *APMRunner) GetAgentConfigFileName() string { + return "" +} + +func GetInstructionsFromTestName(testName string) []dimension.Instruction { + switch testName{ + case APMClientProducerTestName: + return metric.ClientProducerInstructions + case APMServerConsumerTestName: + return metric.ServerConsumerInstructions + default: + return nil + } +} + +var _ test_runner.ITestRunner = (*APMRunner)(nil) diff --git a/test/apm/resources/client_producer.json b/test/apm/resources/client_producer.json new file mode 100644 index 000000000..dc473cbc8 --- /dev/null +++ b/test/apm/resources/client_producer.json @@ -0,0 +1,199 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/resources/config.json b/test/apm/resources/config.json new file mode 100644 index 000000000..fbaf609a9 --- /dev/null +++ b/test/apm/resources/config.json @@ -0,0 +1,19 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "apm": { + "enabled": true + } + } + }, + "traces": { + "traces_collected": { + "apm": { + "enabled": true + } + } + } +} \ No newline at end of file diff --git a/test/apm/resources/server_consumer.json b/test/apm/resources/server_consumer.json new file mode 100644 index 000000000..8e3e263de --- /dev/null +++ b/test/apm/resources/server_consumer.json @@ -0,0 +1,219 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "Error", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Fault", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "SERVER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + } + , + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + }, + { + "name": "Latency", + "unit": "Milliseconds", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CONSUMER" + } + }, + { + "key": "Operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "Service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "K8s.Namespace", + "value": { + "stringValue": "default" + } + }, + { + "key": "K8s.Pod", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "K8s.Node", + "value": { + "stringValue": "i-00000000000000000" + } + }, + { + "key": "K8s.Workload", + "value": { + "stringValue": "sample-app" + } + } + ], + "startTimeUnixNano": START_TIME, + "timeUnixNano": START_TIME, + "sum": 0, + "min": 0, + "max": 0 + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go new file mode 100644 index 000000000..4d41aa8f2 --- /dev/null +++ b/test/metric/apm_util.go @@ -0,0 +1,95 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package metric + +import ( + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/aws-sdk-go-v2/aws" +) + +var ( + APMMetricNames = []string{ + "Error", + "Fault", + "Latency", + } + + ServerConsumerInstructions = []dimension.Instruction{ + { + Key: "EKS.Cluster", + Value: dimension.UnknownDimensionValue(), + }, + { + Key: "K8s.Namespace", + Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, + }, + { + Key: "Service", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name")}, + }, + { + Key: "Operation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("operation")}, + }, + } + + ClientProducerInstructions = []dimension.Instruction{ + { + Key: "EKS.Cluster", + Value: dimension.UnknownDimensionValue(), + }, + { + Key: "K8s.Namespace", + Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, + }, + { + Key: "Service", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name")}, + }, + { + Key: "RemoteService", + Value: dimension.ExpectedDimensionValue{Value: aws.String("service-name-remote")}, + }, + { + Key: "Operation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("operation")}, + }, + { + Key: "RemoteOperation", + Value: dimension.ExpectedDimensionValue{Value: aws.String("remote-operation")}, + }, + { + Key: "RemoteTarget", + Value: dimension.ExpectedDimensionValue{Value: aws.String("remote-target")}, + }, + } +) + +func ValidateAPMMetric(dimFactory dimension.Factory, namespace string, metricName string, instructions []dimension.Instruction) status.TestResult { + testResult := status.TestResult{ + Name: metricName, + Status: status.FAILED, + } + + dims, failed := dimFactory.GetDimensions(instructions) + if len(failed) > 0 { + return testResult + } + + fetcher := MetricValueFetcher{} + values, err := fetcher.Fetch(namespace, metricName, dims, SUM, HighResolutionStatPeriod) + if err != nil { + return testResult + } + + if !IsAllValuesGreaterThanOrEqualToExpectedValue(metricName, values, 0) { + return testResult + } + + testResult.Status = status.SUCCESSFUL + return testResult +} \ No newline at end of file diff --git a/test/metric/dimension/instanceid_provider.go b/test/metric/dimension/instanceid_provider.go index 218208013..0e2819d78 100644 --- a/test/metric/dimension/instanceid_provider.go +++ b/test/metric/dimension/instanceid_provider.go @@ -79,6 +79,14 @@ func (p *EKSClusterNameProvider) IsApplicable() bool { } func (p *EKSClusterNameProvider) GetDimension(instruction Instruction) types.Dimension { + // For APM metrics, cluster name is under EKS.Cluster dimension + if instruction.Key == "EKS.Cluster" { + return types.Dimension{ + Name: aws.String("EKS.Cluster"), + Value: aws.String(p.env.EKSClusterName), + } + } + if instruction.Key != "ClusterName" || instruction.Value.IsKnown() { return types.Dimension{} } diff --git a/test/metric/stat.go b/test/metric/stat.go index ecb9615d3..763028566 100644 --- a/test/metric/stat.go +++ b/test/metric/stat.go @@ -11,5 +11,6 @@ const ( SAMPLE_COUNT Statistics = "SampleCount" MINIMUM Statistics = "Minimum" MAXUMUM Statistics = "Maxmimum" + SUM Statistics = "Sum" HighResolutionStatPeriod = 10 ) From 53295ccb27a18c67cf5d7f5231499e35fda1a3f6 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Sun, 6 Aug 2023 14:38:15 -0400 Subject: [PATCH 02/29] Add APM Trace integration test --- go.mod | 7 +- go.sum | 11 +- terraform/eks/daemon/apm/main.tf | 39 ++++-- test/apm/apm_test.go | 9 +- test/apm/{default_test.go => metrics_test.go} | 14 +-- .../{ => metrics}/client_producer.json | 0 .../{ => metrics}/server_consumer.json | 0 test/apm/resources/traceid_generator.go | 17 +++ test/apm/resources/traces/traces.json | 90 ++++++++++++++ test/apm/traces_test.go | 112 ++++++++++++++++++ util/awsservice/constant.go | 7 +- 11 files changed, 278 insertions(+), 28 deletions(-) rename test/apm/{default_test.go => metrics_test.go} (79%) rename test/apm/resources/{ => metrics}/client_producer.json (100%) rename test/apm/resources/{ => metrics}/server_consumer.json (100%) create mode 100644 test/apm/resources/traceid_generator.go create mode 100644 test/apm/resources/traces/traces.json create mode 100644 test/apm/traces_test.go diff --git a/go.mod b/go.mod index 680f843f8..05051c083 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( collectd.org v0.5.0 github.com/DataDog/datadog-go v4.8.3+incompatible github.com/aws/aws-sdk-go v1.44.262 - github.com/aws/aws-sdk-go-v2 v1.18.0 + github.com/aws/aws-sdk-go-v2 v1.19.1 github.com/aws/aws-sdk-go-v2/config v1.18.10 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 @@ -22,6 +22,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecs v1.23.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.30.1 github.com/aws/aws-sdk-go-v2/service/ssm v1.33.0 + github.com/aws/aws-sdk-go-v2/service/xray v1.16.15 github.com/cenkalti/backoff/v4 v4.2.0 github.com/google/uuid v1.3.0 github.com/mitchellh/mapstructure v1.5.0 @@ -39,8 +40,8 @@ require ( github.com/Microsoft/go-winio v0.6.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 // indirect diff --git a/go.sum b/go.sum index a330665b3..74c317a7b 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,9 @@ github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUY github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.19.1 h1:STs0lbbpXu3byTPcnRLghs2DH0yk9qKDo27TyyJSKsM= +github.com/aws/aws-sdk-go-v2 v1.19.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.10 h1:Znce11DWswdh+5kOsIp+QaNfY9igp1QUN+fZHCKmeCI= @@ -26,14 +27,16 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfI github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 h1:kbk81RlPoC6e4co7cQx2FAvH9TgbzxIqCqiosAFiB+w= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36/go.mod h1:T8Jsn/uNL/AFOXrVYQ1YQaN1r9gN34JU1855/Lyjv+o= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 h1:lMl8S5SB8jNCB+Sty2Em4lnu3IJytceHQd7qbmfqKL0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30/go.mod h1:v3GSCnFxbHzt9dlWBqvA1K1f9lmWuf4ztupZBCAIVs4= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 h1:H/mF2LNWwX00lD6FlYfKpLLZgUW7oIzCBkig78x4Xok= @@ -75,6 +78,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 h1:Jfly6mRxk2ZOSlbCvZfKNS7T github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= github.com/aws/aws-sdk-go-v2/service/sts v1.18.2 h1:J/4wIaGInCEYCGhTSruxCxeoA5cy91a+JT7cHFKFSHQ= github.com/aws/aws-sdk-go-v2/service/sts v1.18.2/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/aws-sdk-go-v2/service/xray v1.16.15 h1:wpFgy/pHHhe+GJ6JX1DiXU7g/FW6E1lFQWVZAhIig3U= +github.com/aws/aws-sdk-go-v2/service/xray v1.16.15/go.mod h1:6n4KNI08QuLLxpmRzr4kHFBiLYrc02OTIz8cnDOm3Dc= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= diff --git a/terraform/eks/daemon/apm/main.tf b/terraform/eks/daemon/apm/main.tf index 6167d48bb..131e966a8 100644 --- a/terraform/eks/daemon/apm/main.tf +++ b/terraform/eks/daemon/apm/main.tf @@ -56,7 +56,8 @@ resource "aws_eks_node_group" "this" { aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy + aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy, + aws_iam_role_policy_attachment.node_AWSXRayDaemonWriteAccess ] } @@ -99,6 +100,10 @@ resource "aws_iam_role_policy_attachment" "node_CloudWatchAgentServerPolicy" { policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" role = aws_iam_role.node_role.name } +resource "aws_iam_role_policy_attachment" "node_AWSXRayDaemonWriteAccess" { + policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess" + role = aws_iam_role.node_role.name +} # TODO: these security groups be created once and then reused # EKS Cluster Security Group @@ -321,22 +326,22 @@ resource "kubernetes_daemonset" "service" { container { name = "apm-client" - image = "public.ecr.aws/amazonlinux/amazonlinux:latest" + image = "public.ecr.aws/docker/library/golang:latest" image_pull_policy = "Always" resources { limits = { "cpu" : "50m", - "memory" : "50Mi" + "memory" : "300Mi" } requests = { "cpu" : "50m", - "memory" : "50Mi" + "memory" : "300Mi" } } command = [ "/bin/sh", "-c", - "while true; do echo '${data.template_file.server_consumer.rendered}' | sed -e \"s/START_TIME/$(date +%s%N)/\" > server_consumer.json; curl -H 'Content-Type: application/json' -d @server_consumer.json -i http://127.0.0.1:4318/v1/metrics --verbose --http0.9; echo '${data.template_file.client_producer.rendered}' | sed -e \"s/START_TIME/$(date +%s%N)/\" > client_producer.json; curl -H 'Content-Type: application/json' -d @client_producer.json -i http://127.0.0.1:4318/v1/metrics --verbose --http0.9; sleep 1; done" + "while true; echo '${data.template_file.traceid_generator.rendered}' > traceid_generator.go && chmod +x traceid_generator.go; export START_TIME=$(date +%s%N); export TRACE_ID=$(go run ./traceid_generator.go); do echo '${data.template_file.server_consumer.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" > server_consumer.json; curl -H 'Content-Type: application/json' -d @server_consumer.json -i http://127.0.0.1:4318/v1/metrics --verbose; echo '${data.template_file.client_producer.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" > client_producer.json; curl -H 'Content-Type: application/json' -d @client_producer.json -i http://127.0.0.1:4318/v1/metrics --verbose; echo '${data.template_file.traces.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" | sed -e \"s/TRACE_ID/$TRACE_ID/\" > traces.json; curl -H 'Content-Type: application/json' -d @traces.json -i http://127.0.0.1:4318/v1/traces --verbose; sleep 1; done" ] env { name = "HOST_IP" @@ -378,9 +383,11 @@ resource "kubernetes_daemonset" "service" { # Template Files ########################################## locals { - cwagent_config = "../../../../${var.test_dir}/resources/config.json" - server_consumer = "../../../../${var.test_dir}/resources/server_consumer.json" - client_producer = "../../../../${var.test_dir}/resources/client_producer.json" + cwagent_config = "../../../../${var.test_dir}/resources/config.json" + server_consumer = "../../../../${var.test_dir}/resources/metrics/server_consumer.json" + client_producer = "../../../../${var.test_dir}/resources/metrics/client_producer.json" + traces = "../../../../${var.test_dir}/resources/traces/traces.json" + traceid_generator = "../../../../${var.test_dir}/resources/traceid_generator.go" } data "template_file" "cwagent_config" { @@ -415,6 +422,18 @@ data "template_file" "client_producer" { } } +data "template_file" "traces" { + template = file(local.traces) + vars = { + } +} + +data "template_file" "traceid_generator" { + template = file(local.traceid_generator) + vars = { + } +} + resource "kubernetes_service_account" "cwagentservice" { depends_on = [kubernetes_namespace.namespace] metadata { @@ -487,9 +506,9 @@ resource "null_resource" "validator" { ] provisioner "local-exec" { command = <<-EOT - echo "Validating EKS metrics/logs for AMF" + echo "Validating EKS metrics/traces for APM" cd ../../../.. - go test ${var.test_dir} -eksClusterName=${aws_eks_cluster.this.name} -computeType=EKS -v -eksDeploymentStrategy=DAEMON + go test ${var.test_dir} -timeout 1h -eksClusterName=${aws_eks_cluster.this.name} -computeType=EKS -v -eksDeploymentStrategy=DAEMON EOT } } diff --git a/test/apm/apm_test.go b/test/apm/apm_test.go index 5e3f34f14..7e991a7b3 100644 --- a/test/apm/apm_test.go +++ b/test/apm/apm_test.go @@ -22,6 +22,7 @@ import ( const ( APMServerConsumerTestName = "APM-Server-Consumer" APMClientProducerTestName = "APM-Client-Producer" + APMTracesTestName = "APM-Traces" ) type APMTestSuite struct { @@ -52,11 +53,15 @@ func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { eksTestRunners = []*test_runner.EKSTestRunner{ { - Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "EKS.Cluster"}, + Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "EKS.Cluster"}, Env: *env, }, { - Runner: &APMRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "EKS.Cluster"}, + Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "EKS.Cluster"}, + Env: *env, + }, + { + Runner: &APMTracesRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMTracesTestName, env.EKSClusterName}, Env: *env, }, } diff --git a/test/apm/default_test.go b/test/apm/metrics_test.go similarity index 79% rename from test/apm/default_test.go rename to test/apm/metrics_test.go index 2b4b46633..1892e9851 100644 --- a/test/apm/default_test.go +++ b/test/apm/metrics_test.go @@ -16,13 +16,13 @@ import ( const testRetryCount = 3 const namespace = "AWS/APM" -type APMRunner struct { +type APMMetricsRunner struct { test_runner.BaseTestRunner testName string dimensionKey string } -func (t *APMRunner) Validate() status.TestGroupResult { +func (t *APMMetricsRunner) Validate() status.TestGroupResult { metricsToFetch := t.GetMeasuredMetrics() testResults := make([]status.TestResult, len(metricsToFetch)) instructions := GetInstructionsFromTestName(t.testName) @@ -45,19 +45,19 @@ func (t *APMRunner) Validate() status.TestGroupResult { } } -func (t *APMRunner) GetTestName() string { +func (t *APMMetricsRunner) GetTestName() string { return t.testName } -func (t *APMRunner) GetAgentRunDuration() time.Duration { +func (t *APMMetricsRunner) GetAgentRunDuration() time.Duration { return 3 * time.Minute } -func (t *APMRunner) GetMeasuredMetrics() []string { +func (t *APMMetricsRunner) GetMeasuredMetrics() []string { return metric.APMMetricNames } -func (e *APMRunner) GetAgentConfigFileName() string { +func (e *APMMetricsRunner) GetAgentConfigFileName() string { return "" } @@ -72,4 +72,4 @@ func GetInstructionsFromTestName(testName string) []dimension.Instruction { } } -var _ test_runner.ITestRunner = (*APMRunner)(nil) +var _ test_runner.ITestRunner = (*APMMetricsRunner)(nil) diff --git a/test/apm/resources/client_producer.json b/test/apm/resources/metrics/client_producer.json similarity index 100% rename from test/apm/resources/client_producer.json rename to test/apm/resources/metrics/client_producer.json diff --git a/test/apm/resources/server_consumer.json b/test/apm/resources/metrics/server_consumer.json similarity index 100% rename from test/apm/resources/server_consumer.json rename to test/apm/resources/metrics/server_consumer.json diff --git a/test/apm/resources/traceid_generator.go b/test/apm/resources/traceid_generator.go new file mode 100644 index 000000000..47740fc06 --- /dev/null +++ b/test/apm/resources/traceid_generator.go @@ -0,0 +1,17 @@ +package main + +import ( + "crypto/rand" + "encoding/binary" + "encoding/hex" + "fmt" + "time" +) + +func main() { + var r [16]byte + epochNow := time.Now().Unix() + binary.BigEndian.PutUint32(r[0:4], uint32(epochNow)) + rand.Read(r[4:]) + fmt.Printf("%s", hex.EncodeToString(r[:])) +} \ No newline at end of file diff --git a/test/apm/resources/traces/traces.json b/test/apm/resources/traces/traces.json new file mode 100644 index 000000000..be2650303 --- /dev/null +++ b/test/apm/resources/traces/traces.json @@ -0,0 +1,90 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "k8s.namespace.name", + "value": { + "stringValue": "default" + } + }, + { + "key": "k8s.pod.name", + "value": { + "stringValue": "pod-name" + } + }, + { + "key": "aws.deployment.name", + "value": { + "stringValue": "deployment-name" + } + }, + { + "key": "host.id", + "value": { + "stringValue": "i-00000000000000000" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "apm-integration-test" + }, + "spans": [ + { + "traceId": "TRACE_ID", + "spanId": "EEE19B7EC3C1B174", + "parentSpanId": "EEE19B7EC3C1B173", + "name": "apm-integration-test-traces", + "startTimeUnixNano": START_TIME, + "endTimeUnixNano": START_TIME, + "kind": 2, + "attributes": [ + { + "key": "aws.span.kind", + "value": { + "stringValue": "CLIENT" + } + }, + { + "key": "aws.local.operation", + "value": { + "stringValue": "operation" + } + }, + { + "key": "aws.local.service", + "value": { + "stringValue": "service-name" + } + }, + { + "key": "aws.remote.operation", + "value": { + "stringValue": "remote-operation" + } + }, + { + "key": "aws.remote.service", + "value": { + "stringValue": "service-name-remote" + } + }, + { + "key": "aws.remote.target", + "value": { + "stringValue": "remote-target" + } + } + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/test/apm/traces_test.go b/test/apm/traces_test.go new file mode 100644 index 000000000..1ecbd7c39 --- /dev/null +++ b/test/apm/traces_test.go @@ -0,0 +1,112 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package apm + +import ( + "context" + "encoding/json" + "fmt" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/xray" + "time" +) + +const ( + lookbackDuration = time.Duration(-80) * time.Minute + EKSClusterAnnotation = "EKS_Cluster" +) + + +var annotations = map[string]string { + "aws_remote_target": "remote-target", + "aws_remote_operation": "remote-operation", + "aws_local_service": "service-name", + "aws_remote_service": "service-name-remote", + "K8s_Namespace": "default", + "aws_local_operation": "operation", +} + +type APMTracesRunner struct { + test_runner.BaseTestRunner + testName string + clusterName string +} + +func (t *APMTracesRunner) Validate() status.TestGroupResult { + testResults := []status.TestResult{{ + Name: t.testName, + Status: status.FAILED, + }} + timeNow := time.Now() + + annotations[EKSClusterAnnotation] = t.clusterName + xrayFilter := FilterExpression(annotations) + traceIds, err := GetTraceIDs(timeNow.Add(lookbackDuration),timeNow, xrayFilter) + if err != nil { + fmt.Printf("error getting trace ids: %v", err) + } else { + fmt.Printf("Trace IDs: %v\n", traceIds) + if len(traceIds) > 0 { + fmt.Println("Successful") + testResults[0].Status = status.SUCCESSFUL + } + } + + return status.TestGroupResult{ + Name: t.GetTestName(), + TestResults: testResults, + } +} + +func (t *APMTracesRunner) GetTestName() string { + return t.testName +} + +func (t *APMTracesRunner) GetAgentRunDuration() time.Duration { + return 3 * time.Minute +} + +func (t *APMTracesRunner) GetMeasuredMetrics() []string { + return nil +} + +func (e *APMTracesRunner) GetAgentConfigFileName() string { + return "" +} + +func GetTraceIDs(startTime time.Time, endTime time.Time, filter string) ([]string, error) { + var traceIDs []string + input := &xray.GetTraceSummariesInput{StartTime: aws.Time(startTime), EndTime: aws.Time(endTime), FilterExpression: aws.String(filter)} + output, err := awsservice.XrayClient.GetTraceSummaries(context.Background(), input) + if err != nil { + return nil, err + } + for _, summary := range output.TraceSummaries { + traceIDs = append(traceIDs, *summary.Id) + } + return traceIDs, nil +} + +func FilterExpression(annotations map[string]string) string { + var expression string + for key, value := range annotations { + fmt.Println(value) + result, err := json.Marshal(value) + if err != nil { + continue + } + if len(expression) != 0 { + expression += " AND " + } + expression += fmt.Sprintf("annotation.%s = %s", key, result) + } + return expression +} + +var _ test_runner.ITestRunner = (*APMTracesRunner)(nil) diff --git a/util/awsservice/constant.go b/util/awsservice/constant.go index a0ce572b0..82b773488 100644 --- a/util/awsservice/constant.go +++ b/util/awsservice/constant.go @@ -5,11 +5,9 @@ package awsservice import ( "context" - "github.com/aws/aws-sdk-go-v2/service/cloudformation" - "time" - "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/aws/aws-sdk-go-v2/service/dynamodb" @@ -17,7 +15,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/xray" backoff "github.com/cenkalti/backoff/v4" + "time" ) const ( @@ -48,4 +48,5 @@ var ( DynamodbClient = dynamodb.NewFromConfig(awsCfg) S3Client = s3.NewFromConfig(awsCfg) CloudformationClient = cloudformation.NewFromConfig(awsCfg) + XrayClient = xray.NewFromConfig(awsCfg) ) From 88a63c17fde7498130877c80fb3a5c7e6c8597d8 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Sun, 6 Aug 2023 16:33:32 -0400 Subject: [PATCH 03/29] Remove debug print statements --- test/apm/traces_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/apm/traces_test.go b/test/apm/traces_test.go index 1ecbd7c39..cb2d545d1 100644 --- a/test/apm/traces_test.go +++ b/test/apm/traces_test.go @@ -53,7 +53,6 @@ func (t *APMTracesRunner) Validate() status.TestGroupResult { } else { fmt.Printf("Trace IDs: %v\n", traceIds) if len(traceIds) > 0 { - fmt.Println("Successful") testResults[0].Status = status.SUCCESSFUL } } @@ -96,7 +95,6 @@ func GetTraceIDs(startTime time.Time, endTime time.Time, filter string) ([]strin func FilterExpression(annotations map[string]string) string { var expression string for key, value := range annotations { - fmt.Println(value) result, err := json.Marshal(value) if err != nil { continue From ff6e726b03a33442d8f16cee97d4b196b4ae486e Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Mon, 7 Aug 2023 13:37:41 -0400 Subject: [PATCH 04/29] Fix formatting --- util/awsservice/constant.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/awsservice/constant.go b/util/awsservice/constant.go index 82b773488..65f765c2d 100644 --- a/util/awsservice/constant.go +++ b/util/awsservice/constant.go @@ -48,5 +48,5 @@ var ( DynamodbClient = dynamodb.NewFromConfig(awsCfg) S3Client = s3.NewFromConfig(awsCfg) CloudformationClient = cloudformation.NewFromConfig(awsCfg) - XrayClient = xray.NewFromConfig(awsCfg) + XrayClient = xray.NewFromConfig(awsCfg) ) From 1af68f82a0b3e139d170dad7b2212f3d80d49d20 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 24 Oct 2023 07:43:55 -0400 Subject: [PATCH 05/29] Fixed the dimensions --- test/metric/apm_util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go index 4d41aa8f2..6dbc379ef 100644 --- a/test/metric/apm_util.go +++ b/test/metric/apm_util.go @@ -20,11 +20,11 @@ var ( ServerConsumerInstructions = []dimension.Instruction{ { - Key: "EKS.Cluster", + Key: "HostedIn.EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "K8s.Namespace", + Key: "HostedIn.K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { @@ -92,4 +92,4 @@ func ValidateAPMMetric(dimFactory dimension.Factory, namespace string, metricNam testResult.Status = status.SUCCESSFUL return testResult -} \ No newline at end of file +} From e942c2cc4aec54db1ee6a3e657b5e31aaccbe71c Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 24 Oct 2023 10:56:27 -0400 Subject: [PATCH 06/29] testing apm --- test/metric/apm_util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go index 6dbc379ef..16c7e06b4 100644 --- a/test/metric/apm_util.go +++ b/test/metric/apm_util.go @@ -39,11 +39,11 @@ var ( ClientProducerInstructions = []dimension.Instruction{ { - Key: "EKS.Cluster", + Key: "HostedIn.EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "K8s.Namespace", + Key: "HostedIn.K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { From ddfaa4f577aa0e9d803c6027952f9401e3551934 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 24 Oct 2023 15:13:59 -0400 Subject: [PATCH 07/29] reverted the hostedIn and changed the port --- terraform/eks/daemon/apm/main.tf | 2 +- test/metric/apm_util.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/eks/daemon/apm/main.tf b/terraform/eks/daemon/apm/main.tf index 131e966a8..1759c4f58 100644 --- a/terraform/eks/daemon/apm/main.tf +++ b/terraform/eks/daemon/apm/main.tf @@ -341,7 +341,7 @@ resource "kubernetes_daemonset" "service" { command = [ "/bin/sh", "-c", - "while true; echo '${data.template_file.traceid_generator.rendered}' > traceid_generator.go && chmod +x traceid_generator.go; export START_TIME=$(date +%s%N); export TRACE_ID=$(go run ./traceid_generator.go); do echo '${data.template_file.server_consumer.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" > server_consumer.json; curl -H 'Content-Type: application/json' -d @server_consumer.json -i http://127.0.0.1:4318/v1/metrics --verbose; echo '${data.template_file.client_producer.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" > client_producer.json; curl -H 'Content-Type: application/json' -d @client_producer.json -i http://127.0.0.1:4318/v1/metrics --verbose; echo '${data.template_file.traces.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" | sed -e \"s/TRACE_ID/$TRACE_ID/\" > traces.json; curl -H 'Content-Type: application/json' -d @traces.json -i http://127.0.0.1:4318/v1/traces --verbose; sleep 1; done" + "while true; echo '${data.template_file.traceid_generator.rendered}' > traceid_generator.go && chmod +x traceid_generator.go; export START_TIME=$(date +%s%N); export TRACE_ID=$(go run ./traceid_generator.go); do echo '${data.template_file.server_consumer.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" > server_consumer.json; curl -H 'Content-Type: application/json' -d @server_consumer.json -i http://127.0.0.1:4316/v1/metrics --verbose; echo '${data.template_file.client_producer.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" > client_producer.json; curl -H 'Content-Type: application/json' -d @client_producer.json -i http://127.0.0.1:4316/v1/metrics --verbose; echo '${data.template_file.traces.rendered}' | sed -e \"s/START_TIME/$START_TIME/\" | sed -e \"s/TRACE_ID/$TRACE_ID/\" > traces.json; curl -H 'Content-Type: application/json' -d @traces.json -i http://127.0.0.1:4316/v1/traces --verbose; sleep 1; done" ] env { name = "HOST_IP" diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go index 16c7e06b4..ed5164ff0 100644 --- a/test/metric/apm_util.go +++ b/test/metric/apm_util.go @@ -20,11 +20,11 @@ var ( ServerConsumerInstructions = []dimension.Instruction{ { - Key: "HostedIn.EKS.Cluster", + Key: "EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "HostedIn.K8s.Namespace", + Key: "K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { @@ -39,11 +39,11 @@ var ( ClientProducerInstructions = []dimension.Instruction{ { - Key: "HostedIn.EKS.Cluster", + Key: "EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "HostedIn.K8s.Namespace", + Key: "K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { From 11e2a507ed4b2c3f575eba459eb651bca59665f4 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 24 Oct 2023 20:37:08 -0400 Subject: [PATCH 08/29] changed the dimensions and ports --- terraform/eks/daemon/apm/main.tf | 6 +++--- test/apm/apm_test.go | 6 +++--- test/metric/apm_util.go | 8 ++++---- test/metric/dimension/instanceid_provider.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/eks/daemon/apm/main.tf b/terraform/eks/daemon/apm/main.tf index 1759c4f58..af79c5d90 100644 --- a/terraform/eks/daemon/apm/main.tf +++ b/terraform/eks/daemon/apm/main.tf @@ -218,8 +218,8 @@ resource "kubernetes_daemonset" "service" { } } port { - container_port = 4318 - host_port = 4318 + container_port = 4316 + host_port = 4316 protocol = "TCP" } env { @@ -349,7 +349,7 @@ resource "kubernetes_daemonset" "service" { field_ref { field_path = "status.hostIP" } - } + }Ï } env { name = "HOST_NAME" diff --git a/test/apm/apm_test.go b/test/apm/apm_test.go index 7e991a7b3..c788e0913 100644 --- a/test/apm/apm_test.go +++ b/test/apm/apm_test.go @@ -22,7 +22,7 @@ import ( const ( APMServerConsumerTestName = "APM-Server-Consumer" APMClientProducerTestName = "APM-Client-Producer" - APMTracesTestName = "APM-Traces" + APMTracesTestName = "APM-Traces" ) type APMTestSuite struct { @@ -53,11 +53,11 @@ func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { eksTestRunners = []*test_runner.EKSTestRunner{ { - Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "EKS.Cluster"}, + Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "HostedIn.EKS.Cluster"}, Env: *env, }, { - Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "EKS.Cluster"}, + Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "HostedIn.EKS.Cluster"}, Env: *env, }, { diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go index ed5164ff0..16c7e06b4 100644 --- a/test/metric/apm_util.go +++ b/test/metric/apm_util.go @@ -20,11 +20,11 @@ var ( ServerConsumerInstructions = []dimension.Instruction{ { - Key: "EKS.Cluster", + Key: "HostedIn.EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "K8s.Namespace", + Key: "HostedIn.K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { @@ -39,11 +39,11 @@ var ( ClientProducerInstructions = []dimension.Instruction{ { - Key: "EKS.Cluster", + Key: "HostedIn.EKS.Cluster", Value: dimension.UnknownDimensionValue(), }, { - Key: "K8s.Namespace", + Key: "HostedIn.K8s.Namespace", Value: dimension.ExpectedDimensionValue{Value: aws.String("default")}, }, { diff --git a/test/metric/dimension/instanceid_provider.go b/test/metric/dimension/instanceid_provider.go index 0e2819d78..abaa51304 100644 --- a/test/metric/dimension/instanceid_provider.go +++ b/test/metric/dimension/instanceid_provider.go @@ -80,9 +80,9 @@ func (p *EKSClusterNameProvider) IsApplicable() bool { func (p *EKSClusterNameProvider) GetDimension(instruction Instruction) types.Dimension { // For APM metrics, cluster name is under EKS.Cluster dimension - if instruction.Key == "EKS.Cluster" { + if instruction.Key == "HostedIn.EKS.Cluster" { return types.Dimension{ - Name: aws.String("EKS.Cluster"), + Name: aws.String("HostedIn.EKS.Cluster"), Value: aws.String(p.env.EKSClusterName), } } From df633340304ef3338cf7e4ca361eb7193e1586b5 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 24 Oct 2023 20:58:17 -0400 Subject: [PATCH 09/29] fixed terraform bug --- terraform/eks/daemon/apm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/eks/daemon/apm/main.tf b/terraform/eks/daemon/apm/main.tf index af79c5d90..5fcbb5800 100644 --- a/terraform/eks/daemon/apm/main.tf +++ b/terraform/eks/daemon/apm/main.tf @@ -349,7 +349,7 @@ resource "kubernetes_daemonset" "service" { field_ref { field_path = "status.hostIP" } - }Ï + } } env { name = "HOST_NAME" From 31f005dfe32124d70372f5e0db712171d6442ce7 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 24 Oct 2023 21:03:09 -0400 Subject: [PATCH 10/29] removed other tests --- generator/test_case_generator.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index af13f06cc..b97389522 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -145,23 +145,6 @@ var testTypeToTestConfig = map[string][]testConfig{ {testDir: "./test/emf"}, }, "eks_daemon": { - { - testDir: "./test/metric_value_benchmark", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, {testDir: "./test/apm", terraformDir: "terraform/eks/daemon/apm", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, From 0aabd78cd784bcce3061739a8ea8d99ee75dc97e Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 11:18:01 -0400 Subject: [PATCH 11/29] changed the client and server jsons --- test/apm/resources/metrics/client_producer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/apm/resources/metrics/client_producer.json b/test/apm/resources/metrics/client_producer.json index dc473cbc8..8a39c60d6 100644 --- a/test/apm/resources/metrics/client_producer.json +++ b/test/apm/resources/metrics/client_producer.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "service-name" + "stringValue": "deployment-name" } }, { From f92f866c56d2115a3c75c61537b2e731bc2b33a6 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 12:21:31 -0400 Subject: [PATCH 12/29] changed the client and server jsons --- test/apm/resources/metrics/client_producer.json | 2 +- test/apm/resources/metrics/server_consumer.json | 4 ++-- test/apm/resources/traces/traces.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/apm/resources/metrics/client_producer.json b/test/apm/resources/metrics/client_producer.json index 8a39c60d6..dc473cbc8 100644 --- a/test/apm/resources/metrics/client_producer.json +++ b/test/apm/resources/metrics/client_producer.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "deployment-name" + "stringValue": "service-name" } }, { diff --git a/test/apm/resources/metrics/server_consumer.json b/test/apm/resources/metrics/server_consumer.json index 8e3e263de..8284fee92 100644 --- a/test/apm/resources/metrics/server_consumer.json +++ b/test/apm/resources/metrics/server_consumer.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "deployment-name" + "stringValue": "service-name" } }, { @@ -162,7 +162,7 @@ { "key": "aws.span.kind", "value": { - "stringValue": "CONSUMER" + "stringValue": "SERVER" } }, { diff --git a/test/apm/resources/traces/traces.json b/test/apm/resources/traces/traces.json index be2650303..31a649ca2 100644 --- a/test/apm/resources/traces/traces.json +++ b/test/apm/resources/traces/traces.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "deployment-name" + "stringValue": "service-name" } }, { From ee1a46523a5c72225f4e6828393b575aaa496805 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 13:03:00 -0400 Subject: [PATCH 13/29] increased retry count --- test/apm/metrics_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/apm/metrics_test.go b/test/apm/metrics_test.go index 1892e9851..03660cbbe 100644 --- a/test/apm/metrics_test.go +++ b/test/apm/metrics_test.go @@ -13,7 +13,7 @@ import ( "time" ) -const testRetryCount = 3 +const testRetryCount = 5 const namespace = "AWS/APM" type APMMetricsRunner struct { @@ -62,7 +62,7 @@ func (e *APMMetricsRunner) GetAgentConfigFileName() string { } func GetInstructionsFromTestName(testName string) []dimension.Instruction { - switch testName{ + switch testName { case APMClientProducerTestName: return metric.ClientProducerInstructions case APMServerConsumerTestName: From e309dab47fed59f80c1349891699cc8b1925509f Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 13:03:06 -0400 Subject: [PATCH 14/29] increased retry count --- test/apm/metrics_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/apm/metrics_test.go b/test/apm/metrics_test.go index 03660cbbe..cf5aac3c8 100644 --- a/test/apm/metrics_test.go +++ b/test/apm/metrics_test.go @@ -34,7 +34,7 @@ func (t *APMMetricsRunner) Validate() status.TestGroupResult { if testResult.Status == status.SUCCESSFUL { break } - time.Sleep(15 * time.Second) + time.Sleep(20 * time.Second) } testResults[i] = testResult } From da2f6e703fc9fc1e5e513e887d72d1357ac675b3 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 13:50:27 -0400 Subject: [PATCH 15/29] increased retry count --- test/apm/metrics_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/apm/metrics_test.go b/test/apm/metrics_test.go index cf5aac3c8..c1fecccc5 100644 --- a/test/apm/metrics_test.go +++ b/test/apm/metrics_test.go @@ -13,7 +13,7 @@ import ( "time" ) -const testRetryCount = 5 +const testRetryCount = 6 const namespace = "AWS/APM" type APMMetricsRunner struct { @@ -34,7 +34,7 @@ func (t *APMMetricsRunner) Validate() status.TestGroupResult { if testResult.Status == status.SUCCESSFUL { break } - time.Sleep(20 * time.Second) + time.Sleep(30 * time.Second) } testResults[i] = testResult } From f0b6d7c24b801b434d00bf72c980267d71442dd0 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 14:32:43 -0400 Subject: [PATCH 16/29] traces changes cluster annotation --- test/apm/traces_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/apm/traces_test.go b/test/apm/traces_test.go index cb2d545d1..5dc284530 100644 --- a/test/apm/traces_test.go +++ b/test/apm/traces_test.go @@ -18,24 +18,23 @@ import ( ) const ( - lookbackDuration = time.Duration(-80) * time.Minute - EKSClusterAnnotation = "EKS_Cluster" + lookbackDuration = time.Duration(-80) * time.Minute + EKSClusterAnnotation = "HostedIn_EKS_Cluster" ) - -var annotations = map[string]string { - "aws_remote_target": "remote-target", +var annotations = map[string]string{ + "aws_remote_target": "remote-target", "aws_remote_operation": "remote-operation", - "aws_local_service": "service-name", - "aws_remote_service": "service-name-remote", - "K8s_Namespace": "default", - "aws_local_operation": "operation", + "aws_local_service": "service-name", + "aws_remote_service": "service-name-remote", + "K8s_Namespace": "default", + "aws_local_operation": "operation", } type APMTracesRunner struct { test_runner.BaseTestRunner - testName string - clusterName string + testName string + clusterName string } func (t *APMTracesRunner) Validate() status.TestGroupResult { @@ -44,10 +43,10 @@ func (t *APMTracesRunner) Validate() status.TestGroupResult { Status: status.FAILED, }} timeNow := time.Now() - + fmt.Printf("CLuster name: %v\n", t.clusterName) annotations[EKSClusterAnnotation] = t.clusterName xrayFilter := FilterExpression(annotations) - traceIds, err := GetTraceIDs(timeNow.Add(lookbackDuration),timeNow, xrayFilter) + traceIds, err := GetTraceIDs(timeNow.Add(lookbackDuration), timeNow, xrayFilter) if err != nil { fmt.Printf("error getting trace ids: %v", err) } else { @@ -103,6 +102,7 @@ func FilterExpression(annotations map[string]string) string { expression += " AND " } expression += fmt.Sprintf("annotation.%s = %s", key, result) + fmt.Printf("annotation.%s = %s", key, result) } return expression } From 9f55f2e6f434d96dfae2774fc2f4b226ba65932a Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 15:04:21 -0400 Subject: [PATCH 17/29] changes namespace --- test/apm/traces_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/apm/traces_test.go b/test/apm/traces_test.go index 5dc284530..c3dc9285a 100644 --- a/test/apm/traces_test.go +++ b/test/apm/traces_test.go @@ -23,12 +23,12 @@ const ( ) var annotations = map[string]string{ - "aws_remote_target": "remote-target", - "aws_remote_operation": "remote-operation", - "aws_local_service": "service-name", - "aws_remote_service": "service-name-remote", - "K8s_Namespace": "default", - "aws_local_operation": "operation", + "aws_remote_target": "remote-target", + "aws_remote_operation": "remote-operation", + "aws_local_service": "service-name", + "aws_remote_service": "service-name-remote", + "HostedIn_K8s_Namespace": "default", + "aws_local_operation": "operation", } type APMTracesRunner struct { From 74a90ae1bba965adfe26f1b59bc31bf4a4eb7eb9 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 15:52:25 -0400 Subject: [PATCH 18/29] reverted few changes --- generator/test_case_generator.go | 17 +++++++++++++++++ test/apm/resources/metrics/client_producer.json | 2 +- test/apm/resources/metrics/server_consumer.json | 2 +- test/apm/resources/traces/traces.json | 2 +- test/apm/traces_test.go | 2 -- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index b97389522..af13f06cc 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -145,6 +145,23 @@ var testTypeToTestConfig = map[string][]testConfig{ {testDir: "./test/emf"}, }, "eks_daemon": { + { + testDir: "./test/metric_value_benchmark", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, {testDir: "./test/apm", terraformDir: "terraform/eks/daemon/apm", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, diff --git a/test/apm/resources/metrics/client_producer.json b/test/apm/resources/metrics/client_producer.json index dc473cbc8..8a39c60d6 100644 --- a/test/apm/resources/metrics/client_producer.json +++ b/test/apm/resources/metrics/client_producer.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "service-name" + "stringValue": "deployment-name" } }, { diff --git a/test/apm/resources/metrics/server_consumer.json b/test/apm/resources/metrics/server_consumer.json index 8284fee92..6748e6fd0 100644 --- a/test/apm/resources/metrics/server_consumer.json +++ b/test/apm/resources/metrics/server_consumer.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "service-name" + "stringValue": "deployment-name" } }, { diff --git a/test/apm/resources/traces/traces.json b/test/apm/resources/traces/traces.json index 31a649ca2..be2650303 100644 --- a/test/apm/resources/traces/traces.json +++ b/test/apm/resources/traces/traces.json @@ -18,7 +18,7 @@ { "key": "aws.deployment.name", "value": { - "stringValue": "service-name" + "stringValue": "deployment-name" } }, { diff --git a/test/apm/traces_test.go b/test/apm/traces_test.go index c3dc9285a..77d7f0373 100644 --- a/test/apm/traces_test.go +++ b/test/apm/traces_test.go @@ -43,7 +43,6 @@ func (t *APMTracesRunner) Validate() status.TestGroupResult { Status: status.FAILED, }} timeNow := time.Now() - fmt.Printf("CLuster name: %v\n", t.clusterName) annotations[EKSClusterAnnotation] = t.clusterName xrayFilter := FilterExpression(annotations) traceIds, err := GetTraceIDs(timeNow.Add(lookbackDuration), timeNow, xrayFilter) @@ -102,7 +101,6 @@ func FilterExpression(annotations map[string]string) string { expression += " AND " } expression += fmt.Sprintf("annotation.%s = %s", key, result) - fmt.Printf("annotation.%s = %s", key, result) } return expression } From 42f49d59e02e7b08ef5660e55c22175b6fb2d4e0 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 17:01:37 -0400 Subject: [PATCH 19/29] fixed linting errors --- test/apm/apm_test.go | 2 +- test/apm/metrics_test.go | 3 ++- test/apm/resources/traceid_generator.go | 3 +++ test/apm/traces_test.go | 8 +++++--- test/metric/apm_util.go | 3 ++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/test/apm/apm_test.go b/test/apm/apm_test.go index c788e0913..d1bbcd442 100644 --- a/test/apm/apm_test.go +++ b/test/apm/apm_test.go @@ -7,13 +7,13 @@ package apm import ( "fmt" - "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" "log" "testing" "github.com/stretchr/testify/suite" "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/environment/computetype" "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" diff --git a/test/apm/metrics_test.go b/test/apm/metrics_test.go index c1fecccc5..0d239eae3 100644 --- a/test/apm/metrics_test.go +++ b/test/apm/metrics_test.go @@ -6,11 +6,12 @@ package apm import ( + "time" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" - "time" ) const testRetryCount = 6 diff --git a/test/apm/resources/traceid_generator.go b/test/apm/resources/traceid_generator.go index 47740fc06..a56fde0f8 100644 --- a/test/apm/resources/traceid_generator.go +++ b/test/apm/resources/traceid_generator.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( diff --git a/test/apm/traces_test.go b/test/apm/traces_test.go index 77d7f0373..d9f059b8d 100644 --- a/test/apm/traces_test.go +++ b/test/apm/traces_test.go @@ -9,12 +9,14 @@ import ( "context" "encoding/json" "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/xray" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/xray" - "time" ) const ( diff --git a/test/metric/apm_util.go b/test/metric/apm_util.go index 16c7e06b4..5520841f5 100644 --- a/test/metric/apm_util.go +++ b/test/metric/apm_util.go @@ -6,9 +6,10 @@ package metric import ( + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/status" - "github.com/aws/aws-sdk-go-v2/aws" ) var ( From 169dde1b22f81a292f9761e9796972059f1f848d Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 17:22:04 -0400 Subject: [PATCH 20/29] changed to App signals --- generator/test_case_generator.go | 2 +- terraform/eks/daemon/{apm => app_signals}/main.tf | 2 +- terraform/eks/daemon/{apm => app_signals}/providers.tf | 0 terraform/eks/daemon/{apm => app_signals}/variables.tf | 2 +- test/{apm/apm_test.go => app_signals/app_signals_test.go} | 2 +- test/{apm => app_signals}/metrics_test.go | 2 +- test/{apm => app_signals}/resources/config.json | 4 ++-- .../resources/metrics/client_producer.json | 0 .../resources/metrics/server_consumer.json | 0 test/{apm => app_signals}/resources/traceid_generator.go | 0 test/{apm => app_signals}/resources/traces/traces.json | 4 ++-- test/{apm => app_signals}/traces_test.go | 2 +- 12 files changed, 10 insertions(+), 10 deletions(-) rename terraform/eks/daemon/{apm => app_signals}/main.tf (99%) rename terraform/eks/daemon/{apm => app_signals}/providers.tf (100%) rename terraform/eks/daemon/{apm => app_signals}/variables.tf (94%) rename test/{apm/apm_test.go => app_signals/app_signals_test.go} (99%) rename test/{apm => app_signals}/metrics_test.go (98%) rename test/{apm => app_signals}/resources/config.json (81%) rename test/{apm => app_signals}/resources/metrics/client_producer.json (100%) rename test/{apm => app_signals}/resources/metrics/server_consumer.json (100%) rename test/{apm => app_signals}/resources/traceid_generator.go (100%) rename test/{apm => app_signals}/resources/traces/traces.json (95%) rename test/{apm => app_signals}/traces_test.go (99%) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index a9d532d20..655d1c8ee 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -187,7 +187,7 @@ var testTypeToTestConfig = map[string][]testConfig{ targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, - {testDir: "./test/apm", terraformDir: "terraform/eks/daemon/apm", + {testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, }, diff --git a/terraform/eks/daemon/apm/main.tf b/terraform/eks/daemon/app_signals/main.tf similarity index 99% rename from terraform/eks/daemon/apm/main.tf rename to terraform/eks/daemon/app_signals/main.tf index 5fcbb5800..d2d42a67c 100644 --- a/terraform/eks/daemon/apm/main.tf +++ b/terraform/eks/daemon/app_signals/main.tf @@ -325,7 +325,7 @@ resource "kubernetes_daemonset" "service" { } container { - name = "apm-client" + name = "app_signals-client" image = "public.ecr.aws/docker/library/golang:latest" image_pull_policy = "Always" resources { diff --git a/terraform/eks/daemon/apm/providers.tf b/terraform/eks/daemon/app_signals/providers.tf similarity index 100% rename from terraform/eks/daemon/apm/providers.tf rename to terraform/eks/daemon/app_signals/providers.tf diff --git a/terraform/eks/daemon/apm/variables.tf b/terraform/eks/daemon/app_signals/variables.tf similarity index 94% rename from terraform/eks/daemon/apm/variables.tf rename to terraform/eks/daemon/app_signals/variables.tf index aacb0eaa1..fb73ec3d6 100644 --- a/terraform/eks/daemon/apm/variables.tf +++ b/terraform/eks/daemon/app_signals/variables.tf @@ -8,7 +8,7 @@ variable "region" { variable "test_dir" { type = string - default = "./test/apm" + default = "./test/app_signals" } variable "cwagent_image_repo" { diff --git a/test/apm/apm_test.go b/test/app_signals/app_signals_test.go similarity index 99% rename from test/apm/apm_test.go rename to test/app_signals/app_signals_test.go index d1bbcd442..418156e95 100644 --- a/test/apm/apm_test.go +++ b/test/app_signals/app_signals_test.go @@ -3,7 +3,7 @@ //go:build !windows -package apm +package app_signals import ( "fmt" diff --git a/test/apm/metrics_test.go b/test/app_signals/metrics_test.go similarity index 98% rename from test/apm/metrics_test.go rename to test/app_signals/metrics_test.go index 0d239eae3..a6b348b42 100644 --- a/test/apm/metrics_test.go +++ b/test/app_signals/metrics_test.go @@ -3,7 +3,7 @@ //go:build !windows -package apm +package app_signals import ( "time" diff --git a/test/apm/resources/config.json b/test/app_signals/resources/config.json similarity index 81% rename from test/apm/resources/config.json rename to test/app_signals/resources/config.json index fbaf609a9..9aa23da0c 100644 --- a/test/apm/resources/config.json +++ b/test/app_signals/resources/config.json @@ -4,14 +4,14 @@ }, "logs": { "metrics_collected": { - "apm": { + "app_signals": { "enabled": true } } }, "traces": { "traces_collected": { - "apm": { + "app_signals": { "enabled": true } } diff --git a/test/apm/resources/metrics/client_producer.json b/test/app_signals/resources/metrics/client_producer.json similarity index 100% rename from test/apm/resources/metrics/client_producer.json rename to test/app_signals/resources/metrics/client_producer.json diff --git a/test/apm/resources/metrics/server_consumer.json b/test/app_signals/resources/metrics/server_consumer.json similarity index 100% rename from test/apm/resources/metrics/server_consumer.json rename to test/app_signals/resources/metrics/server_consumer.json diff --git a/test/apm/resources/traceid_generator.go b/test/app_signals/resources/traceid_generator.go similarity index 100% rename from test/apm/resources/traceid_generator.go rename to test/app_signals/resources/traceid_generator.go diff --git a/test/apm/resources/traces/traces.json b/test/app_signals/resources/traces/traces.json similarity index 95% rename from test/apm/resources/traces/traces.json rename to test/app_signals/resources/traces/traces.json index be2650303..1d6fdf61e 100644 --- a/test/apm/resources/traces/traces.json +++ b/test/app_signals/resources/traces/traces.json @@ -32,14 +32,14 @@ "scopeSpans": [ { "scope": { - "name": "apm-integration-test" + "name": "app_signals-integration-test" }, "spans": [ { "traceId": "TRACE_ID", "spanId": "EEE19B7EC3C1B174", "parentSpanId": "EEE19B7EC3C1B173", - "name": "apm-integration-test-traces", + "name": "app_signals-integration-test-traces", "startTimeUnixNano": START_TIME, "endTimeUnixNano": START_TIME, "kind": 2, diff --git a/test/apm/traces_test.go b/test/app_signals/traces_test.go similarity index 99% rename from test/apm/traces_test.go rename to test/app_signals/traces_test.go index d9f059b8d..2b3729c28 100644 --- a/test/apm/traces_test.go +++ b/test/app_signals/traces_test.go @@ -3,7 +3,7 @@ //go:build !windows -package apm +package app_signals import ( "context" From a9764ed12cf575d665012664837a6a6e43a6830a Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 17:41:16 -0400 Subject: [PATCH 21/29] fixed issue with containers --- terraform/eks/daemon/app_signals/main.tf | 2 +- test/app_signals/resources/traces/traces.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/eks/daemon/app_signals/main.tf b/terraform/eks/daemon/app_signals/main.tf index d2d42a67c..957cd9f6a 100644 --- a/terraform/eks/daemon/app_signals/main.tf +++ b/terraform/eks/daemon/app_signals/main.tf @@ -325,7 +325,7 @@ resource "kubernetes_daemonset" "service" { } container { - name = "app_signals-client" + name = "app-signals-client" image = "public.ecr.aws/docker/library/golang:latest" image_pull_policy = "Always" resources { diff --git a/test/app_signals/resources/traces/traces.json b/test/app_signals/resources/traces/traces.json index 1d6fdf61e..b809f1dd7 100644 --- a/test/app_signals/resources/traces/traces.json +++ b/test/app_signals/resources/traces/traces.json @@ -32,14 +32,14 @@ "scopeSpans": [ { "scope": { - "name": "app_signals-integration-test" + "name": "app-signals-integration-test" }, "spans": [ { "traceId": "TRACE_ID", "spanId": "EEE19B7EC3C1B174", "parentSpanId": "EEE19B7EC3C1B173", - "name": "app_signals-integration-test-traces", + "name": "app-signals-integration-test-traces", "startTimeUnixNano": START_TIME, "endTimeUnixNano": START_TIME, "kind": 2, From 64e33a6bafa9df8e0e20a7697ac11827ab43a5d2 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 18:02:19 -0400 Subject: [PATCH 22/29] changed APM to AppSignals --- terraform/eks/daemon/app_signals/main.tf | 2 +- test/app_signals/app_signals_test.go | 32 +++++++++---------- test/app_signals/metrics_test.go | 24 +++++++------- test/app_signals/traces_test.go | 14 ++++---- .../{apm_util.go => app_signals_util.go} | 4 +-- test/metric/dimension/instanceid_provider.go | 2 +- 6 files changed, 39 insertions(+), 39 deletions(-) rename test/metric/{apm_util.go => app_signals_util.go} (92%) diff --git a/terraform/eks/daemon/app_signals/main.tf b/terraform/eks/daemon/app_signals/main.tf index 957cd9f6a..56aba8a37 100644 --- a/terraform/eks/daemon/app_signals/main.tf +++ b/terraform/eks/daemon/app_signals/main.tf @@ -506,7 +506,7 @@ resource "null_resource" "validator" { ] provisioner "local-exec" { command = <<-EOT - echo "Validating EKS metrics/traces for APM" + echo "Validating EKS metrics/traces for AppSignals" cd ../../../.. go test ${var.test_dir} -timeout 1h -eksClusterName=${aws_eks_cluster.this.name} -computeType=EKS -v -eksDeploymentStrategy=DAEMON EOT diff --git a/test/app_signals/app_signals_test.go b/test/app_signals/app_signals_test.go index 418156e95..01e9976b8 100644 --- a/test/app_signals/app_signals_test.go +++ b/test/app_signals/app_signals_test.go @@ -20,23 +20,23 @@ import ( ) const ( - APMServerConsumerTestName = "APM-Server-Consumer" - APMClientProducerTestName = "APM-Client-Producer" - APMTracesTestName = "APM-Traces" + AppSignalsServerConsumerTestName = "AppSignals-Server-Consumer" + AppSignalsClientProducerTestName = "AppSignals-Client-Producer" + AppSignalsTracesTestName = "AppSignals-Traces" ) -type APMTestSuite struct { +type AppSignalsTestSuite struct { suite.Suite test_runner.TestSuite } -func (suite *APMTestSuite) SetupSuite() { - fmt.Println(">>>> Starting APMTestSuite") +func (suite *AppSignalsTestSuite) SetupSuite() { + fmt.Println(">>>> Starting AppSignalsTestSuite") } -func (suite *APMTestSuite) TearDownSuite() { +func (suite *AppSignalsTestSuite) TearDownSuite() { suite.Result.Print() - fmt.Println(">>>> Finished APMTestSuite") + fmt.Println(">>>> Finished AppSignalsTestSuite") } func init() { @@ -53,15 +53,15 @@ func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { eksTestRunners = []*test_runner.EKSTestRunner{ { - Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMServerConsumerTestName, "HostedIn.EKS.Cluster"}, + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsServerConsumerTestName, "HostedIn.EKS.Cluster"}, Env: *env, }, { - Runner: &APMMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMClientProducerTestName, "HostedIn.EKS.Cluster"}, + Runner: &AppSignalsMetricsRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsClientProducerTestName, "HostedIn.EKS.Cluster"}, Env: *env, }, { - Runner: &APMTracesRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, APMTracesTestName, env.EKSClusterName}, + Runner: &AppSignalsTracesRunner{test_runner.BaseTestRunner{DimensionFactory: factory}, AppSignalsTracesTestName, env.EKSClusterName}, Env: *env, }, } @@ -69,7 +69,7 @@ func getEksTestRunners(env *environment.MetaData) []*test_runner.EKSTestRunner { return eksTestRunners } -func (suite *APMTestSuite) TestAllInSuite() { +func (suite *AppSignalsTestSuite) TestAllInSuite() { env := environment.GetEnvironmentMetaData() switch env.ComputeType { case computetype.EKS: @@ -81,13 +81,13 @@ func (suite *APMTestSuite) TestAllInSuite() { return } - suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "APM Test Suite Failed") + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "AppSignals Test Suite Failed") } -func (suite *APMTestSuite) AddToSuiteResult(r status.TestGroupResult) { +func (suite *AppSignalsTestSuite) AddToSuiteResult(r status.TestGroupResult) { suite.Result.TestGroupResults = append(suite.Result.TestGroupResults, r) } -func TestAPMSuite(t *testing.T) { - suite.Run(t, new(APMTestSuite)) +func TestAppSignalsSuite(t *testing.T) { + suite.Run(t, new(AppSignalsTestSuite)) } diff --git a/test/app_signals/metrics_test.go b/test/app_signals/metrics_test.go index a6b348b42..3568e7620 100644 --- a/test/app_signals/metrics_test.go +++ b/test/app_signals/metrics_test.go @@ -15,15 +15,15 @@ import ( ) const testRetryCount = 6 -const namespace = "AWS/APM" +const namespace = "AWS/AppSignals" -type APMMetricsRunner struct { +type AppSignalsMetricsRunner struct { test_runner.BaseTestRunner testName string dimensionKey string } -func (t *APMMetricsRunner) Validate() status.TestGroupResult { +func (t *AppSignalsMetricsRunner) Validate() status.TestGroupResult { metricsToFetch := t.GetMeasuredMetrics() testResults := make([]status.TestResult, len(metricsToFetch)) instructions := GetInstructionsFromTestName(t.testName) @@ -31,7 +31,7 @@ func (t *APMMetricsRunner) Validate() status.TestGroupResult { for i, metricName := range metricsToFetch { var testResult status.TestResult for j := 0; j < testRetryCount; j++ { - testResult = metric.ValidateAPMMetric(t.DimensionFactory, namespace, metricName, instructions) + testResult = metric.ValidateAppSignalsMetric(t.DimensionFactory, namespace, metricName, instructions) if testResult.Status == status.SUCCESSFUL { break } @@ -46,31 +46,31 @@ func (t *APMMetricsRunner) Validate() status.TestGroupResult { } } -func (t *APMMetricsRunner) GetTestName() string { +func (t *AppSignalsMetricsRunner) GetTestName() string { return t.testName } -func (t *APMMetricsRunner) GetAgentRunDuration() time.Duration { +func (t *AppSignalsMetricsRunner) GetAgentRunDuration() time.Duration { return 3 * time.Minute } -func (t *APMMetricsRunner) GetMeasuredMetrics() []string { - return metric.APMMetricNames +func (t *AppSignalsMetricsRunner) GetMeasuredMetrics() []string { + return metric.AppSignalsMetricNames } -func (e *APMMetricsRunner) GetAgentConfigFileName() string { +func (e *AppSignalsMetricsRunner) GetAgentConfigFileName() string { return "" } func GetInstructionsFromTestName(testName string) []dimension.Instruction { switch testName { - case APMClientProducerTestName: + case AppSignalsClientProducerTestName: return metric.ClientProducerInstructions - case APMServerConsumerTestName: + case AppSignalsServerConsumerTestName: return metric.ServerConsumerInstructions default: return nil } } -var _ test_runner.ITestRunner = (*APMMetricsRunner)(nil) +var _ test_runner.ITestRunner = (*AppSignalsMetricsRunner)(nil) diff --git a/test/app_signals/traces_test.go b/test/app_signals/traces_test.go index 2b3729c28..e7d7da076 100644 --- a/test/app_signals/traces_test.go +++ b/test/app_signals/traces_test.go @@ -33,13 +33,13 @@ var annotations = map[string]string{ "aws_local_operation": "operation", } -type APMTracesRunner struct { +type AppSignalsTracesRunner struct { test_runner.BaseTestRunner testName string clusterName string } -func (t *APMTracesRunner) Validate() status.TestGroupResult { +func (t *AppSignalsTracesRunner) Validate() status.TestGroupResult { testResults := []status.TestResult{{ Name: t.testName, Status: status.FAILED, @@ -63,19 +63,19 @@ func (t *APMTracesRunner) Validate() status.TestGroupResult { } } -func (t *APMTracesRunner) GetTestName() string { +func (t *AppSignalsTracesRunner) GetTestName() string { return t.testName } -func (t *APMTracesRunner) GetAgentRunDuration() time.Duration { +func (t *AppSignalsTracesRunner) GetAgentRunDuration() time.Duration { return 3 * time.Minute } -func (t *APMTracesRunner) GetMeasuredMetrics() []string { +func (t *AppSignalsTracesRunner) GetMeasuredMetrics() []string { return nil } -func (e *APMTracesRunner) GetAgentConfigFileName() string { +func (e *AppSignalsTracesRunner) GetAgentConfigFileName() string { return "" } @@ -107,4 +107,4 @@ func FilterExpression(annotations map[string]string) string { return expression } -var _ test_runner.ITestRunner = (*APMTracesRunner)(nil) +var _ test_runner.ITestRunner = (*AppSignalsTracesRunner)(nil) diff --git a/test/metric/apm_util.go b/test/metric/app_signals_util.go similarity index 92% rename from test/metric/apm_util.go rename to test/metric/app_signals_util.go index 5520841f5..781209ab8 100644 --- a/test/metric/apm_util.go +++ b/test/metric/app_signals_util.go @@ -13,7 +13,7 @@ import ( ) var ( - APMMetricNames = []string{ + AppSignalsMetricNames = []string{ "Error", "Fault", "Latency", @@ -70,7 +70,7 @@ var ( } ) -func ValidateAPMMetric(dimFactory dimension.Factory, namespace string, metricName string, instructions []dimension.Instruction) status.TestResult { +func ValidateAppSignalsMetric(dimFactory dimension.Factory, namespace string, metricName string, instructions []dimension.Instruction) status.TestResult { testResult := status.TestResult{ Name: metricName, Status: status.FAILED, diff --git a/test/metric/dimension/instanceid_provider.go b/test/metric/dimension/instanceid_provider.go index 13d763f50..c07b09e38 100644 --- a/test/metric/dimension/instanceid_provider.go +++ b/test/metric/dimension/instanceid_provider.go @@ -81,7 +81,7 @@ func (p *EKSClusterNameProvider) IsApplicable() bool { } func (p *EKSClusterNameProvider) GetDimension(instruction Instruction) types.Dimension { - // For APM metrics, cluster name is under EKS.Cluster dimension + // For AppSignals metrics, cluster name is under EKS.Cluster dimension if instruction.Key == "HostedIn.EKS.Cluster" { return types.Dimension{ Name: aws.String("HostedIn.EKS.Cluster"), From 415d2cc92b7687907cab9afcbb130f48ace93459 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Wed, 25 Oct 2023 18:05:44 -0400 Subject: [PATCH 23/29] namespace reverted --- test/app_signals/metrics_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/app_signals/metrics_test.go b/test/app_signals/metrics_test.go index 3568e7620..3628e0c2e 100644 --- a/test/app_signals/metrics_test.go +++ b/test/app_signals/metrics_test.go @@ -15,7 +15,7 @@ import ( ) const testRetryCount = 6 -const namespace = "AWS/AppSignals" +const namespace = "AWS/APM" type AppSignalsMetricsRunner struct { test_runner.BaseTestRunner From d929301c402b024fb96569d50b5ce74fcd104528 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Thu, 26 Oct 2023 10:11:12 -0400 Subject: [PATCH 24/29] changed agent image, namespace and config --- generator/test_case_generator.go | 17 ----------------- terraform/eks/daemon/app_signals/variables.tf | 3 ++- test/app_signals/metrics_test.go | 2 +- test/app_signals/resources/config.json | 8 ++------ 4 files changed, 5 insertions(+), 25 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 655d1c8ee..1f0af9aad 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -170,23 +170,6 @@ var testTypeToTestConfig = map[string][]testConfig{ }, }, "eks_daemon": { - { - testDir: "./test/metric_value_benchmark", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, {testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, diff --git a/terraform/eks/daemon/app_signals/variables.tf b/terraform/eks/daemon/app_signals/variables.tf index fb73ec3d6..bd2dbc579 100644 --- a/terraform/eks/daemon/app_signals/variables.tf +++ b/terraform/eks/daemon/app_signals/variables.tf @@ -13,7 +13,8 @@ variable "test_dir" { variable "cwagent_image_repo" { type = string - default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-release" + //Todo: change this to public ECR repo image after GA "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" + default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-pre-release:appsignals" } variable "cwagent_image_tag" { diff --git a/test/app_signals/metrics_test.go b/test/app_signals/metrics_test.go index 3628e0c2e..4771cba10 100644 --- a/test/app_signals/metrics_test.go +++ b/test/app_signals/metrics_test.go @@ -15,7 +15,7 @@ import ( ) const testRetryCount = 6 -const namespace = "AWS/APM" +const namespace = "AppSignals" type AppSignalsMetricsRunner struct { test_runner.BaseTestRunner diff --git a/test/app_signals/resources/config.json b/test/app_signals/resources/config.json index 9aa23da0c..9c03d5915 100644 --- a/test/app_signals/resources/config.json +++ b/test/app_signals/resources/config.json @@ -4,16 +4,12 @@ }, "logs": { "metrics_collected": { - "app_signals": { - "enabled": true - } + "app_signals": {} } }, "traces": { "traces_collected": { - "app_signals": { - "enabled": true - } + "app_signals": {} } } } \ No newline at end of file From 782603577b1c5ee976e71144bb9a6c6cdff47bee Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Thu, 26 Oct 2023 10:44:55 -0400 Subject: [PATCH 25/29] changed image --- terraform/eks/daemon/app_signals/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/eks/daemon/app_signals/variables.tf b/terraform/eks/daemon/app_signals/variables.tf index bd2dbc579..7713cc998 100644 --- a/terraform/eks/daemon/app_signals/variables.tf +++ b/terraform/eks/daemon/app_signals/variables.tf @@ -14,12 +14,12 @@ variable "test_dir" { variable "cwagent_image_repo" { type = string //Todo: change this to public ECR repo image after GA "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" - default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-pre-release:appsignals" + default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-pre-release" } variable "cwagent_image_tag" { type = string - default = "latest" + default = "appsignals" } variable "k8s_version" { From 875a5519d615533feef5e7207d0f45f0eba16a2e Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Thu, 26 Oct 2023 13:54:53 -0400 Subject: [PATCH 26/29] change the default image repo and tag for EKS integ test to public ECR image --- generator/test_case_generator.go | 17 +++++++++++++++++ terraform/eks/daemon/app_signals/variables.tf | 5 ++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 1f0af9aad..655d1c8ee 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -170,6 +170,23 @@ var testTypeToTestConfig = map[string][]testConfig{ }, }, "eks_daemon": { + { + testDir: "./test/metric_value_benchmark", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, {testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, diff --git a/terraform/eks/daemon/app_signals/variables.tf b/terraform/eks/daemon/app_signals/variables.tf index 7713cc998..a0434630e 100644 --- a/terraform/eks/daemon/app_signals/variables.tf +++ b/terraform/eks/daemon/app_signals/variables.tf @@ -13,13 +13,12 @@ variable "test_dir" { variable "cwagent_image_repo" { type = string - //Todo: change this to public ECR repo image after GA "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" - default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-pre-release" + default = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" } variable "cwagent_image_tag" { type = string - default = "appsignals" + default = "latest" } variable "k8s_version" { From 5c0c827de5957bbb204f57a6d5630b37508168d5 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Fri, 27 Oct 2023 15:08:55 -0400 Subject: [PATCH 27/29] refactored code --- test/app_signals/traces_test.go | 49 ++++++--------------------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/test/app_signals/traces_test.go b/test/app_signals/traces_test.go index e7d7da076..f12793a80 100644 --- a/test/app_signals/traces_test.go +++ b/test/app_signals/traces_test.go @@ -6,25 +6,20 @@ package app_signals import ( - "context" - "encoding/json" "fmt" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/xray" - "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) const ( - lookbackDuration = time.Duration(-80) * time.Minute + lookbackDuration = time.Duration(-20) * time.Minute EKSClusterAnnotation = "HostedIn_EKS_Cluster" ) -var annotations = map[string]string{ +var annotations = map[string]interface{}{ "aws_remote_target": "remote-target", "aws_remote_operation": "remote-operation", "aws_local_service": "service-name", @@ -40,26 +35,26 @@ type AppSignalsTracesRunner struct { } func (t *AppSignalsTracesRunner) Validate() status.TestGroupResult { - testResults := []status.TestResult{{ + testResults := status.TestResult{ Name: t.testName, Status: status.FAILED, - }} + } timeNow := time.Now() annotations[EKSClusterAnnotation] = t.clusterName - xrayFilter := FilterExpression(annotations) - traceIds, err := GetTraceIDs(timeNow.Add(lookbackDuration), timeNow, xrayFilter) + xrayFilter := awsservice.FilterExpression(annotations) + traceIds, err := awsservice.GetTraceIDs(timeNow.Add(lookbackDuration), timeNow, xrayFilter) if err != nil { fmt.Printf("error getting trace ids: %v", err) } else { fmt.Printf("Trace IDs: %v\n", traceIds) if len(traceIds) > 0 { - testResults[0].Status = status.SUCCESSFUL + testResults.Status = status.SUCCESSFUL } } return status.TestGroupResult{ Name: t.GetTestName(), - TestResults: testResults, + TestResults: []status.TestResult{testResults}, } } @@ -79,32 +74,4 @@ func (e *AppSignalsTracesRunner) GetAgentConfigFileName() string { return "" } -func GetTraceIDs(startTime time.Time, endTime time.Time, filter string) ([]string, error) { - var traceIDs []string - input := &xray.GetTraceSummariesInput{StartTime: aws.Time(startTime), EndTime: aws.Time(endTime), FilterExpression: aws.String(filter)} - output, err := awsservice.XrayClient.GetTraceSummaries(context.Background(), input) - if err != nil { - return nil, err - } - for _, summary := range output.TraceSummaries { - traceIDs = append(traceIDs, *summary.Id) - } - return traceIDs, nil -} - -func FilterExpression(annotations map[string]string) string { - var expression string - for key, value := range annotations { - result, err := json.Marshal(value) - if err != nil { - continue - } - if len(expression) != 0 { - expression += " AND " - } - expression += fmt.Sprintf("annotation.%s = %s", key, result) - } - return expression -} - var _ test_runner.ITestRunner = (*AppSignalsTracesRunner)(nil) From 994f3df3c3c2ae9f8cd5dae8030530c27287d659 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Fri, 27 Oct 2023 15:21:32 -0400 Subject: [PATCH 28/29] changed the lookback to 30 mins --- test/app_signals/traces_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/app_signals/traces_test.go b/test/app_signals/traces_test.go index f12793a80..37d91ec59 100644 --- a/test/app_signals/traces_test.go +++ b/test/app_signals/traces_test.go @@ -15,7 +15,7 @@ import ( ) const ( - lookbackDuration = time.Duration(-20) * time.Minute + lookbackDuration = time.Duration(-30) * time.Minute EKSClusterAnnotation = "HostedIn_EKS_Cluster" ) From ff3f3a57c62765d174b5c37c6fd967c5eacba15e Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Fri, 27 Oct 2023 17:11:50 -0400 Subject: [PATCH 29/29] change look back to 5mins --- test/app_signals/traces_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/app_signals/traces_test.go b/test/app_signals/traces_test.go index 37d91ec59..fe84a2b0f 100644 --- a/test/app_signals/traces_test.go +++ b/test/app_signals/traces_test.go @@ -15,7 +15,7 @@ import ( ) const ( - lookbackDuration = time.Duration(-30) * time.Minute + lookbackDuration = time.Duration(-5) * time.Minute EKSClusterAnnotation = "HostedIn_EKS_Cluster" )