From a273a24f6b33bceca0c42d211a177c65786a1c12 Mon Sep 17 00:00:00 2001 From: Parampreet Singh <50599809+Paramadon@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:13:15 -0400 Subject: [PATCH] Extend CloudWatch Agent AppSignals EC2 tests for MacOS and Window (#405) --- .../resources/metrics/client_producer.json | 43 +++- .../resources/metrics/server_consumer.json | 72 +++++-- test/feature/mac/agent_config.json | 190 +++++++++--------- test/feature/mac/parameters.yml | 53 +++-- test/feature/windows/agent_config.json | 6 + test/feature/windows/parameters.yml | 30 ++- util/awsservice/constant.go | 50 +++-- util/common/metrics.go | 141 ++++++++++++- validator/models/validation_config.go | 4 +- validator/validators/basic/basic_validator.go | 88 +++++++- 10 files changed, 522 insertions(+), 155 deletions(-) diff --git a/test/app_signals/resources/metrics/client_producer.json b/test/app_signals/resources/metrics/client_producer.json index e75d43125..6095c78ac 100644 --- a/test/app_signals/resources/metrics/client_producer.json +++ b/test/app_signals/resources/metrics/client_producer.json @@ -78,9 +78,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 0, - "min": 0, - "max": 0 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -131,9 +139,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 0, - "min": 0, - "max": 0 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -184,9 +200,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 0, - "min": 0, - "max": 0 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -197,4 +221,3 @@ } ] } - diff --git a/test/app_signals/resources/metrics/server_consumer.json b/test/app_signals/resources/metrics/server_consumer.json index 6adb15d49..cf6c7e247 100644 --- a/test/app_signals/resources/metrics/server_consumer.json +++ b/test/app_signals/resources/metrics/server_consumer.json @@ -84,9 +84,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 0, - "min": 0, - "max": 0 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -143,9 +151,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 0, - "min": 0, - "max": 0 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -202,9 +218,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 0, - "min": 0, - "max": 0 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -259,11 +283,20 @@ } } ], + "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 1, - "min": 1, - "max": 1 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -320,9 +353,17 @@ ], "startTimeUnixNano": START_TIME, "timeUnixNano": START_TIME, - "sum": 1, - "min": 1, - "max": 1 + "count": 0, + "scale": 0, + "zeroCount": 0, + "positive": { + "offset": 0, + "bucket_counts": [1, 2, 3] + }, + "negative": { + "offset": 0, + "bucket_counts": [1, 2, 3] + } } ] } @@ -333,4 +374,3 @@ } ] } - diff --git a/test/feature/mac/agent_config.json b/test/feature/mac/agent_config.json index 4bbff6336..3ceb9c6b3 100644 --- a/test/feature/mac/agent_config.json +++ b/test/feature/mac/agent_config.json @@ -1,103 +1,109 @@ { - "agent": { - "debug": true - }, - "metrics": { - "namespace": "CloudWatchAgentMacFeature", - "metrics_collected": { - "statsd": { - "metrics_aggregation_interval": 60, - "metrics_collection_interval": 60, - "service_address": ":8125" - }, - "cpu": { + "agent": { + "debug": true + }, + "metrics": { + "namespace": "CloudWatchAgentMacFeature", + "metrics_collected": { + "statsd": { + "metrics_aggregation_interval": 60, + "metrics_collection_interval": 60, + "service_address": ":8125" + }, + "cpu": { + "measurement": [ + "time_active", + "time_guest" + ], + "metrics_collection_interval": 1 + }, + "swap": { + "measurement": [ + "free", + "used_percent" + ], + "metrics_collection_interval": 1 + }, + "processes": { + "measurement": [ + "blocked", + "running" + ], + "metrics_collection_interval": 1 + }, + "netstat": { + "measurement": [ + "tcp_close", + "udp_socket" + ], + "metrics_collection_interval": 1 + }, + "mem": { + "measurement": [ + "available_percent", + "used_percent" + ], + "metrics_collection_interval": 1 + }, + "disk": { + "resources": [ + "*" + ], + "measurement": [ + "free", + "used_percent" + ], + "drop_device": true, + "metrics_collection_interval": 1 + }, + "net": { + "resources": [ + "en0" + ], + "measurement": [ + "bytes_sent", + "bytes_recv" + ], + "metrics_collection_interval": 1 + }, + "procstat": [ + { + "exe": "amazon-cloudwatch-agent", "measurement": [ - "time_active", - "time_guest" + "cpu_usage", + "memory_rss" ], "metrics_collection_interval": 1 - }, - "swap": { - "measurement": [ - "free", - "used_percent" - ], - "metrics_collection_interval": 1 - }, - "processes": { - "measurement": [ - "blocked", - "running" - ], - "metrics_collection_interval": 1 - }, - "netstat": { - "measurement": [ - "tcp_close", - "udp_socket" - ], - "metrics_collection_interval": 1 - }, - "mem": { - "measurement": [ - "available_percent", - "used_percent" - ], - "metrics_collection_interval": 1 - }, - "disk": { - "resources": [ - "*" - ], - "measurement": [ - "free", - "used_percent" - ], - "drop_device": true, - "metrics_collection_interval": 1 - }, - "net": { - "resources": [ - "en0" - ], - "measurement": [ - "bytes_sent", - "bytes_recv" - ], - "metrics_collection_interval": 1 - }, - "procstat": [ - { - "exe": "amazon-cloudwatch-agent", - "measurement": [ - "cpu_usage", - "memory_rss" - ], - "metrics_collection_interval": 1 - } - ] - }, - "append_dimensions": { - "InstanceId": "${aws:InstanceId}" - }, - "force_flush_interval": 30 + } + ] }, - "logs": { - "logs_collected": { - "files": { + "append_dimensions": { + "InstanceId": "${aws:InstanceId}" + }, + "force_flush_interval": 30 + }, + "logs": { + "logs_collected": { + "files": { "collect_list": [ { - "file_path": "/tmp/test1.log", - "log_group_name": "{instance_id}", - "log_stream_name": "test1.log", - "timezone": "UTC" + "file_path": "/tmp/test1.log", + "log_group_name": "{instance_id}", + "log_stream_name": "test1.log", + "timezone": "UTC" } ] - } - }, - "metrics_collected": { - "emf": { } - }, - "force_flush_interval": 5 + } + }, + "metrics_collected": { + "app_signals": {}, + "emf": { } + }, + "force_flush_interval": 5 + }, + "traces": { + "traces_collected": { + "app_signals": {} } + } } \ No newline at end of file diff --git a/test/feature/mac/parameters.yml b/test/feature/mac/parameters.yml index 31a15abe7..12bd99100 100644 --- a/test/feature/mac/parameters.yml +++ b/test/feature/mac/parameters.yml @@ -1,5 +1,5 @@ # Receivers that agent needs to tests -receivers: ["system","statsd","emf"] +receivers: ["system","statsd","emf","app_signals","traces"] #Test case name test_case: "macos_feature" @@ -13,7 +13,7 @@ number_monitored_logs: 1 # Number of metrics to be sent or number of log lines being written each minute values_per_minute: "2" # Number of seconds the agent should run and collect the metrics. In this case, 1 minutes -agent_collection_period: 60 +agent_collection_period: 60 cloudwatch_agent_config: "" @@ -27,14 +27,14 @@ metric_validation: - metric_name: "statsd_gauge_1" metric_value: 1.0 metric_sample_count: 1 - metric_dimension: + metric_dimension: - name: "metric_type" value: "gauge" - metric_name: "statsd_counter_1" metric_value: 1.0 metric_sample_count: 1 - metric_dimension: + metric_dimension: - name: "metric_type" value: "counter" @@ -43,38 +43,38 @@ metric_validation: - metric_name: "emf_time_1" metric_value: 1.0 metric_sample_count: 1 - + - metric_name: "emf_time_2" metric_value: 2.0 metric_sample_count: 1 - metric_name: "cpu_time_active" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "cpu" value: "cpu-total" - metric_name: "cpu_time_guest" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "cpu" value: "cpu-total" - metric_name: "net_bytes_sent" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "interface" value: "en0" - metric_name: "net_bytes_recv" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "interface" value: "en0" - metric_name: "disk_free" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "fstype" value: "devfs" - name: "path" @@ -82,7 +82,7 @@ metric_validation: - metric_name: "disk_used_percent" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "fstype" value: "devfs" - name: "path" @@ -90,7 +90,7 @@ metric_validation: - metric_name: "procstat_cpu_usage" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "exe" value: "amazon-cloudwatch-agent" - name: "process_name" @@ -98,7 +98,7 @@ metric_validation: - metric_name: "procstat_memory_rss" metric_sample_count: 60 - metric_dimension: + metric_dimension: - name: "exe" value: "amazon-cloudwatch-agent" - name: "process_name" @@ -128,6 +128,33 @@ metric_validation: - metric_name: "mem_used_percent" metric_sample_count: 60 metric_dimension: [] + - metric_name: "Fault" + metric_sample_count: 60 + metric_dimension: + - name: "HostedIn.Environment" + value: "Generic" + - name: "Operation" + value: "operation" + - name: "Service" + value: "service-name" + - metric_name: "Latency" + metric_sample_count: 60 + metric_dimension: + - name: "HostedIn.Environment" + value: "Generic" + - name: "Operation" + value: "operation" + - name: "Service" + value: "service-name" + - metric_name: "Error" + metric_sample_count: 60 + metric_dimension: + - name: "HostedIn.Environment" + value: "Generic" + - name: "Operation" + value: "operation" + - name: "Service" + value: "service-name" # Logs that the test needs to validate; moreover, the feature validation already has # InstanceID as a log group; therefore, does not need to pass it diff --git a/test/feature/windows/agent_config.json b/test/feature/windows/agent_config.json index 8a605053a..5ddf146ce 100644 --- a/test/feature/windows/agent_config.json +++ b/test/feature/windows/agent_config.json @@ -2,6 +2,11 @@ "agent": { "debug": true }, + "traces": { + "traces_collected": { + "app_signals": {} + } + }, "metrics": { "namespace": "CloudWatchAgentWinFeature", "metrics_collected": { @@ -217,6 +222,7 @@ } }, "metrics_collected": { + "app_signals": {}, "emf": { }, "prometheus": { "prometheus_config_path": "C:/jmx_workload/prometheus.yaml", diff --git a/test/feature/windows/parameters.yml b/test/feature/windows/parameters.yml index 76f409a9f..0273d5641 100644 --- a/test/feature/windows/parameters.yml +++ b/test/feature/windows/parameters.yml @@ -1,5 +1,5 @@ # Receivers that agent needs to tests -receivers: ["system","statsd","emf"] +receivers: ["system","statsd","emf","app_signals","traces"] #Test case name test_case: "win_feature" @@ -198,6 +198,34 @@ metric_validation: metric_dimension: - name: "objectname" value: "TCPv6" + + - metric_name: "Fault" + metric_sample_count: 12 + metric_dimension: + - name: "HostedIn.Environment" + value: "Generic" + - name: "Operation" + value: "operation" + - name: "Service" + value: "service-name" + - metric_name: "Latency" + metric_sample_count: 12 + metric_dimension: + - name: "HostedIn.Environment" + value: "Generic" + - name: "Operation" + value: "operation" + - name: "Service" + value: "service-name" + - metric_name: "Error" + metric_sample_count: 12 + metric_dimension: + - name: "HostedIn.Environment" + value: "Generic" + - name: "Operation" + value: "operation" + - name: "Service" + value: "service-name" # Logs that the test needs to validate; moreover, the feature validation already has # InstanceID as a log group; therefore, does not need to pass it # https://github.com/aws/amazon-cloudwatch-agent-test/blob/96f576e865b55de5e2aa88e4cf80b79c4d3dad70/validator/validators/feature/feature_validator.go#L108-L111 diff --git a/util/awsservice/constant.go b/util/awsservice/constant.go index 1009a08e2..53da7d910 100644 --- a/util/awsservice/constant.go +++ b/util/awsservice/constant.go @@ -5,6 +5,7 @@ package awsservice import ( "context" + "fmt" "time" "github.com/aws/aws-sdk-go-v2/config" @@ -38,16 +39,41 @@ var ( ) var ( - ctx = context.Background() - awsCfg, _ = config.LoadDefaultConfig(ctx) - Ec2Client = ec2.NewFromConfig(awsCfg) - EcsClient = ecs.NewFromConfig(awsCfg) - SsmClient = ssm.NewFromConfig(awsCfg) - ImdsClient = imds.NewFromConfig(awsCfg) - CwmClient = cloudwatch.NewFromConfig(awsCfg) - CwlClient = cloudwatchlogs.NewFromConfig(awsCfg) - DynamodbClient = dynamodb.NewFromConfig(awsCfg) - S3Client = s3.NewFromConfig(awsCfg) - CloudformationClient = cloudformation.NewFromConfig(awsCfg) - XrayClient = xray.NewFromConfig(awsCfg) + ctx context.Context + + // AWS Clients + Ec2Client *ec2.Client + EcsClient *ecs.Client + SsmClient *ssm.Client + ImdsClient *imds.Client + CwmClient *cloudwatch.Client + CwlClient *cloudwatchlogs.Client + DynamodbClient *dynamodb.Client + S3Client *s3.Client + CloudformationClient *cloudformation.Client + XrayClient *xray.Client ) + +func init() { + ctx = context.Background() + var err error + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-west-2")) + if err != nil { + // handle error + fmt.Println("There was an error trying to load default config: ", err) + return + } + fmt.Println("This is the aws region: ", awsCfg.Region) + + // Initialize AWS Clients with the configured awsCfg + Ec2Client = ec2.NewFromConfig(awsCfg) + EcsClient = ecs.NewFromConfig(awsCfg) + SsmClient = ssm.NewFromConfig(awsCfg) + ImdsClient = imds.NewFromConfig(awsCfg) + CwmClient = cloudwatch.NewFromConfig(awsCfg) + CwlClient = cloudwatchlogs.NewFromConfig(awsCfg) + DynamodbClient = dynamodb.NewFromConfig(awsCfg) + S3Client = s3.NewFromConfig(awsCfg) + CloudformationClient = cloudformation.NewFromConfig(awsCfg) + XrayClient = xray.NewFromConfig(awsCfg) +} diff --git a/util/common/metrics.go b/util/common/metrics.go index ddd93b492..bf9435f8a 100644 --- a/util/common/metrics.go +++ b/util/common/metrics.go @@ -4,10 +4,19 @@ package common import ( + "bytes" "context" + "crypto/rand" + "encoding/binary" + "encoding/hex" "errors" "fmt" "net" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" "time" "collectd.org/api" @@ -17,6 +26,11 @@ import ( "github.com/prozz/aws-embedded-metrics-golang/emf" ) +const SleepDuration = 5 * time.Second + +const TracesEndpoint = "4316/v1/traces" +const MetricEndpoint = "4316/v1/metrics" + // StartSendingMetrics will generate metrics load based on the receiver (e.g 5000 statsd metrics per minute) func StartSendingMetrics(receiver string, duration, sendingInterval time.Duration, metricPerInterval int, metricLogGroup, metricNamespace string) (err error) { go func() { @@ -27,14 +41,71 @@ func StartSendingMetrics(receiver string, duration, sendingInterval time.Duratio err = SendCollectDMetrics(metricPerInterval, sendingInterval, duration) case "emf": err = SendEMFMetrics(metricPerInterval, metricLogGroup, metricNamespace, sendingInterval, duration) + case "app_signals": + err = SendAppSignalMetrics(duration) //does app signals have dimension for metric? + case "traces": + err = SendAppSignalsTraceMetrics(duration) //does app signals have dimension for metric? + default: } - }() return err } +func SendAppSignalsTraceMetrics(duration time.Duration) error { + baseDir := getBaseDir() + + for i := 0; i < int(duration/(5*time.Second)); i++ { + startTime := time.Now().UnixNano() + traceID := generateTraceID() + traceIDStr := hex.EncodeToString(traceID[:]) + + err := processTraceFile(filepath.Join(baseDir, "traces.json"), startTime, traceIDStr) + if err != nil { + fmt.Println("Error processing trace file:", err) + return err + } + + time.Sleep(5 * time.Second) + } + + return nil +} + +func getBaseDir() string { + if runtime.GOOS == "windows" { + return "C:\\Users\\Administrator\\amazon-cloudwatch-agent-test\\test\\app_signals\\resources\\traces" + } + return "/Users/ec2-user/amazon-cloudwatch-agent-test/test/app_signals/resources/traces" +} + +func generateTraceID() [16]byte { + var r [16]byte + epochNow := time.Now().Unix() + binary.BigEndian.PutUint32(r[0:4], uint32(epochNow)) + rand.Read(r[4:]) + return r +} + +func processTraceFile(filePath string, startTime int64, traceID string) error { + data, err := os.ReadFile(filePath) + if err != nil { + return err + } + + modifiedData := strings.ReplaceAll(string(data), "START_TIME", fmt.Sprintf("%d", startTime)) + modifiedData = strings.ReplaceAll(modifiedData, "TRACE_ID", traceID) + + url := "http://127.0.0.1:" + TracesEndpoint + _, err = http.Post(url, "application/json", bytes.NewBufferString(modifiedData)) + if err != nil { + return err + } + + return nil +} + func SendCollectDMetrics(metricPerInterval int, sendingInterval, duration time.Duration) error { // https://github.com/collectd/go-collectd/tree/92e86f95efac5eb62fa84acc6033e7a57218b606 ctx, cancel := context.WithCancel(context.Background()) @@ -130,6 +201,73 @@ func SendCollectDMetrics(metricPerInterval int, sendingInterval, duration time.D } } +} +func processFile(filePath string, startTime int64) error { + data, err := os.ReadFile(filePath) + if err != nil { + fmt.Println("Error reading file:", err) + return nil + } + + //replace START_TIME with the current time + modifiedData := strings.ReplaceAll(string(data), "START_TIME", fmt.Sprintf("%d", startTime)) + + //curl command + url := "http://127.0.0.1:" + MetricEndpoint + _, err = http.Post(url, "application/json", bytes.NewBufferString(modifiedData)) + + _, err = http.Post(url, "application/json", bytes.NewBufferString(modifiedData)) + if err != nil { + fmt.Println("Failed to send POST request to", url) + fmt.Printf("Error: %v\n", err) + return err + } + return nil + +} + +func SendAppSignalMetrics(duration time.Duration) error { + // The bash script to be executed asynchronously. + dir, err := os.Getwd() + if err != nil { + fmt.Println("Error getting current directory:", err) + return err + } + fmt.Println("Current Directory:", dir) + + // Determine the base directory for the files based on the OS + var baseDir string + if runtime.GOOS == "windows" { + baseDir = filepath.Join("C:", "Users", "Administrator", "amazon-cloudwatch-agent-test", "test", "app_signals", "resources", "metrics") + } else { // assuming macOS or Unix-like system + baseDir = filepath.Join("/", "Users", "ec2-user", "amazon-cloudwatch-agent-test", "test", "app_signals", "resources", "metrics") + } + + fmt.Println("Base directory:", baseDir) + + for i := 0; i < int(duration/SleepDuration); i++ { + if err != nil { + return err + } + + //start time to send to process file + startTime := time.Now().UnixNano() + + //process files + err = processFile(filepath.Join(baseDir, "server_consumer.json"), startTime) + if err != nil { + return err + } + err = processFile(filepath.Join(baseDir, "client_producer.json"), startTime) + if err != nil { + return err + } + + time.Sleep(5 * time.Second) + } + + return nil + } func SendStatsdMetrics(metricPerInterval int, metricDimension []string, sendingInterval, duration time.Duration) error { @@ -210,4 +348,5 @@ func SendEMFMetrics(metricPerInterval int, metricLogGroup, metricNamespace strin return nil } } + } diff --git a/validator/models/validation_config.go b/validator/models/validation_config.go index 6d29ee2c7..26fc57a91 100644 --- a/validator/models/validation_config.go +++ b/validator/models/validation_config.go @@ -15,7 +15,8 @@ import ( "gopkg.in/yaml.v3" ) -var supportedReceivers = []string{"logs", "statsd", "collectd", "system", "emf", "xray"} +var supportedReceivers = []string{"logs", "statsd", "collectd", "system", "emf", "xray", "app_signals", "traces"} +var retryCount = 0 type ValidateConfig interface { GetPluginsConfig() []string @@ -55,6 +56,7 @@ type validatorConfig struct { CommitHash string `yaml:"commit_hash"` CommitDate string `yaml:"commit_date"` + retryCount int } type MetricValidation struct { diff --git a/validator/validators/basic/basic_validator.go b/validator/validators/basic/basic_validator.go index 885b5a61c..67fad58cf 100644 --- a/validator/validators/basic/basic_validator.go +++ b/validator/validators/basic/basic_validator.go @@ -14,6 +14,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "go.uber.org/multierr" + AppSignalMetrics "github.com/aws/amazon-cloudwatch-agent-test/test/metric" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" "github.com/aws/amazon-cloudwatch-agent-test/util/common" "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces" @@ -22,6 +23,7 @@ import ( ) const metricErrorBound = 0.1 +const AppSignalNamespace = "AppSignals" type BasicValidator struct { vConfig models.ValidateConfig @@ -68,24 +70,46 @@ func (s *BasicValidator) CheckData(startTime, endTime time.Time) error { ) for _, metric := range validationMetric { - metricDimensions := []cwtypes.Dimension{ - { - Name: aws.String("InstanceId"), - Value: aws.String(ec2InstanceId), - }, + metricDimensions := []cwtypes.Dimension{} + //App Signal Metrics don't have instanceid dimension + if !isAppSignalMetric(metric) { + metricDimensions = []cwtypes.Dimension{ + { + Name: aws.String("InstanceId"), + Value: aws.String(ec2InstanceId), + }, + } } + for _, dimension := range metric.MetricDimension { metricDimensions = append(metricDimensions, cwtypes.Dimension{ Name: aws.String(dimension.Name), Value: aws.String(dimension.Value), }) } - err := s.ValidateMetric(metric.MetricName, metricNamespace, metricDimensions, metric.MetricValue, metric.MetricSampleCount, startTime, endTime) - if err != nil { - multiErr = multierr.Append(multiErr, err) + + //App Signals metric testing (This is because we want to use a different checking method (same that was done for linux test)) + if metric.MetricName == "Latency" || metric.MetricName == "Fault" || metric.MetricName == "Error" { + err := s.ValidateAppSignalMetrics(metric, metricDimensions) + if err != nil { + multiErr = multierr.Append(multiErr, err) + } else { + fmt.Println("App Signal Metrics are correct!") + } + } else { + err := s.ValidateMetric(metric.MetricName, metricNamespace, metricDimensions, metric.MetricValue, metric.MetricSampleCount, startTime, endTime) + if err != nil { + return err + } } - } + } + err := s.ValidateTracesMetrics() + if err != nil { + multiErr = multierr.Append(multiErr, err) + } else { + fmt.Println("Traces Metrics are correct!") + } for _, logValidation := range logValidations { err := s.ValidateLogs(logValidation.LogStream, logValidation.LogValue, logValidation.LogLevel, logValidation.LogSource, logValidation.LogLines, startTime, endTime) if err != nil { @@ -109,6 +133,52 @@ func (s *BasicValidator) Cleanup() error { return nil } +func isAppSignalMetric(metric models.MetricValidation) bool { + if metric.MetricName == "Latency" || metric.MetricName == "Fault" || metric.MetricName == "Error" { + return true + } + return false +} +func (s *BasicValidator) ValidateAppSignalMetrics(metric models.MetricValidation, metricDimensions []cwtypes.Dimension) error { + + if metric.MetricName == "Latency" || metric.MetricName == "Fault" || metric.MetricName == "Error" { + fetcher := AppSignalMetrics.MetricValueFetcher{} + values, err := fetcher.Fetch(AppSignalNamespace, metric.MetricName, metricDimensions, "Sum", 60) + if err != nil { + return err + } + if !AppSignalMetrics.IsAllValuesGreaterThanOrEqualToExpectedValue(metric.MetricName, values, 0) { + fmt.Printf("Error values are not the epected values%v", err) + return err + } + } + + return nil +} + +func (s *BasicValidator) ValidateTracesMetrics() error { + lookbackDuration := time.Duration(-5) * time.Minute + serviceName := "service-name" + serviceType := "AWS::EC2::Instance" + //filtering traces + filterExpression := fmt.Sprintf("(service(id(name: \"%s\", type: \"%s\")))", serviceName, serviceType) + timeNow := time.Now() + + traceIds, err := awsservice.GetTraceIDs(timeNow.Add(lookbackDuration), timeNow, filterExpression) + if err != nil { + fmt.Printf("error getting trace ids: %v", err) + return err + } else { + fmt.Printf("Trace IDs: %v\n", traceIds) + if len(traceIds) > 0 { + fmt.Println("Trace IDs look good") + } else { + return err + } + } + return nil +} + func (s *BasicValidator) ValidateLogs(logStream, logLine, logLevel, logSource string, expectedMinimumEventCount int, startTime, endTime time.Time) error { logGroup := awsservice.GetInstanceId() log.Printf("Start to validate that substring '%s' has at least %d log event(s) within log group %s, log stream %s, between %v and %v", logLine, expectedMinimumEventCount, logGroup, logStream, startTime, endTime)