diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index fa2cd2274..e4c53f843 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -187,22 +187,6 @@ var testTypeToTestConfig = map[string][]testConfig{ testDir: "./test/metric_value_benchmark", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, - { - testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, - {testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, }, "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, diff --git a/terraform/eks/daemon/main.tf b/terraform/eks/daemon/main.tf index 2ca2f063f..c618da422 100644 --- a/terraform/eks/daemon/main.tf +++ b/terraform/eks/daemon/main.tf @@ -408,12 +408,24 @@ resource "kubernetes_cluster_role_binding" "rolebinding" { } } +resource "time_sleep" "wait_2_min" { + depends_on = [ + aws_eks_node_group.this, + kubernetes_daemonset.service, + kubernetes_cluster_role_binding.rolebinding, + kubernetes_service_account.cwagentservice + ] + + create_duration = "2m" +} + resource "null_resource" "validator" { depends_on = [ aws_eks_node_group.this, kubernetes_daemonset.service, kubernetes_cluster_role_binding.rolebinding, kubernetes_service_account.cwagentservice, + time_sleep.wait_2_min ] provisioner "local-exec" { command = <<-EOT diff --git a/test/cloudwatchlogs/resources/config_windows_event_log.json b/test/cloudwatchlogs/resources/config_windows_event_log.json new file mode 100644 index 000000000..ddfdf2705 --- /dev/null +++ b/test/cloudwatchlogs/resources/config_windows_event_log.json @@ -0,0 +1,34 @@ +{ + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "C:\\Users\\Administrator\\Desktop\\CWMetricsLogs", + "log_group_name": "CWMetricsLogs", + "log_stream_name": "{instance_id}", + "retention_in_days": -1 + } + ] + }, + "windows_events": { + "collect_list": [ + { + "event_format": "xml", + "event_levels": [ + "VERBOSE", + "INFORMATION", + "WARNING", + "ERROR", + "CRITICAL" + ], + "event_name": "System", + "log_group_name": "CloudWatchAgent", + "log_stream_name": "{instance_id}", + "retention_in_days": -1 + } + ] + } + } + } +} \ No newline at end of file diff --git a/test/cloudwatchlogs/windows_event_log_test.go b/test/cloudwatchlogs/windows_event_log_test.go new file mode 100644 index 000000000..6d7a45944 --- /dev/null +++ b/test/cloudwatchlogs/windows_event_log_test.go @@ -0,0 +1,65 @@ +//go:build windows + +package cloudwatchlogs + +import ( + "fmt" + "log" + "strings" + "testing" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/internal/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/internal/common" + "github.com/stretchr/testify/assert" +) + +func TestWindowsEventLog(t *testing.T) { + cfgFilePath := "resources/config_windows_event_log.json" + + instanceId := awsservice.GetInstanceId() + log.Printf("Found instance id %s", instanceId) + logGroup := "CloudWatchAgent" + logStream := instanceId + + start := time.Now() + common.CopyFile(cfgFilePath, configOutputPath) + + common.StartAgent(configOutputPath, true) + + // ensure that there is enough time from the "start" time and the first log line, + // so we don't miss it in the GetLogEvents call + time.Sleep(agentRuntime) + t.Log("Writing logs from windows event log plugin") + time.Sleep(agentRuntime) + common.StopAgent() + + lines := []string{ + fmt.Sprintf("{\"Metric\": \"%s\"}", strings.Repeat("12345", 10)), + fmt.Sprintf("{\"Metric\": \"%s\"}", strings.Repeat("09876", 10)), + fmt.Sprintf("{\"Metric\": \"%s\"}", strings.Repeat("1234567890", 10)), + } + + end := time.Now() + + ok, err := awsservice.ValidateLogs(logGroup, logStream, &start, &end, func(logs []string) bool { + log.Printf("logs length: %s ", len(logs)) + if len(logs) != len(lines) { + return false + } + + for i := 0; i < len(logs); i++ { + log.Printf("lines[%s] is %s", i, lines[i]) + log.Printf("logs[%s] is %s", i, logs[i]) + expected := strings.ReplaceAll(lines[i], "'", "\"") + actual := strings.ReplaceAll(logs[i], "'", "\"") + if expected != actual { + return false + } + } + + return true + }) + assert.NoError(t, err) + assert.True(t, ok) +} diff --git a/test/metric/metric_list_query.go b/test/metric/metric_list_query.go index 3bc9099a4..f58b7efc0 100644 --- a/test/metric/metric_list_query.go +++ b/test/metric/metric_list_query.go @@ -46,3 +46,29 @@ func (n *MetricListFetcher) Fetch(namespace, metricName string, dimensions []typ return output.Metrics, nil } + +func (n *MetricListFetcher) FetchByDimension(namespace string, dimensions []types.Dimension) ([]types.Metric, error) { + var dims []types.DimensionFilter + for _, dim := range dimensions { + dims = append(dims, types.DimensionFilter{ + Name: dim.Name, + Value: dim.Value, + }) + } + + listMetricInput := cloudwatch.ListMetricsInput{ + Namespace: aws.String(namespace), + Dimensions: dims, + } + + log.Printf("Metric data input: namespace %v, dimensions %v", namespace, fmt.Sprint(&dims)) + + output, err := awsservice.CwmClient.ListMetrics(context.Background(), &listMetricInput) + if err != nil { + return nil, fmt.Errorf("Error getting metric data %v", err) + } + + log.Printf("Metrics fetched : %v", output.Metrics) + + return output.Metrics, nil +} diff --git a/test/metric/metric_value_query.go b/test/metric/metric_value_query.go index eea358074..402953932 100644 --- a/test/metric/metric_value_query.go +++ b/test/metric/metric_value_query.go @@ -51,7 +51,7 @@ func (n *MetricValueFetcher) Fetch(namespace, metricName string, metricSpecificD } endTime := time.Now() - startTime := subtractMinutes(endTime, 10) + startTime := subtractMinutes(endTime, 2) getMetricDataInput := cloudwatch.GetMetricDataInput{ StartTime: &startTime, EndTime: &endTime, diff --git a/test/metric/stat.go b/test/metric/stat.go index 763028566..d633985d3 100644 --- a/test/metric/stat.go +++ b/test/metric/stat.go @@ -13,4 +13,6 @@ const ( MAXUMUM Statistics = "Maxmimum" SUM Statistics = "Sum" HighResolutionStatPeriod = 10 + + MinuteStatPeriod = 60 ) diff --git a/test/metric_value_benchmark/eks_daemonset_test.go b/test/metric_value_benchmark/eks_daemonset_test.go index 35e4298a7..b131f0d3f 100644 --- a/test/metric_value_benchmark/eks_daemonset_test.go +++ b/test/metric_value_benchmark/eks_daemonset_test.go @@ -9,19 +9,18 @@ import ( "encoding/json" "errors" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "log" + "strings" "time" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "golang.org/x/exp/slices" - "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/test/metric" - "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/metric_value_benchmark/eks_resources" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" ) const containerInsightsNamespace = "ContainerInsights" @@ -35,12 +34,14 @@ type EKSDaemonTestRunner struct { } func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult { - metrics := e.GetMeasuredMetrics() testResults := make([]status.TestResult, 0) - for _, name := range metrics { - testResults = append(testResults, e.validateInstanceMetrics(name)) + testMap := e.getMetricsInClusterDimension() + for dim, metrics := range eks_resources.DimensionStringToMetricsMap { + testResults = append(testResults, e.validateMetricsAvailability(dim, metrics, testMap)) + for _, m := range metrics { + testResults = append(testResults, e.validateMetricData(m, e.translateDimensionStringToDimType(dim))) + } } - testResults = append(testResults, e.validateLogs(e.env)) return status.TestGroupResult{ Name: e.GetTestName(), @@ -48,54 +49,117 @@ func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult { } } -func (e *EKSDaemonTestRunner) validateInstanceMetrics(name string) status.TestResult { +type void struct{} + +func (e *EKSDaemonTestRunner) getMetricsInClusterDimension() map[string]map[string]void { + listFetcher := metric.MetricListFetcher{} + log.Printf("Fetching by cluster dimension") + actualMetrics, err := listFetcher.FetchByDimension(containerInsightsNamespace, e.translateDimensionStringToDimType("ClusterName")) + if err != nil { + log.Println("failed to fetch metric list", err) + return nil + } + + if len(actualMetrics) < 1 { + log.Println("cloudwatch metric list is empty") + return nil + } + log.Printf("length of metrics %d", len(actualMetrics)) + testMap := make(map[string]map[string]void) + for _, m := range actualMetrics { + var s string + for i, d := range m.Dimensions { + if i == len(m.Dimensions)-1 { + s += *d.Name + break + } + s += *d.Name + "-" + } + log.Printf("for dimension string %s", s) + if testMap == nil || testMap[s] == nil { + mtr := make(map[string]void) + mtr[*m.MetricName] = void{} + testMap[s] = mtr + } else { + testMap[s][*m.MetricName] = void{} + } + } + + return testMap +} + +func logMetrics(metrics map[string]void, dim string) { + log.Printf("\t dimension: %s, Metrics:\n", dim) + for d, _ := range metrics { + log.Printf("metric name: %s", d) + } +} + +func (e *EKSDaemonTestRunner) validateMetricsAvailability(dimensionString string, metrics []string, testMap map[string]map[string]void) status.TestResult { + log.Printf("validateMetricsAvailability for dimension: %v", dimensionString) testResult := status.TestResult{ - Name: name, + Name: dimensionString, Status: status.FAILED, } - - dims, failed := e.DimensionFactory.GetDimensions([]dimension.Instruction{ - { - Key: "ClusterName", - Value: dimension.UnknownDimensionValue(), - }, - }) - if len(failed) > 0 { - log.Println("failed to get dimensions") - return testResult + actualMetrics := testMap[dimensionString] + + //verify the result metrics with expected metrics + log.Printf("length of actual metrics %d", len(actualMetrics)) + log.Printf("length of expected metrics %d", len(metrics)) + logMetrics(actualMetrics, dimensionString) + if compareMaptoList(actualMetrics, metrics) { + testResult.Status = status.SUCCESSFUL } + return testResult +} - // get list of metrics that has more dimensions for container insights - // this is to avoid adding more dimension provider for non-trivial dimensions e.g. PodName - listFetcher := metric.MetricListFetcher{} - if slices.Contains(metricsWithMoreDimensions, name) { - metrics, err := listFetcher.Fetch(containerInsightsNamespace, name, dims) - if err != nil { - log.Println("failed to fetch metric list", err) - return testResult - } +func (e *EKSDaemonTestRunner) translateDimensionStringToDimType(dimensionString string) []types.Dimension { + split := strings.Split(dimensionString, "-") + var dims []types.Dimension + for _, str := range split { + dims = append(dims, types.Dimension{ + Name: aws.String(str), + Value: aws.String(e.getDimensionValue(str)), + }) + log.Printf("dim key %s", str) + log.Printf("dim value %s", e.getDimensionValue(str)) + } + log.Printf("dimensions length %d", len(dims)) + return dims +} - if len(metrics) < 1 { - log.Println("metric list is empty") - return testResult - } +func (e *EKSDaemonTestRunner) getDimensionValue(dim string) string { + switch dim { + case "ClusterName": + return e.env.EKSClusterName + case "Namespace": + return "amazon-cloudwatch" + default: + return "" + } +} - // just verify 1 of returned metrics for values - for _, dim := range metrics[0].Dimensions { - // skip since it's provided by dimension provider - if *dim.Name == "ClusterName" { - continue - } +func compareMaptoList(metricMap map[string]void, metricList []string) bool { + if len(metricMap) != len(metricList) { + return false + } - dims = append(dims, types.Dimension{ - Name: dim.Name, - Value: dim.Value, - }) + for _, key := range metricList { + if _, ok := metricMap[key]; !ok { + return false } } + return true +} +func (e *EKSDaemonTestRunner) validateMetricData(name string, dims []types.Dimension) status.TestResult { + log.Printf("validateMetricData with metric: %s", name) + testResult := status.TestResult{ + Name: name, + Status: status.FAILED, + } valueFetcher := metric.MetricValueFetcher{} - values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.AVERAGE, metric.HighResolutionStatPeriod) + values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.SAMPLE_COUNT, metric.MinuteStatPeriod) if err != nil { log.Println("failed to fetch metrics", err) return testResult diff --git a/test/metric_value_benchmark/eks_resources/util.go b/test/metric_value_benchmark/eks_resources/util.go index d4e73912a..3c94c80e4 100644 --- a/test/metric_value_benchmark/eks_resources/util.go +++ b/test/metric_value_benchmark/eks_resources/util.go @@ -3,7 +3,10 @@ package eks_resources -import _ "embed" +import ( + _ "embed" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" +) var ( //go:embed test_schemas/cluster.json @@ -52,3 +55,13 @@ var ( "PodNet": eksPodNetSchema, } ) + +type DimensionsToMetricsMap struct { + Dims []types.Dimension + Metrics []string +} + +var DimensionStringToMetricsMap = map[string][]string{ + "ClusterName-Namespace": {"pod_cpu_utilization", "pod_memory_utilization", "pod_network_rx_bytes", "pod_network_tx_bytes", + "pod_cpu_utilization_over_pod_limit", "pod_memory_utilization_over_pod_limit", "pod_interface_network_rx_dropped", "pod_interface_network_tx_dropped", "namespace_number_of_running_pods"}, +}