Test ci (#392)

* added windows event log integration tests * added base test * extra logs * added dim value * testing with just cluster * test * test map * test metricData * stat period change * test metric sample count * test metric sample count * test metric sample count * changed the node scaling config to 2 * changed the node scaling config to 2 --------- Co-authored-by: Pooja Reddy Nathala <[email protected]>
aws · Feb 26, 2024 · 3dce60e · 3dce60e
1 parent 1b02a53
commit 3dce60e
Show file tree

Hide file tree

Showing 9 changed files with 262 additions and 62 deletions.
diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go
@@ -187,22 +187,6 @@ var testTypeToTestConfig = map[string][]testConfig{
 			testDir: "./test/metric_value_benchmark",
 			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
 		},
-		{
-			testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd",
-			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
-		},
-		{
-			testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf",
-			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
-		},
-		{
-			testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d",
-			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
-		},
-		{testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"},
-		{testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals",
-			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
-		},
 	},
 	"eks_deployment": {
 		{testDir: "./test/metric_value_benchmark"},

diff --git a/terraform/eks/daemon/main.tf b/terraform/eks/daemon/main.tf
@@ -408,12 +408,24 @@ resource "kubernetes_cluster_role_binding" "rolebinding" {
   }
 }
 
+resource "time_sleep" "wait_2_min" {
+  depends_on = [
+    aws_eks_node_group.this,
+    kubernetes_daemonset.service,
+    kubernetes_cluster_role_binding.rolebinding,
+    kubernetes_service_account.cwagentservice
+  ]
+
+  create_duration = "2m"
+}
+
 resource "null_resource" "validator" {
   depends_on = [
     aws_eks_node_group.this,
     kubernetes_daemonset.service,
     kubernetes_cluster_role_binding.rolebinding,
     kubernetes_service_account.cwagentservice,
+    time_sleep.wait_2_min
   ]
   provisioner "local-exec" {
     command = <<-EOT

diff --git a/test/cloudwatchlogs/resources/config_windows_event_log.json b/test/cloudwatchlogs/resources/config_windows_event_log.json
@@ -0,0 +1,34 @@
+{
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {
+            "file_path": "C:\\Users\\Administrator\\Desktop\\CWMetricsLogs",
+            "log_group_name": "CWMetricsLogs",
+            "log_stream_name": "{instance_id}",
+            "retention_in_days": -1
+          }
+        ]
+      },
+      "windows_events": {
+        "collect_list": [
+          {
+            "event_format": "xml",
+            "event_levels": [
+              "VERBOSE",
+              "INFORMATION",
+              "WARNING",
+              "ERROR",
+              "CRITICAL"
+            ],
+            "event_name": "System",
+            "log_group_name": "CloudWatchAgent",
+            "log_stream_name": "{instance_id}",
+            "retention_in_days": -1
+          }
+        ]
+      }
+    }
+  }
+}
diff --git a/test/cloudwatchlogs/windows_event_log_test.go b/test/cloudwatchlogs/windows_event_log_test.go
@@ -0,0 +1,65 @@
+//go:build windows
+
+package cloudwatchlogs
+
+import (
+	"fmt"
+	"log"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/amazon-cloudwatch-agent-test/internal/awsservice"
+	"github.com/aws/amazon-cloudwatch-agent-test/internal/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWindowsEventLog(t *testing.T) {
+	cfgFilePath := "resources/config_windows_event_log.json"
+
+	instanceId := awsservice.GetInstanceId()
+	log.Printf("Found instance id %s", instanceId)
+	logGroup := "CloudWatchAgent"
+	logStream := instanceId
+
+	start := time.Now()
+	common.CopyFile(cfgFilePath, configOutputPath)
+
+	common.StartAgent(configOutputPath, true)
+
+	// ensure that there is enough time from the "start" time and the first log line,
+	// so we don't miss it in the GetLogEvents call
+	time.Sleep(agentRuntime)
+	t.Log("Writing logs from windows event log plugin")
+	time.Sleep(agentRuntime)
+	common.StopAgent()
+
+	lines := []string{
+		fmt.Sprintf("{\"Metric\": \"%s\"}", strings.Repeat("12345", 10)),
+		fmt.Sprintf("{\"Metric\": \"%s\"}", strings.Repeat("09876", 10)),
+		fmt.Sprintf("{\"Metric\": \"%s\"}", strings.Repeat("1234567890", 10)),
+	}
+
+	end := time.Now()
+
+	ok, err := awsservice.ValidateLogs(logGroup, logStream, &start, &end, func(logs []string) bool {
+		log.Printf("logs length: %s ", len(logs))
+		if len(logs) != len(lines) {
+			return false
+		}
+
+		for i := 0; i < len(logs); i++ {
+			log.Printf("lines[%s] is %s", i, lines[i])
+			log.Printf("logs[%s] is %s", i, logs[i])
+			expected := strings.ReplaceAll(lines[i], "'", "\"")
+			actual := strings.ReplaceAll(logs[i], "'", "\"")
+			if expected != actual {
+				return false
+			}
+		}
+
+		return true
+	})
+	assert.NoError(t, err)
+	assert.True(t, ok)
+}
diff --git a/test/metric/metric_list_query.go b/test/metric/metric_list_query.go
@@ -46,3 +46,29 @@ func (n *MetricListFetcher) Fetch(namespace, metricName string, dimensions []typ
 
 	return output.Metrics, nil
 }
+
+func (n *MetricListFetcher) FetchByDimension(namespace string, dimensions []types.Dimension) ([]types.Metric, error) {
+	var dims []types.DimensionFilter
+	for _, dim := range dimensions {
+		dims = append(dims, types.DimensionFilter{
+			Name:  dim.Name,
+			Value: dim.Value,
+		})
+	}
+
+	listMetricInput := cloudwatch.ListMetricsInput{
+		Namespace:  aws.String(namespace),
+		Dimensions: dims,
+	}
+
+	log.Printf("Metric data input: namespace %v, dimensions %v", namespace, fmt.Sprint(&dims))
+
+	output, err := awsservice.CwmClient.ListMetrics(context.Background(), &listMetricInput)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting metric data %v", err)
+	}
+
+	log.Printf("Metrics fetched : %v", output.Metrics)
+
+	return output.Metrics, nil
+}
diff --git a/test/metric/metric_value_query.go b/test/metric/metric_value_query.go
@@ -51,7 +51,7 @@ func (n *MetricValueFetcher) Fetch(namespace, metricName string, metricSpecificD
 	}
 
 	endTime := time.Now()
-	startTime := subtractMinutes(endTime, 10)
+	startTime := subtractMinutes(endTime, 2)
 	getMetricDataInput := cloudwatch.GetMetricDataInput{
 		StartTime:         &startTime,
 		EndTime:           &endTime,

diff --git a/test/metric/stat.go b/test/metric/stat.go
@@ -13,4 +13,6 @@ const (
 	MAXUMUM                  Statistics = "Maxmimum"
 	SUM                      Statistics = "Sum"
 	HighResolutionStatPeriod            = 10
+
+	MinuteStatPeriod = 60
 )
diff --git a/test/metric_value_benchmark/eks_daemonset_test.go b/test/metric_value_benchmark/eks_daemonset_test.go
@@ -9,19 +9,18 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"log"
+	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
-	"golang.org/x/exp/slices"
-
 	"github.com/aws/amazon-cloudwatch-agent-test/environment"
 	"github.com/aws/amazon-cloudwatch-agent-test/test/metric"
-	"github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension"
 	"github.com/aws/amazon-cloudwatch-agent-test/test/metric_value_benchmark/eks_resources"
 	"github.com/aws/amazon-cloudwatch-agent-test/test/status"
 	"github.com/aws/amazon-cloudwatch-agent-test/test/test_runner"
 	"github.com/aws/amazon-cloudwatch-agent-test/util/awsservice"
+	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 )
 
 const containerInsightsNamespace = "ContainerInsights"
@@ -35,67 +34,132 @@ type EKSDaemonTestRunner struct {
 }
 
 func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult {
-	metrics := e.GetMeasuredMetrics()
 	testResults := make([]status.TestResult, 0)
-	for _, name := range metrics {
-		testResults = append(testResults, e.validateInstanceMetrics(name))
+	testMap := e.getMetricsInClusterDimension()
+	for dim, metrics := range eks_resources.DimensionStringToMetricsMap {
+		testResults = append(testResults, e.validateMetricsAvailability(dim, metrics, testMap))
+		for _, m := range metrics {
+			testResults = append(testResults, e.validateMetricData(m, e.translateDimensionStringToDimType(dim)))
+		}
 	}
-
 	testResults = append(testResults, e.validateLogs(e.env))
 	return status.TestGroupResult{
 		Name:        e.GetTestName(),
 		TestResults: testResults,
 	}
 }
 
-func (e *EKSDaemonTestRunner) validateInstanceMetrics(name string) status.TestResult {
+type void struct{}
+
+func (e *EKSDaemonTestRunner) getMetricsInClusterDimension() map[string]map[string]void {
+	listFetcher := metric.MetricListFetcher{}
+	log.Printf("Fetching by cluster dimension")
+	actualMetrics, err := listFetcher.FetchByDimension(containerInsightsNamespace, e.translateDimensionStringToDimType("ClusterName"))
+	if err != nil {
+		log.Println("failed to fetch metric list", err)
+		return nil
+	}
+
+	if len(actualMetrics) < 1 {
+		log.Println("cloudwatch metric list is empty")
+		return nil
+	}
+	log.Printf("length of metrics %d", len(actualMetrics))
+	testMap := make(map[string]map[string]void)
+	for _, m := range actualMetrics {
+		var s string
+		for i, d := range m.Dimensions {
+			if i == len(m.Dimensions)-1 {
+				s += *d.Name
+				break
+			}
+			s += *d.Name + "-"
+		}
+		log.Printf("for dimension string %s", s)
+		if testMap == nil || testMap[s] == nil {
+			mtr := make(map[string]void)
+			mtr[*m.MetricName] = void{}
+			testMap[s] = mtr
+		} else {
+			testMap[s][*m.MetricName] = void{}
+		}
+	}
+
+	return testMap
+}
+
+func logMetrics(metrics map[string]void, dim string) {
+	log.Printf("\t dimension: %s, Metrics:\n", dim)
+	for d, _ := range metrics {
+		log.Printf("metric name: %s", d)
+	}
+}
+
+func (e *EKSDaemonTestRunner) validateMetricsAvailability(dimensionString string, metrics []string, testMap map[string]map[string]void) status.TestResult {
+	log.Printf("validateMetricsAvailability for dimension: %v", dimensionString)
 	testResult := status.TestResult{
-		Name:   name,
+		Name:   dimensionString,
 		Status: status.FAILED,
 	}
-
-	dims, failed := e.DimensionFactory.GetDimensions([]dimension.Instruction{
-		{
-			Key:   "ClusterName",
-			Value: dimension.UnknownDimensionValue(),
-		},
-	})
-	if len(failed) > 0 {
-		log.Println("failed to get dimensions")
-		return testResult
+	actualMetrics := testMap[dimensionString]
+
+	//verify the result metrics with expected metrics
+	log.Printf("length of actual metrics %d", len(actualMetrics))
+	log.Printf("length of expected metrics %d", len(metrics))
+	logMetrics(actualMetrics, dimensionString)
+	if compareMaptoList(actualMetrics, metrics) {
+		testResult.Status = status.SUCCESSFUL
 	}
+	return testResult
+}
 
-	// get list of metrics that has more dimensions for container insights
-	// this is to avoid adding more dimension provider for non-trivial dimensions e.g. PodName
-	listFetcher := metric.MetricListFetcher{}
-	if slices.Contains(metricsWithMoreDimensions, name) {
-		metrics, err := listFetcher.Fetch(containerInsightsNamespace, name, dims)
-		if err != nil {
-			log.Println("failed to fetch metric list", err)
-			return testResult
-		}
+func (e *EKSDaemonTestRunner) translateDimensionStringToDimType(dimensionString string) []types.Dimension {
+	split := strings.Split(dimensionString, "-")
+	var dims []types.Dimension
+	for _, str := range split {
+		dims = append(dims, types.Dimension{
+			Name:  aws.String(str),
+			Value: aws.String(e.getDimensionValue(str)),
+		})
+		log.Printf("dim key %s", str)
+		log.Printf("dim value %s", e.getDimensionValue(str))
+	}
+	log.Printf("dimensions length %d", len(dims))
+	return dims
+}
 
-		if len(metrics) < 1 {
-			log.Println("metric list is empty")
-			return testResult
-		}
+func (e *EKSDaemonTestRunner) getDimensionValue(dim string) string {
+	switch dim {
+	case "ClusterName":
+		return e.env.EKSClusterName
+	case "Namespace":
+		return "amazon-cloudwatch"
+	default:
+		return ""
+	}
+}
 
-		// just verify 1 of returned metrics for values
-		for _, dim := range metrics[0].Dimensions {
-			// skip since it's provided by dimension provider
-			if *dim.Name == "ClusterName" {
-				continue
-			}
+func compareMaptoList(metricMap map[string]void, metricList []string) bool {
+	if len(metricMap) != len(metricList) {
+		return false
+	}
 
-			dims = append(dims, types.Dimension{
-				Name:  dim.Name,
-				Value: dim.Value,
-			})
+	for _, key := range metricList {
+		if _, ok := metricMap[key]; !ok {
+			return false
 		}
 	}
+	return true
+}
 
+func (e *EKSDaemonTestRunner) validateMetricData(name string, dims []types.Dimension) status.TestResult {
+	log.Printf("validateMetricData with metric: %s", name)
+	testResult := status.TestResult{
+		Name:   name,
+		Status: status.FAILED,
+	}
 	valueFetcher := metric.MetricValueFetcher{}
-	values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.AVERAGE, metric.HighResolutionStatPeriod)
+	values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.SAMPLE_COUNT, metric.MinuteStatPeriod)
 	if err != nil {
 		log.Println("failed to fetch metrics", err)
 		return testResult