[APM] High Cardinality: Integration testing

.github/workflows/build-check.yml

@@ -25,6 +25,9 @@ jobs:
       - name: Lint Terraform
         if: runner.os == 'Linux'
         run: cd terraform && make check-fmt
+      - name: Lint Go
+        if: runner.os == 'Linux'
+        run: make simple-lint
       - name: Compile tests
         run: |
           echo "Compile tests"

Makefile

@@ -1,8 +1,16 @@
 BASE_SPACE=$(shell pwd)
 BUILD_SPACE=$(BASE_SPACE)/build
 
+IMPORT_PATH=github.com/aws/amazon-cloudwatch-agent-test
+ALL_SRC := $(shell find . -name '*.go' -type f | sort)
 TOOLS_BIN_DIR := $(abspath ./build/tools)
+
+GOIMPORTS = $(TOOLS_BIN_DIR)/goimports
 LINTER = $(TOOLS_BIN_DIR)/golangci-lint
+IMPI = $(TOOLS_BIN_DIR)/impi
+ADDLICENSE = $(TOOLS_BIN_DIR)/addlicense
+
+GOIMPORTS_OPT?= -w -local $(IMPORT_PATH)
 
 WIN_BUILD = GOOS=windows GOARCH=amd64 go build -trimpath -o $(BUILD_SPACE)
 LINUX_AMD64_BUILD = CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -o $(BUILD_SPACE)
@@ -22,14 +30,56 @@ LOADGEN_LINUX_ARM64_BUILD = $(LINUX_ARM64_BUILD)/linux/arm64
 LOADGEN_DARWIN_AMD64_BUILD = $(DARWIN_AMD64_BUILD)/darwin/amd64
 LOADGEN_DARWIN_ARM64_BUILD = $(DARWIN_ARM64_BUILD)/darwin/arm64
 
-install-tools:
+install-goimports:
+	GOBIN=$(TOOLS_BIN_DIR) go install golang.org/x/tools/cmd/goimports
+
+install-impi:
+	GOBIN=$(TOOLS_BIN_DIR) go install github.com/pavius/impi/cmd/[email protected]
+
+install-addlicense:
+	# Using 04bfe4e to get SPDX template changes that are not present in the most recent tag v1.0.0
+	# This is required to be able to easily omit the year in our license header.
+	GOBIN=$(TOOLS_BIN_DIR) go install github.com/google/addlicense@04bfe4e
+
+install-golang-lint:
 	#Install from source for golangci-lint is not recommended based on https://golangci-lint.run/usage/install/#install-from-source so using binary
 	#installation
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN_DIR) v1.50.1
 
-lint: install-tools
+fmt: install-goimports addlicense
+	go fmt ./...
+	@echo $(ALL_SRC) | xargs -n 10 $(GOIMPORTS) $(GOIMPORTS_OPT)
+
+impi: install-impi
+	@echo $(ALL_SRC) | xargs -n 10 $(IMPI) --local $(IMPORT_PATH) --scheme stdThirdPartyLocal
+	@echo "Check import order/grouping finished"
+
+simple-lint: checklicense impi
+
+lint: install-golang-lint simple-lint
 	${LINTER} run ./...
 
+addlicense: install-addlicense
+	@ADDLICENSEOUT=`$(ADDLICENSE) -y="" -s=only -l="mit" -c="Amazon.com, Inc. or its affiliates. All Rights Reserved." $(ALL_SRC) 2>&1`; \
+    		if [ "$$ADDLICENSEOUT" ]; then \
+    			echo "$(ADDLICENSE) FAILED => add License errors:\n"; \
+    			echo "$$ADDLICENSEOUT\n"; \
+    			exit 1; \
+    		else \
+    			echo "Add License finished successfully"; \
+    		fi
+
+checklicense: install-addlicense
+	@ADDLICENSEOUT=`$(ADDLICENSE) -check $(ALL_SRC) 2>&1`; \
+    		if [ "$$ADDLICENSEOUT" ]; then \
+    			echo "$(ADDLICENSE) FAILED => add License errors:\n"; \
+    			echo "$$ADDLICENSEOUT\n"; \
+    			echo "Use 'make addlicense' to fix this."; \
+    			exit 1; \
+    		else \
+    			echo "Check License finished successfully"; \
+    		fi
+
 compile:
 	# this is a workaround to compile and cache all of the tests without actually running any of them
 	go test -run=NO_MATCH ./...

cmd/emf-generator/emf-generator.go

@@ -1,3 +1,6 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
 package main
 
 import (

cmd/log-generator/log-generator.go

@@ -1,3 +1,6 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
 package main
 
 import (

cmd/statsd-generator/statsd-generator.go

@@ -1,3 +1,6 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
 package main
 
 import (

filesystem/unix_permission.go

@@ -7,8 +7,9 @@ package filesystem
 
 import (
 	"fmt"
-	"golang.org/x/sys/unix"
 	"os/user"
+
+	"golang.org/x/sys/unix"
 )
 
 type FilePermission string

filesystem/windows_permission.go

@@ -7,9 +7,10 @@ package filesystem
 
 import (
 	"fmt"
-	"golang.org/x/sys/windows"
 	"os"
 	"unsafe"
+
+	"golang.org/x/sys/windows"
 )
 
 // CheckFileRights check that the given filename has access controls and system permission for Administrator, Local System

generator/resources/ecs_ec2_daemon_test_matrix.json

@@ -2,6 +2,13 @@
   {
     "instanceType":"t3a.xlarge",
     "ami": "cloudwatch-agent-integration-test-al2*",
-    "family": "linux"
+    "family": "linux",
+    "metadataEnabled": "enabled"
+  },
+  {
+    "instanceType":"t3a.xlarge",
+    "ami": "cloudwatch-agent-integration-test-al2*",
+    "family": "linux",
+    "metadataEnabled": "disabled"
   }
 ]

generator/test_case_generator.go

@@ -31,16 +31,21 @@ type matrixRow struct {
 	TerraformDir        string `json:"terraform_dir"`
 	UseSSM              bool   `json:"useSSM"`
 	ExcludedTests       string `json:"excludedTests"`
+	MetadataEnabled     string `json:"metadataEnabled"`
+	MaxAttempts         int    `json:"max_attempts"`
 }
 
 type testConfig struct {
 	// this gives more flexibility to define terraform dir when there should be a different set of terraform files
 	// e.g. statsd can have a multiple terraform module sets for difference test scenarios (ecs, eks or ec2)
-	testDir      string
-	terraformDir string
+	testDir       string
+	terraformDir  string
+	runMockServer bool
 	// define target matrix field as set(s)
 	// empty map means a testConfig will be created with a test entry for each entry from *_test_matrix.json
 	targets map[string]map[string]struct{}
+	// maxAttempts limits the number of times a test will be run.
+	maxAttempts int
 }
 
 const (
@@ -59,6 +64,11 @@ var testTypeToTestConfig = map[string][]testConfig{
 			testDir: "./test/metrics_number_dimension",
 			targets: map[string]map[string]struct{}{"os": {"al2": {}}},
 		},
+		{
+			testDir:     "./test/emf_concurrent",
+			targets:     map[string]map[string]struct{}{"os": {"al2": {}}},
+			maxAttempts: 1,
+		},
 		{testDir: "./test/metric_value_benchmark"},
 		{testDir: "./test/run_as_user"},
 		{testDir: "./test/collection_interval"},
@@ -111,6 +121,7 @@ var testTypeToTestConfig = map[string][]testConfig{
 		{testDir: "../../../test/feature/windows"},
 		{testDir: "../../../test/restart"},
 		{testDir: "../../../test/acceptance"},
+		{testDir: "../../../test/feature/windows/event_logs"},
 		// assume role test doesn't add much value, and it already being tested with linux
 		//{testDir: "../../../test/assume_role"},
 	},
@@ -120,6 +131,7 @@ var testTypeToTestConfig = map[string][]testConfig{
 		{testDir: "../../test/performance/system"},
 		{testDir: "../../test/performance/statsd"},
 		{testDir: "../../test/performance/collectd"},
+		{testDir: "../../test/performance/trace/xray", runMockServer: true},
 	},
 	"ec2_windows_performance": {
 		{testDir: "../../test/performance/windows/logs"},
@@ -140,9 +152,22 @@ var testTypeToTestConfig = map[string][]testConfig{
 		{testDir: "./test/ecs/ecs_metadata"},
 	},
 	"ecs_ec2_daemon": {
-		{testDir: "./test/metric_value_benchmark"},
-		{testDir: "./test/statsd"},
-		{testDir: "./test/emf"},
+		{
+			testDir: "./test/metric_value_benchmark",
+			targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}},
+		},
+		{
+			testDir: "./test/statsd",
+			targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}},
+		},
+		{
+			testDir: "./test/emf",
+			targets: map[string]map[string]struct{}{"metadataEnabled": {"disabled": {}}},
+		},
+		{
+			testDir: "./test/emf",
+			targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}},
+		},
 	},
 	"eks_daemon": {
 		{
@@ -162,6 +187,24 @@ var testTypeToTestConfig = map[string][]testConfig{
 			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
 		},
 		{testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"},
+		{testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals",
+			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
+		},
+		{
+			testDir:      "./test/app_signals/high_cardinality_drop",
+			terraformDir: "terraform/eks/daemon/app_signals",
+			targets:      map[string]map[string]struct{}{"arc": {"amd64": {}}},
+		},
+		{
+			testDir:      "./test/app_signals/high_cardinality_keep",
+			terraformDir: "terraform/eks/daemon/app_signals",
+			targets:      map[string]map[string]struct{}{"arc": {"amd64": {}}},
+		},
+		{
+			testDir:      "./test/app_signals/high_cardinality_replace",
+			terraformDir: "terraform/eks/daemon/app_signals",
+			targets:      map[string]map[string]struct{}{"arc": {"amd64": {}}},
+		},
 	},
 	"eks_deployment": {
 		{testDir: "./test/metric_value_benchmark"},
@@ -170,7 +213,7 @@ var testTypeToTestConfig = map[string][]testConfig{
 
 func copyAllEC2LinuxTestForOnpremTesting() {
 	/* Some tests need to be fixed in order to run in both environment, so for now for PoC, run one that works.
-	testTypeToTestConfig["ec2_linux_onprem"] = testTypeToTestConfig[testTypeKeyEc2Linux]
+	   testTypeToTestConfig["ec2_linux_onprem"] = testTypeToTestConfig[testTypeKeyEc2Linux]
 	*/
 	testTypeToTestConfig["ec2_linux_onprem"] = []testConfig{
 		{
@@ -209,7 +252,12 @@ func genMatrix(testType string, testConfigs []testConfig) []matrixRow {
 	testMatrixComplete := make([]matrixRow, 0, len(testMatrix))
 	for _, test := range testMatrix {
 		for _, testConfig := range testConfigs {
-			row := matrixRow{TestDir: testConfig.testDir, TestType: testType, TerraformDir: testConfig.terraformDir}
+			row := matrixRow{
+				TestDir:      testConfig.testDir,
+				TestType:     testType,
+				TerraformDir: testConfig.terraformDir,
+				MaxAttempts:  testConfig.maxAttempts,
+			}
 			err = mapstructure.Decode(test, &row)
 			if err != nil {
 				log.Panicf("can't decode map test %v to metric line struct with error %v", testConfig, err)
@@ -232,6 +280,8 @@ func shouldAddTest(row *matrixRow, targets map[string]map[string]struct{}) bool
 			rowVal = row.Arc
 		} else if key == "os" {
 			rowVal = row.Os
+		} else if key == "metadataEnabled" {
+			rowVal = row.MetadataEnabled
 		}
 
 		if rowVal == "" {

install/install_agent.go

@@ -11,16 +11,16 @@ import (
 )
 
 const (
-	retryNumber       = 10
-	retryTime         = 30 * time.Second
-	debInstall        = "deb"
-	rpmInstall        = "rpm"
+	retryNumber = 10
+	retryTime   = 30 * time.Second
+	debInstall  = "deb"
+	rpmInstall  = "rpm"
 )
 
 func main() {
 	installType := os.Args[1]
 	installCommand := ""
-	
+
 	debInstallCommand := "sudo dpkg -i -E ./amazon-cloudwatch-agent.deb"
 	rpmInstallCommand := "sudo rpm -U ./amazon-cloudwatch-agent.rpm"
 	if os.Geteuid() == 0 {

mockserver/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.19 AS build
+WORKDIR $GOPATH/main
+COPY . .
+RUN go env -w GOPROXY=direct
+RUN GO111MODULE=on go mod download
+RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -o=/bin/main .
+EXPOSE 80 
+EXPOSE 443
+FROM scratch
+ENV AWS_REGION="us-west-2"
+COPY --from=build /bin/main /bin/main
+COPY certificates certificates
+ENTRYPOINT ["/bin/main"]

mockserver/Makefile

@@ -0,0 +1,26 @@
+# Header above mocked_servers cert in bundle
+CERT_HEADER=mockserver
+
+CONFIG_PATH=openssl.conf
+CERT_PATH=certificates/ssl/certificate.crt
+KEY_PATH=certificates/private.key
+BUNDLE_PATH=certificates/ssl/ca-bundle.crt
+.PHONY: clean
+clean: 
+	rm -rf $(CERT_PATH) $(BUNDLE_PATH) $(KEY_PATH)
+	touch $(CERT_PATH) $(BUNDLE_PATH) $(KEY_PATH)
+.PHONY: update-certs
+update-certs: clean gen-cert update-bundle
+
+# Expects mocked_servers cert to be the last cert in the bundle
+# Cuts until the first instance of "mocked_servers" in the bundle
+# and concatenates it with the current cert
+.PHONY: update-bundle
+update-bundle:
+	sed /$(CERT_HEADER)/q $(BUNDLE_PATH) | cat - $(CERT_PATH) > $(BUNDLE_PATH).tmp && \
+	mv $(BUNDLE_PATH).tmp $(BUNDLE_PATH)
+
+# Generates the annual cert and private key using the config
+.PHONY: gen-cert
+gen-cert:
+	openssl req -config $(CONFIG_PATH) -new -x509 -nodes -days 365 -out $(CERT_PATH) -keyout $(KEY_PATH)

mockserver/README.md

@@ -0,0 +1,41 @@
+# The Mock Server
+
+## Overview
+
+The Mock Server is a simple server designed for receiving metric and trace data, providing a simulated endpoint for testing purposes. It listens on two separate ports: 8080 and 443.
+## Running the server
+This server is runs as a docker container to run this server:
+1. First build the docker container with
+```sh
+sudo docker build -t mockserver . 
+```
+2. Run the container by mapping the ports you would like to use, for example:
+```sh
+sudo docker run --name mockserver -d -p 8080:8080 -p 443:443  mockserver
+```
+
+## How it Works
+### The Receiver
+
+The receiver component of the Mock Server operates on port 443. It is responsible for receiving messages and incrementing the transaction count. To simulate real-world conditions, there is a built-in 15ms latency between each received message. The data received can be sent to three possible routes:
+
+- **Check Receiver Status:** You can check if the receiver is alive by making a request to `/ping`.
+
+- **Send Data:** Use the `/put-data` route to send data. This route supports two sub-routes:
+    - `/put-data/trace/v1`: Use this sub-route for sending trace data.
+    - `/put-data/metrics`: Use this sub-route for sending metrics data.
+
+> [!Important]
+> Currently, both traces and metrics are handled in the same way.
+
+### The Verifier
+
+The verifier component can be accessed via a listener on port 8080. It provides information about the transactions, including:
+
+- **Transactions per Minute:** You can obtain the transactions per minute by making a request to `/tpm`.
+
+- **Transaction Count:** To check the total transaction count, use the `/check-data` route.
+
+- **Verifier Status:** Determine if the verification server is alive by sending a request to `/ping`.
+
+