test: add test support for k8s metrics (#249)

newrelic · Feb 25, 2025 · 141b6b8 · 141b6b8
1 parent ebf47ee
commit 141b6b8
Show file tree

Hide file tree

Showing 29 changed files with 658 additions and 439 deletions.
diff --git a/.github/workflows/ci-base.yaml b/.github/workflows/ci-base.yaml
@@ -13,7 +13,7 @@ on:
       test_cluster_name:
         required: false
         type: string
-        default: 'ci-e2etest'
+        default: 'ci-e2etest-${{ github.run_id }}-${{ github.run_attempt }}'
     secrets:
       registry:
         required: false

diff --git a/.github/workflows/ci-nightly.yaml b/.github/workflows/ci-nightly.yaml
@@ -26,7 +26,7 @@ jobs:
     with:
       nightly: true
       distribution: ${{ matrix.distribution }}
-      test_cluster_name: 'ci-e2etest-nightly-${{ matrix.distribution }}'
+      test_cluster_name: 'ci-e2etest-nightly-${{ matrix.distribution }}-${{ github.run_id }}-${{ github.run_attempt }}'
     secrets:
       docker_hub_username: ${{ secrets.OTELCOMM_DOCKER_HUB_USERNAME }}
       docker_hub_password: ${{ secrets.OTELCOMM_DOCKER_HUB_PASSWORD }}

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -22,7 +22,7 @@ jobs:
     with:
       distribution: ${{ matrix.distribution }}
       # namespace by distro to avoid issues with cleanup (distro 1 still running tests while distro 2 cleans up cluster)
-      test_cluster_name: 'ci-e2etest-${{ matrix.distribution }}'
+      test_cluster_name: 'ci-e2etest-${{ matrix.distribution }}-${{ github.run_id }}-${{ github.run_attempt }}'
     secrets:
       docker_hub_username: ${{ secrets.OTELCOMM_DOCKER_HUB_USERNAME }}
       docker_hub_password: ${{ secrets.OTELCOMM_DOCKER_HUB_PASSWORD }}

diff --git a/distributions/nrdot-collector-host/test-spec.yaml b/distributions/nrdot-collector-host/test-spec.yaml
@@ -1,3 +1,8 @@
+slow:
+  testCaseSpecs:
+    - host
 nightly:
   ec2:
-    enabled: true
+    enabled: true
+  testCaseSpecs:
+    - host
diff --git a/distributions/nrdot-collector-k8s/test-spec.yaml b/distributions/nrdot-collector-k8s/test-spec.yaml
@@ -1,3 +1,9 @@
+slow:
+  testCaseSpecs:
+    - k8s
+    - host
 nightly:
   ec2:
-    enabled: false
+    enabled: false
+  testCaseSpecs:
+    - host
diff --git a/test/charts/mocked_backend/templates/daemonset.yaml b/test/charts/mocked_backend/templates/daemonset.yaml
@@ -2,7 +2,7 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: &app nrdot-collector
+  name: &app nrdot-collector-daemonset
   labels:
     app: *app
 spec:
@@ -32,13 +32,13 @@ spec:
           env:
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
               value: "http://validation-backend:4318"
-            - name: NEW_RELIC_MEMORY_LIMIT_MIB
-              value: "100"
-            # mock values
-                # k8s distro expects this to be present
+            - name: NEW_RELIC_LICENSE_KEY
+              value: "NR_LICENSE_KEY_PLACEHOLDER"
+            # k8s distro expects this to be present
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-            - name: NEW_RELIC_LICENSE_KEY
-              value: "NR_LICENSE_KEY_PLACEHOLDER"
+            # used to populate k8s.cluster.name
+            - name: K8S_CLUSTER_NAME
+              value: {{ .Values.clusterName }}
diff --git a/test/charts/mocked_backend/templates/deployment-collector.yaml b/test/charts/mocked_backend/templates/deployment-collector.yaml
@@ -0,0 +1,40 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: &app nrdot-collector-deployment
+  labels:
+    app: *app
+spec:
+  replicas: {{ .Values.image.repository | hasSuffix "k8s" | ternary 1 0 }}
+  selector:
+    matchLabels:
+      app: *app
+  template:
+    metadata:
+      labels:
+        app: *app
+    spec:
+      serviceAccountName: read-k8s-api-account
+      containers:
+        - name: *app
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+          args: ["--config", "/etc/nrdot-collector-k8s/config-deployment.yaml"]
+          ports:
+            - name: health
+              containerPort: 13133
+          env:
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: "http://validation-backend:4318"
+              # mock values
+              # k8s distro expects this to be present
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: NEW_RELIC_LICENSE_KEY
+              value: "NR_LICENSE_KEY_PLACEHOLDER"
+            # used to populate k8s.cluster.name
+            - name: K8S_CLUSTER_NAME
+              value: {{ .Values.clusterName }}
diff --git a/.../mocked_backend/templates/deployment.yaml → ...plates/deployment-validation-backend.yaml b/.../mocked_backend/templates/deployment.yaml → ...plates/deployment-validation-backend.yaml
diff --git a/test/charts/mocked_backend/templates/k8s-api-access.yaml b/test/charts/mocked_backend/templates/k8s-api-access.yaml
@@ -7,14 +7,16 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+# namespace suffix to avoid conflicts when installing chart multiple times
   name: read-k8s-api-role-{{ .Release.Namespace }}
-# required rules copied from:
+# if not mentioned otherwise, required rules copied from:
 # - k8seventsreceiever: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/k8seventsreceiver/README.md#rbac
 rules:
   - apiGroups:
       - ""
     resources:
       - events
+      - endpoints # prometheus/api_server
       - namespaces
       - namespaces/status
       - nodes

diff --git a/test/charts/mocked_backend/values.yaml b/test/charts/mocked_backend/values.yaml
@@ -3,3 +3,5 @@ image:
   tag: latest
   # Avoid accidentally pulling remote images in CI
   pullPolicy: Never
+
+clusterName: default-cluster-name
diff --git a/test/charts/nr_backend/templates/daemonset.yaml b/test/charts/nr_backend/templates/daemonset.yaml
@@ -2,7 +2,7 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: &app nrdot-collector
+  name: &app nrdot-collector-daemonset
   labels:
     app: *app
 spec:
@@ -39,15 +39,16 @@ spec:
                 secretKeyRef:
                   name: daemonset-secrets
                   key: backendUrl
-            - name: NEW_RELIC_MEMORY_LIMIT_MIB
-              value: "100"
             - name: NEW_RELIC_LICENSE_KEY
               valueFrom:
                 secretKeyRef:
                   name: daemonset-secrets
                   key: nrIngestKey
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: "host.name={{ .Values.collector.hostname }}-$(KUBE_NODE_NAME)"
+            # used to populate k8s.cluster.name
+            - name: K8S_CLUSTER_NAME
+              value: {{ .Values.clusterName }}
 ---
 apiVersion: v1
 kind: Secret

diff --git a/test/charts/nr_backend/templates/deployment-collector.yaml b/test/charts/nr_backend/templates/deployment-collector.yaml
@@ -0,0 +1,45 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: &app nrdot-collector-deployment
+  labels:
+    app: *app
+spec:
+  replicas: {{ .Values.image.repository | hasSuffix "k8s" | ternary 1 0 }}
+  selector:
+    matchLabels:
+      app: *app
+  template:
+    metadata:
+      labels:
+        app: *app
+    spec:
+      serviceAccountName: read-k8s-api-account
+      containers:
+        - name: *app
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+          args: ["--config", "/etc/nrdot-collector-k8s/config-deployment.yaml"]
+          ports:
+            - name: health
+              containerPort: 13133
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: daemonset-secrets
+                  key: backendUrl
+            - name: NEW_RELIC_LICENSE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: daemonset-secrets
+                  key: nrIngestKey
+            # used to populate k8s.cluster.name
+            - name: K8S_CLUSTER_NAME
+              value: {{ .Values.clusterName }}
+
diff --git a/test/charts/nr_backend/templates/k8s-api-access.yaml b/test/charts/nr_backend/templates/k8s-api-access.yaml
@@ -7,14 +7,16 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  # namespace suffix to avoid conflicts when installing chart multiple times
   name: read-k8s-api-role-{{ .Release.Namespace }}
-# required rules copied from:
+# if not mentioned otherwise, required rules copied from:
 # - k8seventsreceiever: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/k8seventsreceiver/README.md#rbac
 rules:
   - apiGroups:
       - ""
     resources:
       - events
+      - endpoints # prometheus/api_server
       - namespaces
       - namespaces/status
       - nodes

diff --git a/test/charts/nr_backend/values.yaml b/test/charts/nr_backend/values.yaml
@@ -9,4 +9,6 @@ secrets:
   nrIngestKey: PLACEHOLDER
 
 collector:
-  hostname: nrdot-collector-default-hostname
+  hostname: nrdot-collector-default-hostname
+
+clusterName: default-cluster-name
diff --git a/test/e2e/hostmetrics/hostmetrics_test.go b/test/e2e/hostmetrics/hostmetrics_test.go
@@ -31,7 +31,7 @@ type ValidationPayload struct {
 	Transactions     uint32 `json:"transactions"`
 }
 
-func TestStartupBehavior(t *testing.T) {
+func TestFast(t *testing.T) {
 	testutil.TagAsFastTest(t)
 	kubectlOptions = k8sutil.NewKubectlOptions(TestNamespace)
 	testChart = chart.NewMockedBackendChart()

diff --git a/test/e2e/hostmetrics_nightly/hostmetrics_nightly_test.go b/test/e2e/hostmetrics_nightly/hostmetrics_nightly_test.go
@@ -31,7 +31,7 @@ var k8sNode = spec.NightlySystemUnderTest{
 	ExcludedMetrics: []string{"system.paging.usage"},
 }
 
-func TestNightlyHostMetrics(t *testing.T) {
+func TestNightly(t *testing.T) {
 	testutil.TagAsNightlyTest(t)
 	testSpec := spec.LoadTestSpec()
 
@@ -45,18 +45,27 @@ func TestNightlyHostMetrics(t *testing.T) {
 			t.Logf("Skipping nightly system-under-test: %s", sut.HostNamePattern)
 			continue
 		}
-		for i, testCase := range spec.GetOnHostTestCasesWithout(sut.ExcludedMetrics) {
-			t.Run(fmt.Sprintf("%s/%d/%s", sut.HostNamePattern, i, testCase.Name), func(t *testing.T) {
-				t.Parallel()
-				assertionFactory := assert.NewNrMetricAssertionFactory(
-					fmt.Sprintf("WHERE host.name like '%s'", sut.HostNamePattern),
-					"2 hour ago",
-				)
-				assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
-				// space out requests to avoid rate limiting
-				time.Sleep(time.Duration(i) * requestSpacing)
-				assertion.ExecuteWithRetries(t, client, 15, 5*time.Second)
-			})
+		testEnvironment := map[string]string{
+			"hostName": sut.HostNamePattern,
+		}
+		for _, testCaseSpecName := range testSpec.Nightly.TestCaseSpecs {
+			testCaseSpec := spec.LoadTestCaseSpec(testCaseSpecName)
+			whereClause := testCaseSpec.RenderWhereClause(testEnvironment)
+			counter := 0
+			for caseName, testCase := range testCaseSpec.GetTestCasesWithout(sut.ExcludedMetrics) {
+				t.Run(fmt.Sprintf("%s/%s/%s", sut.HostNamePattern, testCaseSpecName, caseName), func(t *testing.T) {
+					t.Parallel()
+					assertionFactory := assert.NewNrMetricAssertionFactory(
+						whereClause,
+						"2 hour ago",
+					)
+					assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
+					// space out requests to avoid rate limiting
+					time.Sleep(time.Duration(counter) * requestSpacing)
+					assertion.ExecuteWithRetries(t, client, 24, 5*time.Second)
+				})
+				counter += 1
+			}
 		}
 	}
 }
diff --git a/test/e2e/hostmetrics_slow/hostmetrics_slow_test.go b/test/e2e/hostmetrics_slow/hostmetrics_slow_test.go
@@ -23,8 +23,10 @@ var (
 	testChart      chart.NrBackendChart
 )
 
-func TestStartupBehavior(t *testing.T) {
+func TestSlow(t *testing.T) {
 	testutil.TagAsSlowTest(t)
+	testSpec := spec.LoadTestSpec()
+
 	kubectlOptions = k8sutil.NewKubectlOptions(TestNamespace)
 	testId := testutil.NewTestId()
 	testChart = chart.NewNrBackendChart(testId)
@@ -39,17 +41,29 @@ func TestStartupBehavior(t *testing.T) {
 	requestSpacing := time.Duration((1/requestsPerSecond)*1000) * time.Millisecond
 	client := nr.NewClient()
 
-	for i, testCase := range spec.GetOnHostTestCases() {
-		t.Run(fmt.Sprintf(testCase.Name), func(t *testing.T) {
-			t.Parallel()
-			assertionFactory := assert.NewNrMetricAssertionFactory(
-				fmt.Sprintf("WHERE host.name like '%s'", testChart.NrQueryHostNamePattern),
-				"5 minutes ago",
-			)
-			assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
-			// space out requests to avoid rate limiting
-			time.Sleep(time.Duration(i) * requestSpacing)
-			assertion.ExecuteWithRetries(t, client, 15, 5*time.Second)
-		})
+	testEnvironment := map[string]string{
+		"hostName":    testChart.NrQueryHostNamePattern,
+		"clusterName": kubectlOptions.ContextName,
+	}
+	for _, testCaseSpecName := range testSpec.Slow.TestCaseSpecs {
+		testCaseSpec := spec.LoadTestCaseSpec(testCaseSpecName)
+		whereClause := testCaseSpec.RenderWhereClause(testEnvironment)
+		t.Logf("test case spec where clause: %s", whereClause)
+
+		counter := 0
+		for caseName, testCase := range testCaseSpec.TestCases {
+			t.Run(fmt.Sprintf("%s/%s", testCaseSpecName, caseName), func(t *testing.T) {
+				t.Parallel()
+				assertionFactory := assert.NewNrMetricAssertionFactory(
+					whereClause,
+					"5 minutes ago",
+				)
+				assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
+				// space out requests to avoid rate limiting
+				time.Sleep(time.Duration(counter) * requestSpacing)
+				assertion.ExecuteWithRetries(t, client, 24, 5*time.Second)
+			})
+			counter += 1
+		}
 	}
 }