[processor/k8sattributes] Add first version of k8s e2e tests (open-te…

…lemetry#17410)

Signed-off-by: Ziqi Zhao <[email protected]>
Co-authored-by: Dmitrii Anoshin <[email protected]>

.chloggen/k8s-components-e2e.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: k8sattributesprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add end to end test framework for Kubernetes related components
+
+# One or more tracking issues related to the change
+issues: [15651]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.github/workflows/e2e/k8s/collector-helm-values.yml

@@ -0,0 +1,38 @@
+mode: "statefulset"
+
+presets:
+  kubernetesAttributes:
+    enabled: true
+
+config:
+  exporters:
+    logging: {}
+    file/trace:
+      path: /tmp/trace.json
+  service:
+    pipelines:
+      traces:
+        exporters:
+          - file/trace
+
+image:
+  repository: otel/opentelemetry-collector-contrib-dev
+  tag: "latest"
+
+command:
+  name: otelcontribcol
+
+extraVolumes:
+- name: filevolume
+  emptyDir: {}
+extraVolumeMounts: 
+- mountPath: /tmp
+  name: filevolume
+
+extraContainers: 
+- name: filecp
+  image: busybox
+  command: ["sh", "-c", "sleep 36000"]   
+  volumeMounts:
+  - name: filevolume
+    mountPath: /tmp

.github/workflows/e2e/k8s/telemetrygen-job.yml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-telemetrygen
+  namespace: default
+spec:
+  template:
+    metadata:
+      labels:
+        component: test-telemetrygen
+    spec:
+      containers:
+      - command:
+        - /telemetrygen
+        - traces
+        - --otlp-insecure
+        - --otlp-endpoint=test-opentelemetry-collector:4317
+        - --duration=5s
+        image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest
+        imagePullPolicy: IfNotPresent
+        name: telemetrygen
+        resources: {}
+        securityContext: {}
+      dnsPolicy: ClusterFirst
+      restartPolicy: Never
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30

.github/workflows/k8s-related-components-e2e.yml

@@ -0,0 +1,76 @@
+name: Test Kubernetes Related Components
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  kubernetes-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19   
+      - name: Cache Go
+        id: go-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} 
+      - name: Install dependencies
+        if: steps.go-cache.outputs.cache-hit != 'true'
+        run: make -j2 gomoddownload
+      - name: Build Collector
+        run: make otelcontribcol
+      - name: Build Docker Image
+        run: |
+            make docker-otelcontribcol
+            docker tag otelcontribcol:latest otel/opentelemetry-collector-contrib-dev:$GITHUB_SHA
+      - name: Set up Helm
+        uses: azure/[email protected]
+        with:
+          version: v3.9.0
+      - name: Set up chart-testing
+        uses: helm/[email protected]
+      - name: Create kind cluster
+        uses: helm/[email protected]
+      - name: Check kind
+        run: |
+          kubectl cluster-info --context kind-chart-testing
+          kubectl get node
+          docker ps -a --filter label=io.x-k8s.kind.cluster=chart-testing
+      - name: Kind load image
+        run: |
+          kind load docker-image otel/opentelemetry-collector-contrib-dev:$GITHUB_SHA --name chart-testing
+      - name: Add Dependencies
+        shell: bash
+        run: |
+            helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+      - uses: actions/checkout@v3
+        with:
+          repository: 'open-telemetry/opentelemetry-helm-charts'
+          path: opentelemetry-helm-charts
+      - name: Helm install collector
+        run: helm install test --set image.tag=$GITHUB_SHA -f .github/workflows/e2e/k8s/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
+      - name: check collector status
+        run: |
+          kubectl wait --for=condition=Ready --timeout=60s pod/test-opentelemetry-collector-0
+      - name: start telemetrygen job
+        run: |
+          kubectl -n default create -f .github/workflows/e2e/k8s/telemetrygen-job.yml
+      - name: check telemetrygen job status
+        run: |
+          kubectl wait --for=condition=Complete --timeout=60s job/test-telemetrygen
+      - name: copy trace output
+        run: |
+          kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json processor/k8sattributesprocessor/testdata/trace.json
+      - name: run e2e test to check output
+        run: |    
+          cd processor/k8sattributesprocessor
+          go test -v --tags=e2e

processor/k8sattributesprocessor/e2e_test.go

@@ -0,0 +1,71 @@
+// Copyright 2020 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build e2e
+// +build e2e
+
+package k8sattributesprocessor
+
+import (
+	"bufio"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+)
+
+func TestJobE2E(t *testing.T) {
+	f, err := os.Open("./testdata/trace.json")
+	assert.NoError(t, err)
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+
+	const maxCapacity int = 8388608
+	buf := make([]byte, maxCapacity)
+	scanner.Buffer(buf, maxCapacity)
+
+	unmarshaler := ptrace.JSONUnmarshaler{}
+	jobKeyFound := false
+	nsKeyFound := false
+	for scanner.Scan() {
+		traces, err := unmarshaler.UnmarshalTraces(scanner.Bytes())
+		assert.NoError(t, err)
+
+		len := traces.ResourceSpans().Len()
+		for i := 0; i < len; i++ {
+			resourceSpans := traces.ResourceSpans().At(i)
+
+			resourceSpans.Resource().Attributes().Range(
+				func(k string, v pcommon.Value) bool {
+					if k == "k8s.job.name" && v.Str() == "test-telemetrygen" {
+						jobKeyFound = true
+					}
+					if k == "k8s.namespace.name" && v.Str() == "default" {
+						nsKeyFound = true
+					}
+					return true
+				},
+			)
+		}
+	}
+
+	assert.True(t, jobKeyFound && nsKeyFound, "job and namespace attributes are not found")
+
+	if err := scanner.Err(); err != nil {
+		assert.NoError(t, err)
+	}
+}

processor/k8sattributesprocessor/go.mod

@@ -50,6 +50,7 @@ require (
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opentelemetry.io/collector/featuregate v0.70.1-0.20230130215412-26bd7b2bf678 // indirect
 	go.opentelemetry.io/otel v1.12.0 // indirect
@@ -82,3 +83,9 @@ require (
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => ./../../internal/k8sconfig
 
 retract v0.65.0
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => ../../internal/coreinternal