[Elastic Agent]-PR- Introduce log message for not supported annotatio…

…ns for Hints based autodiscover (#4360) * first commit with checks for hints vocabulary * updating to elastic-agent-autodiscovery v.0.6.13 * adding test TestGenerateHints * adding test TestGenerateHints * adding function to override sonarcube --------- Co-authored-by: Tetiana Kravchenko <[email protected]> Co-authored-by: Denis <[email protected]>
elastic · May 9, 2024 · 62958d0 · 62958d0
1 parent f25124a
commit 62958d0
Show file tree

Hide file tree

Showing 8 changed files with 235 additions and 9 deletions.
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -722,11 +722,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.6.8
+Version: v0.6.13
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].13/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/changelog/fragments/1709656001-logshints.yaml b/changelog/fragments/1709656001-logshints.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Introduce log message for not supported annotations for Hints based autodiscover
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4360
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/3064
diff --git a/go.mod b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/docker/go-units v0.5.0
 	github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
 	github.com/elastic/e2e-testing v1.2.1
-	github.com/elastic/elastic-agent-autodiscover v0.6.8
+	github.com/elastic/elastic-agent-autodiscover v0.6.13
 	github.com/elastic/elastic-agent-client/v7 v7.8.1
 	github.com/elastic/elastic-agent-libs v0.9.7
 	github.com/elastic/elastic-agent-system-metrics v0.9.2

diff --git a/go.sum b/go.sum
@@ -791,8 +791,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/e2e-testing v1.2.1 h1:jIuikohPtTxtO+bfoVEyKAWmcsAl21lxiiTK8Fj+G8U=
 github.com/elastic/e2e-testing v1.2.1/go.mod h1:8q2d8dmwavJXISowwaoreHFBnbR/uK4qanfRGhC/W9A=
-github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8=
-github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
+github.com/elastic/elastic-agent-autodiscover v0.6.13 h1:zBeTxV+o2efEKntY+o6iMMNJ1AVjDXUqY3o6uzIkKaw=
+github.com/elastic/elastic-agent-autodiscover v0.6.13/go.mod h1:7P6YVKxuBT0qE/VxuA87obwZUAEU0O44mCN3r4/6x8w=
 github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM=
 github.com/elastic/elastic-agent-client/v7 v7.8.1/go.mod h1:axl1nkdqc84YRFkeJGD9jExKNPUrOrzf3DFo2m653nY=
 github.com/elastic/elastic-agent-libs v0.9.7 h1:LZdfxbq724Y1zAdE3COp+OIPwU8SquOCLIXpI/twcdQ=

diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go
@@ -30,6 +30,8 @@ const (
 	processors  = "processors"
 )
 
+var allSupportedHints = []string{"enabled", integration, datastreams, host, period, timeout, metricspath, username, password, stream, processors}
+
 type hintsBuilder struct {
 	Key string
 
@@ -288,7 +290,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre
 		}
 	}
 
-	hintsExtracted := utils.GenerateHints(annotations, cName, prefix)
+	hintsExtracted, _ := utils.GenerateHints(annotations, cName, prefix, false, allSupportedHints)
 	if len(hintsExtracted) == 0 {
 		return hintData
 	}

diff --git a/internal/pkg/composable/providers/kubernetes/hints_test.go b/internal/pkg/composable/providers/kubernetes/hints_test.go
@@ -367,6 +367,137 @@ func TestGenerateHintsMappingWithLogStream(t *testing.T) {
 	assert.Equal(t, expected, hintsMapping)
 }
 
+func TestGenerateHintsMappingWithDefaultsandTypo(t *testing.T) {
+	logger := getLogger()
+	pod := &kubernetes.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "testpod",
+			UID:       types.UID(uid),
+			Namespace: "testns",
+			Labels: map[string]string{
+				"foo": "bar",
+			},
+			Annotations: map[string]string{
+				"app": "production",
+			},
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		Spec: kubernetes.PodSpec{
+			NodeName: "testnode",
+		},
+		Status: kubernetes.PodStatus{PodIP: "127.0.0.5"},
+	}
+
+	mapping := map[string]interface{}{
+		"namespace": pod.GetNamespace(),
+		"pod": mapstr.M{
+			"uid":  string(pod.GetUID()),
+			"name": pod.GetName(),
+			"ip":   pod.Status.PodIP,
+		},
+		"namespace_annotations": mapstr.M{
+			"nsa": "nsb",
+		},
+		"labels": mapstr.M{
+			"foo": "bar",
+		},
+		"annotations": mapstr.M{
+			"app": "production",
+		},
+	}
+	hints := mapstr.M{
+		"hints": mapstr.M{
+			"host":             "${kubernetes.pod.ip}:6379",
+			"package":          "redis",
+			"notsupportedhint": "/metrics", // on purpose we have introduced a typo
+			"timeout":          "42s",
+			"period":           "42s",
+		},
+	}
+
+	expected := mapstr.M{
+		"redis": mapstr.M{
+			"enabled": true,
+			"host":    "127.0.0.5:6379",
+			"timeout": "42s",
+			"period":  "42s",
+		},
+	}
+
+	hintsMapping := GenerateHintsMapping(hints, mapping, logger, "")
+
+	assert.Equal(t, expected, hintsMapping)
+}
+
+func TestGenerateHintsMappingWithInfoandTypo(t *testing.T) {
+	logger := getLogger()
+	pod := &kubernetes.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "testpod",
+			UID:       types.UID(uid),
+			Namespace: "testns",
+			Labels: map[string]string{
+				"foo": "bar",
+			},
+			Annotations: map[string]string{
+				"app": "production",
+			},
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		Spec: kubernetes.PodSpec{
+			NodeName: "testnode",
+		},
+		Status: kubernetes.PodStatus{PodIP: "127.0.0.5"},
+	}
+
+	mapping := map[string]interface{}{
+		"namespace": pod.GetNamespace(),
+		"pod": mapstr.M{
+			"uid":  string(pod.GetUID()),
+			"name": pod.GetName(),
+			"ip":   pod.Status.PodIP,
+		},
+		"namespace_annotations": mapstr.M{
+			"nsa": "nsb",
+		},
+		"labels": mapstr.M{
+			"foo": "bar",
+		},
+		"annotations": mapstr.M{
+			"app": "production",
+		},
+	}
+	hints := mapstr.M{
+		"hints": mapstr.M{
+			"package":      "redis",
+			"data_streams": "info",
+			"info": mapstr.M{
+				"period":           "5m",
+				"notsupportedhint": "stdout", // On purpose we have added this typo
+			},
+		},
+	}
+
+	expected := mapstr.M{
+		"redis": mapstr.M{
+			"info": mapstr.M{
+				"enabled": true,
+				"period":  "5m",
+			},
+		},
+	}
+
+	hintsMapping := GenerateHintsMapping(hints, mapping, logger, "")
+
+	assert.Equal(t, expected, hintsMapping)
+}
+
 func TestGenerateHintsMappingWithProcessors(t *testing.T) {
 	logger := getLogger()
 	pod := &kubernetes.Pod{

diff --git a/internal/pkg/composable/providers/kubernetes/pod.go b/internal/pkg/composable/providers/kubernetes/pod.go
@@ -9,10 +9,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/elastic/elastic-agent-autodiscover/utils"
-
 	"github.com/elastic/elastic-agent-autodiscover/kubernetes"
 	"github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata"
+	"github.com/elastic/elastic-agent-autodiscover/utils"
 	c "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
@@ -217,7 +216,7 @@ func (p *pod) emitRunning(pod *kubernetes.Pod) {
 		if !p.managed {
 			if ann, ok := data.mapping["annotations"]; ok {
 				annotations, _ := ann.(mapstr.M)
-				hints := utils.GenerateHints(annotations, "", p.config.Prefix)
+				hints, _ := hintsCheck(annotations, "", p.config.Prefix, true, allSupportedHints, p.logger, pod)
 				if len(hints) > 0 {
 					p.logger.Debugf("Extracted hints are :%v", hints)
 					hintsMapping := GenerateHintsMapping(hints, data.mapping, p.logger, "")
@@ -525,3 +524,12 @@ func updateProcessors(newprocessors []mapstr.M, processors []map[string]interfac
 
 	return processors
 }
+
+// HintsCheck geenrates hints from provided annotations of the pod and logs any possible incorrect annotations that have been provided in the pod
+func hintsCheck(annotations mapstr.M, container string, prefix string, validate bool, allSupportedHints []string, logger *logp.Logger, pod *kubernetes.Pod) (mapstr.M, []string) {
+	hints, incorrecthints := utils.GenerateHints(annotations, container, prefix, validate, allSupportedHints)
+	for _, value := range incorrecthints { //We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+		logger.Warnf("provided hint: %s/%s is not recognised as supported annotation for pod %s in namespace %s", prefix, value, pod.Name, pod.ObjectMeta.Namespace)
+	}
+	return hints, incorrecthints
+}
diff --git a/internal/pkg/composable/providers/kubernetes/pod_test.go b/internal/pkg/composable/providers/kubernetes/pod_test.go
@@ -375,6 +375,59 @@ func TestEphemeralContainers(t *testing.T) {
 
 }
 
+func TestGenerateHints(t *testing.T) {
+	pod := &kubernetes.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "testpod",
+			UID:       types.UID(uid),
+			Namespace: "testns",
+			Labels: map[string]string{
+				"foo": "bar",
+			},
+			Annotations: map[string]string{
+				"app":                                "production",
+				"co.elastic.hints/host":              "${kubernetes.pod.ip}:6379",
+				"co.elastic.hints/package":           "redis",
+				"co.elastic.hints/metricssssssspath": "/metrics",
+				"co.elastic.hints/period":            "42s",
+			},
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		Spec: kubernetes.PodSpec{
+			NodeName: "testnode",
+		},
+		Status: kubernetes.PodStatus{PodIP: "127.0.0.5"},
+	}
+
+	data := generatePodData(pod, &podMeta{}, mapstr.M{})
+
+	hints_result := mapstr.M{
+		"hints": mapstr.M{
+			"host":              "${kubernetes.pod.ip}:6379",
+			"package":           "redis",
+			"metricssssssspath": "/metrics", // on purpose we have introduced a typo
+			"period":            "42s",
+		},
+	}
+	incorrecthints_results := []string{"hints/metricssssssspath"}
+
+	ann := data.mapping["annotations"]
+	annotations, _ := ann.(mapstr.M)
+	prefix := "co.elastic"
+
+	log, err := logger.New("hint-test", true)
+	assert.NoError(t, err)
+
+	hints, incorrecthints := hintsCheck(annotations, "", prefix, true, allSupportedHints, log, pod)
+
+	assert.Equal(t, string(pod.GetUID()), data.uid)
+	assert.Equal(t, hints, hints_result)
+	assert.Equal(t, incorrecthints, incorrecthints_results)
+}
+
 func TestPodEventer_Namespace_Node_Watcher(t *testing.T) {
 	client := k8sfake.NewSimpleClientset()