[Beats] Adding support for elastic-agent-autodiscovery v0.6.13 Introduce log message for not supported annotations

CHANGELOG.next.asciidoc

@@ -19,6 +19,7 @@ you can achieve this by overwriting the value using an `add_fields` processor. {
 version of Elastic Agent as their own version. This includes all additional
 fields added to events containing the Beats version. {pull}37553[37553]
 - The behavior of `queue.mem.flush.min_events` has been simplified. It now serves as a simple maximum on the size of all event batches. There are no longer performance implications in its relationship to `bulk_max_size`. {pull}37795[37795]
+-  Introduce log message for not supported annotations for Hints based autodiscover {pull}38213[38213]
 
 *Auditbeat*
 

NOTICE.txt

@@ -12313,11 +12313,11 @@ various licenses:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.6.7
+Version: v0.6.9
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].7/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].9/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -202,7 +202,7 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.4.0
-	github.com/elastic/elastic-agent-autodiscover v0.6.7
+	github.com/elastic/elastic-agent-autodiscover v0.6.9
 	github.com/elastic/elastic-agent-libs v0.7.5
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.9.2

go.sum

@@ -673,8 +673,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI=
 github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ=
-github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g=
-github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
+github.com/elastic/elastic-agent-autodiscover v0.6.9 h1:F2BeSia5cM0q2NNVXTA3OSrVK0ayPohDQcOhZo514Hc=
+github.com/elastic/elastic-agent-autodiscover v0.6.9/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
 github.com/elastic/elastic-agent-client/v7 v7.8.0 h1:GHFzDJIWpdgI0qDk5EcqbQJGvwTsl2E2vQK3/xe+MYQ=
 github.com/elastic/elastic-agent-client/v7 v7.8.0/go.mod h1:ihtjqJzYiIltlRhNruaSSc0ogxIhqPD5hOMKq16cI1s=
 github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg=

libbeat/autodiscover/providers/docker/docker.go

@@ -383,7 +383,7 @@ func (d *Provider) generateHints(event bus.Event) bus.Event {
 		e["ports"] = ports
 	}
 	if labels, err := dockerMeta.GetValue("labels"); err == nil {
-		hints := utils.GenerateHints(labels.(mapstr.M), "", d.config.Prefix)
+		hints, _ := utils.GenerateHints(labels.(mapstr.M), "", d.config.Prefix, []string{})
 		e["hints"] = hints
 	}
 	return e

libbeat/autodiscover/providers/kubernetes/config.go

@@ -30,6 +30,8 @@
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
+var AllSupportedHints = []string{"enabled", "module", "metricsets", "host", "period", "timeout", "metricspath", "username", "password", "stream", "processors", "multiline", "json", "disable", "ssl", "metrics_filters", "raw", "include_lines", "exclude_lines", "fileset", "pipeline"}
+
 // Config for kubernetes autodiscover provider
 type Config struct {
 	KubeConfig        string                       `config:"kube_config"`
@@ -57,7 +59,7 @@
 	AddResourceMetadata *metadata.AddResourceMetadataConfig `config:"add_resource_metadata"`
 }
 
 // Public variable, so specific beats (as Filebeat) can set a different cleanup timeout if they need it.
 var DefaultCleanupTimeout time.Duration = 0
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/kubernetes/node.go

@@ -153,7 +153,13 @@ func (n *node) GenerateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 
-	hints := utils.GenerateHints(annotations, "", n.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, "", n.config.Prefix, AllSupportedHints)
+	//We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	if len(incorrecthints) > 0 {
+		for _, value := range incorrecthints {
+			n.logger.Warnf("provided hint: %s/%s is not in the supported list", n.config.Prefix, value)
+		}
+	}
 	n.logger.Debugf("Generated hints %+v", hints)
 	if len(hints) != 0 {
 		e["hints"] = hints

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -246,7 +246,13 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event {
 	cname := utils.GetContainerName(container)
 
 	// Generate hints based on the cumulative of both namespace and pod annotations.
-	hints := utils.GenerateHints(annotations, cname, p.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, cname, p.config.Prefix, AllSupportedHints)
+	//We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	if len(incorrecthints) > 0 {
+		for _, value := range incorrecthints {
+			p.logger.Warnf("provided hint: %s/%s is not in the supported list", p.config.Prefix, value)
+		}
+	}
 	p.logger.Debugf("Generated hints %+v", hints)
 
 	if len(hints) != 0 {

libbeat/autodiscover/providers/kubernetes/pod_test.go

@@ -108,6 +108,7 @@ func TestGenerateHints(t *testing.T) {
 						"co.elastic.logs/multiline.pattern":    "^test",
 						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/timeouts":          "5s", //On purpose we added this annotation with typo
 						"co.elastic.metrics/period":            "10s",
 						"co.elastic.metrics.foobar/period":     "15s",
 						"not.to.include":                       "true",
@@ -125,6 +126,7 @@ func TestGenerateHints(t *testing.T) {
 						"co.elastic.logs/multiline.pattern":    "^test",
 						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/timeouts":          "5s",
 						"not.to.include":                       "true",
 						"co.elastic.metrics/period":            "10s",
 						"co.elastic.metrics.foobar/period":     "15s",
@@ -145,8 +147,9 @@ func TestGenerateHints(t *testing.T) {
 						},
 					},
 					"metrics": mapstr.M{
-						"module": "prometheus",
-						"period": "15s",
+						"module":   "prometheus",
+						"period":   "15s",
+						"timeouts": "5s",
 					},
 				},
 				"container": mapstr.M{
@@ -2140,6 +2143,10 @@ func (s *mockUpdaterWatcher) Start() error {
 	return err
 }
 
+func (s *mockUpdaterWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
+	return nil
+}
+
 func (s *mockUpdaterWatcher) Stop() {
 }
 

libbeat/autodiscover/providers/kubernetes/service.go

@@ -155,7 +155,13 @@
 		e["port"] = port
 	}
 
-	hints := utils.GenerateHints(annotations, "", s.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, "", s.config.Prefix, AllSupportedHints)
+	//We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	if len(incorrecthints) > 0 {
+		for _, value := range incorrecthints {
+			s.logger.Warnf("provided hint: %s/%s is not in the supported list", s.config.Prefix, value)
+		}
+	}
 	s.logger.Debugf("Generated hints %+v", hints)
 
 	if len(hints) != 0 {
@@ -220,7 +226,7 @@
 		}
 	}
 
 	var events []bus.Event
 	for _, port := range svc.Spec.Ports {
 		event := bus.Event{
 			"provider":   s.uuid,

metricbeat/module/kubernetes/util/kubernetes_test.go

@@ -161,6 +161,10 @@ type mockWatcher struct {
 	started bool
 }
 
+func (m *mockWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
+	return m.handler
+}
+
 func (m *mockWatcher) Start() error {
 	m.started = true
 	return nil

x-pack/libbeat/autodiscover/providers/nomad/nomad.go

@@ -277,7 +277,7 @@ func (p *Provider) generateHints(event bus.Event) bus.Event {
 	}
 
 	cname := utils.GetContainerName(container)
-	hints := utils.GenerateHints(tasks, cname, p.config.Prefix)
+	hints, _ := utils.GenerateHints(tasks, cname, p.config.Prefix, []string{})
 	if len(hints) > 0 {
 		e["hints"] = hints
 	}