[Beats] Adding support for elastic-agent-autodiscovery v0.6.13 Introduce log message for not supported annotations

CHANGELOG.next.asciidoc

@@ -10,7 +10,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
-
 *Auditbeat*
 
 
@@ -182,6 +181,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Update to Go 1.21.9. {pulk}38727[38727]
 - Enable early event encoding in the Elasticsearch output, improving cpu and memory use {pull}38572[38572]
 - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669]
+- Introduce log message for not supported annotations for Hints based autodiscover {pull}38213[38213]
+
 
 *Auditbeat*
 
@@ -379,6 +380,5 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
-
 
 

NOTICE.txt

@@ -11300,11 +11300,11 @@ third-party archives.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/docker/docker
-Version: v24.0.7+incompatible
+Version: v24.0.9+incompatible
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/docker/[email protected].7+incompatible/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/docker/[email protected].9+incompatible/LICENSE:
 
 
                                  Apache License
@@ -12525,11 +12525,11 @@ various licenses:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.6.8
+Version: v0.6.13
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].13/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -59,7 +59,7 @@ require (
 	github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 // indirect
 	github.com/dgraph-io/badger/v3 v3.2103.1
 	github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8
 	github.com/docker/go-units v0.5.0
@@ -203,7 +203,7 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
-	github.com/elastic/elastic-agent-autodiscover v0.6.8
+	github.com/elastic/elastic-agent-autodiscover v0.6.13
 	github.com/elastic/elastic-agent-libs v0.7.5
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.9.2

go.sum

@@ -630,8 +630,8 @@ github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
 github.com/docker/docker v1.4.2-0.20170802015333-8af4db6f002a/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
+github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
@@ -675,8 +675,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ebpfevents v0.6.0 h1:BrL3m7JFK7U6h2jkbk3xAWWs//IZnugCHEDds5u2v68=
 github.com/elastic/ebpfevents v0.6.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU=
-github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8=
-github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
+github.com/elastic/elastic-agent-autodiscover v0.6.13 h1:zBeTxV+o2efEKntY+o6iMMNJ1AVjDXUqY3o6uzIkKaw=
+github.com/elastic/elastic-agent-autodiscover v0.6.13/go.mod h1:7P6YVKxuBT0qE/VxuA87obwZUAEU0O44mCN3r4/6x8w=
 github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM=
 github.com/elastic/elastic-agent-client/v7 v7.8.1/go.mod h1:axl1nkdqc84YRFkeJGD9jExKNPUrOrzf3DFo2m653nY=
 github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg=

libbeat/autodiscover/providers/docker/config.go

@@ -27,6 +27,9 @@ import (
 	"github.com/elastic/elastic-agent-libs/config"
 )
 
+// AllSupportedHints includes the set of all supported hints for both logs and metrics autodiscovery
+var AllSupportedHints = []string{"enabled", "module", "metricsets", "hosts", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable", "ssl", "metrics_filters", "raw", "include_lines", "exclude_lines", "fileset", "pipeline", "raw"}
+
 // Config for docker autodiscover provider
 type Config struct {
 	Host           string                  `config:"host"`
@@ -40,7 +43,7 @@ type Config struct {
 	CleanupTimeout time.Duration           `config:"cleanup_timeout" validate:"positive"`
 }
 
-// Public variable, so specific beats (as Filebeat) can set a different cleanup timeout if they need it.
+// DefaultCleanupTimeout Public variable, so specific beats (as Filebeat) can set a different cleanup timeout if they need it.
 var DefaultCleanupTimeout time.Duration = 0
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/docker/docker.go

@@ -383,7 +383,11 @@ func (d *Provider) generateHints(event bus.Event) bus.Event {
 		e["ports"] = ports
 	}
 	if labels, err := dockerMeta.GetValue("labels"); err == nil {
-		hints := utils.GenerateHints(labels.(mapstr.M), "", d.config.Prefix)
+		hints, incorrecthints := utils.GenerateHints(labels.(mapstr.M), "", d.config.Prefix, true, AllSupportedHints)
+		// We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+		for _, value := range incorrecthints {
+			d.logger.Debugf("provided hint: %s/%s is not in the supported list", d.config.Prefix, value)
+		}
 		e["hints"] = hints
 	}
 	return e

libbeat/autodiscover/providers/kubernetes/config.go

@@ -30,6 +30,9 @@ import (
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
+// AllSupportedHints includes the set of all supported hints for both logs and metrics autodiscovery
+var AllSupportedHints = []string{"enabled", "module", "metricsets", "hosts", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable", "ssl", "metrics_filters", "raw", "include_lines", "exclude_lines", "fileset", "pipeline", "raw"}
+
 // Config for kubernetes autodiscover provider
 type Config struct {
 	KubeConfig        string                       `config:"kube_config"`

libbeat/autodiscover/providers/kubernetes/node.go

@@ -153,7 +153,11 @@ func (n *node) GenerateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 
-	hints := utils.GenerateHints(annotations, "", n.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, "", n.config.Prefix, true, AllSupportedHints)
+	// We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	for _, value := range incorrecthints {
+		n.logger.Debugf("provided hint: %s/%s is not in the supported list", n.config.Prefix, value)
+	}
 	n.logger.Debugf("Generated hints %+v", hints)
 	if len(hints) != 0 {
 		e["hints"] = hints

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -251,7 +251,11 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event {
 	cname := utils.GetContainerName(container)
 
 	// Generate hints based on the cumulative of both namespace and pod annotations.
-	hints := utils.GenerateHints(annotations, cname, p.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, cname, p.config.Prefix, true, AllSupportedHints)
+	// We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	for _, value := range incorrecthints {
+		p.logger.Debugf("provided hint: %s/%s is not in the supported list", p.config.Prefix, value)
+	}
 	p.logger.Debugf("Generated hints %+v", hints)
 
 	if len(hints) != 0 {

libbeat/autodiscover/providers/kubernetes/pod_test.go

@@ -108,6 +108,7 @@
 						"co.elastic.logs/multiline.pattern":    "^test",
 						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/timeoutssssssss":   "5s", //On purpose we added this annotation with typo
 						"co.elastic.metrics/period":            "10s",
 						"co.elastic.metrics.foobar/period":     "15s",
 						"not.to.include":                       "true",
@@ -125,6 +126,7 @@
 						"co.elastic.logs/multiline.pattern":    "^test",
 						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/timeoutssssssss":   "5s",
 						"not.to.include":                       "true",
 						"co.elastic.metrics/period":            "10s",
 						"co.elastic.metrics.foobar/period":     "15s",
@@ -145,8 +147,9 @@
 						},
 					},
 					"metrics": mapstr.M{
-						"module": "prometheus",
-						"period": "15s",
+						"module":          "prometheus",
+						"period":          "15s",
+						"timeoutssssssss": "5s",
 					},
 				},
 				"container": mapstr.M{
@@ -230,6 +233,7 @@
 						"co.elastic.metrics/module":        "prometheus",
 						"co.elastic.metrics/period":        "10s",
 						"co.elastic.metrics.foobar/period": "15s",
+						"co.elastic.metrics/hosts":         "127.0.0.1:9090",
 						"not.to.include":                   "true",
 					}),
 					"namespace_annotations": getNestedAnnotations(mapstr.M{
@@ -251,6 +255,7 @@
 						"co.elastic.metrics/module":        "prometheus",
 						"co.elastic.metrics/period":        "10s",
 						"co.elastic.metrics.foobar/period": "15s",
+						"co.elastic.metrics/hosts":         "127.0.0.1:9090",
 						"not.to.include":                   "true",
 					}),
 					"namespace_annotations": getNestedAnnotations(mapstr.M{
@@ -268,6 +273,7 @@
 				"hints": mapstr.M{
 					"metrics": mapstr.M{
 						"module": "prometheus",
+						"hosts":  "127.0.0.1:9090",
 						"period": "15s",
 					},
 				},
@@ -2190,8 +2196,10 @@
 	}
 
 	for _, test := range tests {
+
 		t.Run(test.name, func(t *testing.T) {
-			config := conf.MustNewConfigFrom(&test.cfg)
+			testcfg := &test.cfg
+			config := conf.MustNewConfigFrom(testcfg)
 			c := defaultConfig()
 			err = config.Unpack(&c)
 			assert.NoError(t, err)
@@ -2247,6 +2255,10 @@
 	return err
 }
 
+func (s *mockUpdaterWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
+	return nil
+}
+
 func (s *mockUpdaterWatcher) Stop() {
 }
 
@@ -2257,10 +2269,6 @@
 func (s *mockUpdaterWatcher) AddEventHandler(kubernetes.ResourceEventHandler) {
 }
 
-func (s *mockUpdaterWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
-	return nil
-}
-
 func (s *mockUpdaterStore) List() []interface{} {
 	return s.objects
 }

libbeat/autodiscover/providers/kubernetes/service.go

@@ -157,7 +157,11 @@ func (s *service) GenerateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 
-	hints := utils.GenerateHints(annotations, "", s.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, "", s.config.Prefix, true, AllSupportedHints)
+	// We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	for _, value := range incorrecthints {
+		s.logger.Debugf("provided hint: %s/%s is not in the supported list", s.config.Prefix, value)
+	}
 	s.logger.Debugf("Generated hints %+v", hints)
 
 	if len(hints) != 0 {
@@ -222,7 +226,7 @@ func (s *service) emit(svc *kubernetes.Service, flag string) {
 		}
 	}
 
-	var events []bus.Event
+	events := []bus.Event{}
 	for _, port := range svc.Spec.Ports {
 		event := bus.Event{
 			"provider":   s.uuid,

metricbeat/module/kubernetes/util/kubernetes_test.go

@@ -718,6 +718,10 @@ type mockWatcher struct {
 	handler kubernetes.ResourceEventHandler
 }
 
+func (m *mockWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
+	return m.handler
+}
+
 func (m *mockWatcher) Start() error {
 	return nil
 }
@@ -730,10 +734,6 @@ func (m *mockWatcher) AddEventHandler(r kubernetes.ResourceEventHandler) {
 	m.handler = r
 }
 
-func (m *mockWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
-	return m.handler
-}
-
 func (m *mockWatcher) Store() cache.Store {
 	return nil
 }

x-pack/libbeat/autodiscover/providers/nomad/nomad.go

@@ -277,7 +277,7 @@ func (p *Provider) generateHints(event bus.Event) bus.Event {
 	}
 
 	cname := utils.GetContainerName(container)
-	hints := utils.GenerateHints(tasks, cname, p.config.Prefix)
+	hints, _ := utils.GenerateHints(tasks, cname, p.config.Prefix, false, []string{}) // Parameter validate=false of utils.GenerateHints. This disables the validation of hints
 	if len(hints) > 0 {
 		e["hints"] = hints
 	}