[Beats] Adding support for elastic-agent-autodiscovery v0.6.13 Introduce log message for not supported annotations

CHANGELOG.next.asciidoc

@@ -9,6 +9,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 ==== Breaking changes
 
 *Affecting all Beats*
+-  Introduce log message for not supported annotations for Hints based autodiscover {pull}38213[38213]
 
 
 *Auditbeat*
@@ -368,6 +369,5 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
-
 
 

NOTICE.txt

@@ -11088,11 +11088,11 @@ third-party archives.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/docker/docker
-Version: v24.0.7+incompatible
+Version: v24.0.9+incompatible
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/docker/[email protected].7+incompatible/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/docker/[email protected].9+incompatible/LICENSE:
 
 
                                  Apache License
@@ -12313,11 +12313,11 @@ various licenses:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.6.8
+Version: v0.6.12
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].12/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -59,7 +59,7 @@ require (
 	github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 // indirect
 	github.com/dgraph-io/badger/v3 v3.2103.1
 	github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8
 	github.com/docker/go-units v0.5.0
@@ -202,7 +202,7 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
-	github.com/elastic/elastic-agent-autodiscover v0.6.8
+	github.com/elastic/elastic-agent-autodiscover v0.6.12
 	github.com/elastic/elastic-agent-libs v0.7.5
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.9.2

go.sum

@@ -629,8 +629,8 @@ github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
 github.com/docker/docker v1.4.2-0.20170802015333-8af4db6f002a/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
+github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
@@ -674,8 +674,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ebpfevents v0.6.0 h1:BrL3m7JFK7U6h2jkbk3xAWWs//IZnugCHEDds5u2v68=
 github.com/elastic/ebpfevents v0.6.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU=
-github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8=
-github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
+github.com/elastic/elastic-agent-autodiscover v0.6.12 h1:tAKM7ImtMA+mKFrjVfAi4M1pTBvb+8EZSTaOF0RxQaA=
+github.com/elastic/elastic-agent-autodiscover v0.6.12/go.mod h1:7P6YVKxuBT0qE/VxuA87obwZUAEU0O44mCN3r4/6x8w=
 github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM=
 github.com/elastic/elastic-agent-client/v7 v7.8.1/go.mod h1:axl1nkdqc84YRFkeJGD9jExKNPUrOrzf3DFo2m653nY=
 github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg=

libbeat/autodiscover/providers/docker/docker.go

@@ -383,7 +383,7 @@ func (d *Provider) generateHints(event bus.Event) bus.Event {
 		e["ports"] = ports
 	}
 	if labels, err := dockerMeta.GetValue("labels"); err == nil {
-		hints := utils.GenerateHints(labels.(mapstr.M), "", d.config.Prefix)
+		hints, _ := utils.GenerateHints(labels.(mapstr.M), "", d.config.Prefix, []string{})
 		e["hints"] = hints
 	}
 	return e

libbeat/autodiscover/providers/kubernetes/config.go

@@ -30,6 +30,9 @@ import (
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
+// AllSupportedHints includes the set of all supported hints for both logs and metrics autodiscovery
+var AllSupportedHints = []string{"enabled", "module", "metricsets", "host", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable", "ssl", "metrics_filters", "raw", "include_lines", "exclude_lines", "fileset", "pipeline", "raw"}
+
 // Config for kubernetes autodiscover provider
 type Config struct {
 	KubeConfig        string                       `config:"kube_config"`

libbeat/autodiscover/providers/kubernetes/node.go

@@ -153,7 +153,11 @@ func (n *node) GenerateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 
-	hints := utils.GenerateHints(annotations, "", n.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, "", n.config.Prefix, AllSupportedHints)
+	//We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	for _, value := range incorrecthints {
+		n.logger.Debugf("provided hint: %s/%s is not in the supported list", n.config.Prefix, value)
+	}
 	n.logger.Debugf("Generated hints %+v", hints)
 	if len(hints) != 0 {
 		e["hints"] = hints

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -246,7 +246,11 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event {
 	cname := utils.GetContainerName(container)
 
 	// Generate hints based on the cumulative of both namespace and pod annotations.
-	hints := utils.GenerateHints(annotations, cname, p.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, cname, p.config.Prefix, AllSupportedHints)
+	//We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	for _, value := range incorrecthints {
+		p.logger.Debugf("provided hint: %s/%s is not in the supported list", p.config.Prefix, value)
+	}
 	p.logger.Debugf("Generated hints %+v", hints)
 
 	if len(hints) != 0 {

libbeat/autodiscover/providers/kubernetes/pod_test.go

@@ -108,6 +108,7 @@ func TestGenerateHints(t *testing.T) {
 						"co.elastic.logs/multiline.pattern":    "^test",
 						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/timeoutssssssss":   "5s", //On purpose we added this annotation with typo
 						"co.elastic.metrics/period":            "10s",
 						"co.elastic.metrics.foobar/period":     "15s",
 						"not.to.include":                       "true",
@@ -125,6 +126,7 @@ func TestGenerateHints(t *testing.T) {
 						"co.elastic.logs/multiline.pattern":    "^test",
 						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/timeoutssssssss":   "5s",
 						"not.to.include":                       "true",
 						"co.elastic.metrics/period":            "10s",
 						"co.elastic.metrics.foobar/period":     "15s",
@@ -145,8 +147,9 @@ func TestGenerateHints(t *testing.T) {
 						},
 					},
 					"metrics": mapstr.M{
-						"module": "prometheus",
-						"period": "15s",
+						"module":   "prometheus",
+						"period":   "15s",
+						"timeouts": "5s",
 					},
 				},
 				"container": mapstr.M{
@@ -2140,6 +2143,10 @@ func (s *mockUpdaterWatcher) Start() error {
 	return err
 }
 
+func (s *mockUpdaterWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
+	return nil
+}
+
 func (s *mockUpdaterWatcher) Stop() {
 }
 
@@ -2150,10 +2157,6 @@ func (s *mockUpdaterWatcher) Store() caches.Store {
 func (s *mockUpdaterWatcher) AddEventHandler(kubernetes.ResourceEventHandler) {
 }
 
-func (s *mockUpdaterWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
-	return nil
-}
-
 func (s *mockUpdaterStore) List() []interface{} {
 	return s.objects
 }

libbeat/autodiscover/providers/kubernetes/service.go

@@ -155,7 +155,11 @@ func (s *service) GenerateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 
-	hints := utils.GenerateHints(annotations, "", s.config.Prefix)
+	hints, incorrecthints := utils.GenerateHints(annotations, "", s.config.Prefix, AllSupportedHints)
+	//We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic
+	for _, value := range incorrecthints {
+		s.logger.Debugf("provided hint: %s/%s is not in the supported list", s.config.Prefix, value)
+	}
 	s.logger.Debugf("Generated hints %+v", hints)
 
 	if len(hints) != 0 {
@@ -220,7 +224,7 @@ func (s *service) emit(svc *kubernetes.Service, flag string) {
 		}
 	}
 
-	var events []bus.Event
+	events := []bus.Event{}
 	for _, port := range svc.Spec.Ports {
 		event := bus.Event{
 			"provider":   s.uuid,

metricbeat/module/kubernetes/util/kubernetes_test.go

@@ -706,6 +706,10 @@ type mockWatcher struct {
 	handler kubernetes.ResourceEventHandler
 }
 
+func (m *mockWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
+	return m.handler
+}
+
 func (m *mockWatcher) Start() error {
 	return nil
 }
@@ -718,10 +722,6 @@ func (m *mockWatcher) AddEventHandler(r kubernetes.ResourceEventHandler) {
 	m.handler = r
 }
 
-func (m *mockWatcher) GetEventHandler() kubernetes.ResourceEventHandler {
-	return m.handler
-}
-
 func (m *mockWatcher) Store() cache.Store {
 	return nil
 }

x-pack/libbeat/autodiscover/providers/nomad/nomad.go

@@ -277,7 +277,7 @@ func (p *Provider) generateHints(event bus.Event) bus.Event {
 	}
 
 	cname := utils.GetContainerName(container)
-	hints := utils.GenerateHints(tasks, cname, p.config.Prefix)
+	hints, _ := utils.GenerateHints(tasks, cname, p.config.Prefix, []string{})
 	if len(hints) > 0 {
 		e["hints"] = hints
 	}