From 442ef3ee00aaece351804e54fc098d966874c2b2 Mon Sep 17 00:00:00 2001 From: constanca Date: Tue, 18 Jun 2024 15:18:08 +0200 Subject: [PATCH 1/9] Fix namespace filter. Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 1 + .../add_kubernetes_metadata/kubernetes.go | 33 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 372b8e35244..45b9a7f0c1f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -101,6 +101,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Filebeat* +- Fix `namespace` filter option on `add_kubernetes_metadata` processor. {pull}[] - [Gcs Input] - Added missing locks for safe concurrency {pull}34914[34914] - Fix the ignore_inactive option being ignored in Filebeat's filestream input {pull}34770[34770] - Fix TestMultiEventForEOFRetryHandlerInput unit test of CometD input {pull}34903[34903] diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index f9143cdf289..e8600b6d85c 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -187,9 +187,10 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) { } watcher, err := kubernetes.NewNamedWatcher("add_kubernetes_metadata_pod", client, &kubernetes.Pod{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Node: config.Node, - Namespace: config.Namespace, + SyncTimeout: config.SyncPeriod, + Node: config.Node, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { k.log.Errorf("Couldn't create kubernetes watcher for %T", &kubernetes.Pod{}) @@ -198,14 +199,12 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) { metaConf := config.AddResourceMetadata - options := kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Node: config.Node, - Namespace: config.Namespace, - } - if metaConf.Node.Enabled() { - nodeWatcher, err = kubernetes.NewNamedWatcher("add_kubernetes_metadata_node", client, &kubernetes.Node{}, options, nil) + nodeWatcher, err = kubernetes.NewNamedWatcher("add_kubernetes_metadata_node", client, &kubernetes.Node{}, kubernetes.WatchOptions{ + SyncTimeout: config.SyncPeriod, + Node: config.Node, + HonorReSyncs: true, + }, nil) if err != nil { k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err) } @@ -213,20 +212,24 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) { if metaConf.Namespace.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("add_kubernetes_metadata_namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err) } } - // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to + // Resource is Pod, so we need to create watchers for Replicasets and Jobs that it might belong to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod // CronJob -> job -> Pod if metaConf.Deployment { replicaSetWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_rs", client, &kubernetes.ReplicaSet{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { k.log.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.ReplicaSet{}, err) @@ -235,7 +238,9 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) { } if metaConf.CronJob { jobWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_job", client, &kubernetes.Job{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { k.log.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.Job{}, err) From 4a281d70373db4a9d788798a37e2dcf4da9f4c1e Mon Sep 17 00:00:00 2001 From: constanca Date: Tue, 18 Jun 2024 15:50:11 +0200 Subject: [PATCH 2/9] Fix namespace filter. Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 45b9a7f0c1f..5e5b8c66e44 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -101,7 +101,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Filebeat* -- Fix `namespace` filter option on `add_kubernetes_metadata` processor. {pull}[] +- Fix `namespace` filter option on `add_kubernetes_metadata` processor. {pull}39934[39934] - [Gcs Input] - Added missing locks for safe concurrency {pull}34914[34914] - Fix the ignore_inactive option being ignored in Filebeat's filestream input {pull}34770[34770] - Fix TestMultiEventForEOFRetryHandlerInput unit test of CometD input {pull}34903[34903] From 1df9175725941b2395235d57cfe83792209f6ab8 Mon Sep 17 00:00:00 2001 From: constanca Date: Wed, 19 Jun 2024 09:04:39 +0200 Subject: [PATCH 3/9] Fix namespace filter. Signed-off-by: constanca --- metricbeat/docs/modules/kubernetes.asciidoc | 6 +++--- metricbeat/module/kubernetes/util/kubernetes.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index ceae7f80692..db08962fc77 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -295,11 +295,11 @@ metricbeat.modules: # When used outside the cluster: #node: node_name - # Set the namespace to watch for resources + # Set the namespace to add metadata #namespace: staging # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. - # By default all labels will be included while annotations are not added by default. + # By default, all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: # include_labels: ["namespacelabel1"] @@ -326,7 +326,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # Set the namespace to watch for events + # Set the namespace to add metadata #namespace: staging # Set the sync period of the watchers #sync_period: 10m diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index a0c409ca14e..f44eb531835 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -29,13 +29,14 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" - kubernetes2 "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-autodiscover/kubernetes" "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + + kubernetes2 "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" + "github.com/elastic/beats/v7/metricbeat/mb" ) type kubernetesConfig struct { @@ -281,8 +282,7 @@ func getWatchOptions(config *kubernetesConfig, nodeScope bool, client k8sclient. } func isNamespaced(resourceName string) bool { - if resourceName == NodeResource || resourceName == PersistentVolumeResource || resourceName == StorageClassResource || - resourceName == NamespaceResource { + if resourceName == NodeResource || resourceName == PersistentVolumeResource || resourceName == StorageClassResource { return false } return true From c760ef6443bf1217cadee93de59ceca6128067ac Mon Sep 17 00:00:00 2001 From: constanca Date: Wed, 19 Jun 2024 09:07:06 +0200 Subject: [PATCH 4/9] Fix namespace filter. Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5e5b8c66e44..e78401f7118 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -152,6 +152,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* +- Fix `namespace` filter option on metricset `state_namespace` enricher. {pull}39934[39934] - Fix `namespace` filter option at Kubernetes provider level. {pull}39881[39881] - Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294] - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] From 3c45d0d97242189449653c172fd9f1e88b31b80d Mon Sep 17 00:00:00 2001 From: constanca Date: Wed, 19 Jun 2024 09:11:02 +0200 Subject: [PATCH 5/9] Ignore lint error on metagen variables Signed-off-by: constanca --- metricbeat/module/kubernetes/util/kubernetes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index f44eb531835..1731afe76b4 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -590,8 +590,8 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - var specificMetaGen metadata.MetaGen - var generalMetaGen *metadata.Resource + var specificMetaGen metadata.MetaGen //nolint:all + var generalMetaGen *metadata.Resource //nolint:all // Create the metadata generator to be used in the watcher's event handler. // Both specificMetaGen and generalMetaGen implement Generate method for metadata collection. if resourceName == ServiceResource || resourceName == PodResource { From aa1d4e52470b7837f792514bffcad76d69a92c17 Mon Sep 17 00:00:00 2001 From: constanca Date: Wed, 19 Jun 2024 09:18:48 +0200 Subject: [PATCH 6/9] make update Signed-off-by: constanca --- metricbeat/docs/modules/kubernetes.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index db08962fc77..ceae7f80692 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -295,11 +295,11 @@ metricbeat.modules: # When used outside the cluster: #node: node_name - # Set the namespace to add metadata + # Set the namespace to watch for resources #namespace: staging # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. - # By default, all labels will be included while annotations are not added by default. + # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: # include_labels: ["namespacelabel1"] @@ -326,7 +326,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # Set the namespace to add metadata + # Set the namespace to watch for events #namespace: staging # Set the sync period of the watchers #sync_period: 10m From 420ba93b1ab709749fcfc42c20b76ebd82178255 Mon Sep 17 00:00:00 2001 From: constanca Date: Wed, 19 Jun 2024 09:55:48 +0200 Subject: [PATCH 7/9] lint errors Signed-off-by: constanca --- metricbeat/module/kubernetes/util/kubernetes.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 1731afe76b4..c7f97971373 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -590,8 +590,8 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - var specificMetaGen metadata.MetaGen //nolint:all - var generalMetaGen *metadata.Resource //nolint:all + var specificMetaGen metadata.MetaGen //nolint:unused // False positive, ignore the lint error that it is not used + var generalMetaGen *metadata.Resource //nolint:unused // False positive, ignore the lint error that it is not used // Create the metadata generator to be used in the watcher's event handler. // Both specificMetaGen and generalMetaGen implement Generate method for metadata collection. if resourceName == ServiceResource || resourceName == PodResource { @@ -604,11 +604,11 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - // updateFunc to be used as the resource watcher's add and update handler. + // updateFunc to be used as the resource watchers add and update handler. // The handler function is executed when a watcher is triggered(i.e. new/updated resource). - // It is responsible for generating the metadata for a detected resource by executing the metadata generator's Generate method. + // It is responsible for generating the metadata for a detected resource by executing the metadata generators Generate method. // It is a common handler for all resource watchers. The kind of resource(e.g. pod or deployment) is checked inside the function. - // It returns a map of a resourse identifier(i.e namespace-resource_name) as key and the metadata as value. + // It returns a map of a resource identifier(i.e. namespace-resource_name) as key and the metadata as value. updateFunc := func(r kubernetes.Resource) map[string]mapstr.M { accessor, _ := meta.Accessor(r) id := accessor.GetName() @@ -691,7 +691,7 @@ func NewResourceMetadataEnricher( indexFunc := func(e mapstr.M) string { name := getString(e, "name") namespace := getString(e, mb.ModuleDataKey+".namespace") - id := "" + var id string if name != "" && namespace != "" { id = join(namespace, name) } else if namespace != "" { From 9b84de88ad8179686f265b4500e3534e12b7528d Mon Sep 17 00:00:00 2001 From: constanca Date: Wed, 19 Jun 2024 10:10:50 +0200 Subject: [PATCH 8/9] lint errors Signed-off-by: constanca --- metricbeat/module/kubernetes/util/kubernetes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index c7f97971373..f94d424ec0e 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -590,8 +590,8 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - var specificMetaGen metadata.MetaGen //nolint:unused // False positive, ignore the lint error that it is not used - var generalMetaGen *metadata.Resource //nolint:unused // False positive, ignore the lint error that it is not used + var specificMetaGen metadata.MetaGen + var generalMetaGen *metadata.Resource // Create the metadata generator to be used in the watcher's event handler. // Both specificMetaGen and generalMetaGen implement Generate method for metadata collection. if resourceName == ServiceResource || resourceName == PodResource { From e555e290e26d5cc33b72ed1150ffdfb21958ed51 Mon Sep 17 00:00:00 2001 From: constanca Date: Thu, 20 Jun 2024 10:31:12 +0200 Subject: [PATCH 9/9] Update changelog Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e78401f7118..cfff93f7589 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Affecting all Beats* +- Fix `namespace` filter option on `add_kubernetes_metadata` processor. {pull}39934[39934] - Support for multiline zookeeper logs {issue}2496[2496] - Add checks to ensure reloading of units if the configuration actually changed. {pull}34346[34346] - Fix namespacing on self-monitoring {pull}32336[32336] @@ -101,7 +102,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Filebeat* -- Fix `namespace` filter option on `add_kubernetes_metadata` processor. {pull}39934[39934] - [Gcs Input] - Added missing locks for safe concurrency {pull}34914[34914] - Fix the ignore_inactive option being ignored in Filebeat's filestream input {pull}34770[34770] - Fix TestMultiEventForEOFRetryHandlerInput unit test of CometD input {pull}34903[34903]