Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[add_kubernetes_metadata][Metricset state_namespace] Fix namespace filter #39934

Merged
merged 10 commits into from
Jun 26, 2024
Merged

[add_kubernetes_metadata][Metricset state_namespace] Fix namespace filter #39934

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
442ef3e
Fix namespace filter.
constanca-m Jun 18, 2024
4a281d7
Fix namespace filter.
constanca-m Jun 18, 2024
1df9175
Fix namespace filter.
constanca-m Jun 19, 2024
c760ef6
Fix namespace filter.
constanca-m Jun 19, 2024
3c45d0d
Ignore lint error on metagen variables
constanca-m Jun 19, 2024
aa1d4e5
make update
constanca-m Jun 19, 2024
420ba93
lint errors
constanca-m Jun 19, 2024
9b84de8
lint errors
constanca-m Jun 19, 2024
e555e29
Update changelog
constanca-m Jun 20, 2024
a1784d6
Merge branch 'main' into k8s-metadata-processor
constanca-m Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]

*Filebeat*

- Fix `namespace` filter option on `add_kubernetes_metadata` processor. {pull}39934[39934]
constanca-m marked this conversation as resolved.
Show resolved Hide resolved
- [Gcs Input] - Added missing locks for safe concurrency {pull}34914[34914]
- Fix the ignore_inactive option being ignored in Filebeat's filestream input {pull}34770[34770]
- Fix TestMultiEventForEOFRetryHandlerInput unit test of CometD input {pull}34903[34903]
Expand Down Expand Up @@ -151,6 +152,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]

*Metricbeat*

- Fix `namespace` filter option on metricset `state_namespace` enricher. {pull}39934[39934]
constanca-m marked this conversation as resolved.
Show resolved Hide resolved
- Fix `namespace` filter option at Kubernetes provider level. {pull}39881[39881]
- Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294]
- Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720]
Expand Down
33 changes: 19 additions & 14 deletions libbeat/processors/add_kubernetes_metadata/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,10 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) {
}

watcher, err := kubernetes.NewNamedWatcher("add_kubernetes_metadata_pod", client, &kubernetes.Pod{}, kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
Node: config.Node,
Namespace: config.Namespace,
SyncTimeout: config.SyncPeriod,
Node: config.Node,
Namespace: config.Namespace,
HonorReSyncs: true,
}, nil)
if err != nil {
k.log.Errorf("Couldn't create kubernetes watcher for %T", &kubernetes.Pod{})
Expand All @@ -198,35 +199,37 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) {

metaConf := config.AddResourceMetadata

options := kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
Node: config.Node,
Namespace: config.Namespace,
}

if metaConf.Node.Enabled() {
nodeWatcher, err = kubernetes.NewNamedWatcher("add_kubernetes_metadata_node", client, &kubernetes.Node{}, options, nil)
nodeWatcher, err = kubernetes.NewNamedWatcher("add_kubernetes_metadata_node", client, &kubernetes.Node{}, kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
Node: config.Node,
HonorReSyncs: true,
}, nil)
if err != nil {
k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
}
}

if metaConf.Namespace.Enabled() {
namespaceWatcher, err = kubernetes.NewNamedWatcher("add_kubernetes_metadata_namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
SyncTimeout: config.SyncPeriod,
Namespace: config.Namespace,
HonorReSyncs: true,
}, nil)
if err != nil {
k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
}
}

// Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to
// Resource is Pod, so we need to create watchers for Replicasets and Jobs that it might belong to
// in order to be able to retrieve 2nd layer Owner metadata like in case of:
// Deployment -> Replicaset -> Pod
// CronJob -> job -> Pod
if metaConf.Deployment {
replicaSetWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_rs", client, &kubernetes.ReplicaSet{}, kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
SyncTimeout: config.SyncPeriod,
Namespace: config.Namespace,
HonorReSyncs: true,
}, nil)
if err != nil {
k.log.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.ReplicaSet{}, err)
Expand All @@ -235,7 +238,9 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) {
}
if metaConf.CronJob {
jobWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_job", client, &kubernetes.Job{}, kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
SyncTimeout: config.SyncPeriod,
Namespace: config.Namespace,
HonorReSyncs: true,
}, nil)
if err != nil {
k.log.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.Job{}, err)
Expand Down
16 changes: 8 additions & 8 deletions metricbeat/module/kubernetes/util/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,14 @@
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"

kubernetes2 "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes"
"github.com/elastic/beats/v7/metricbeat/mb"
"github.com/elastic/elastic-agent-autodiscover/kubernetes"
"github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata"
conf "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/elastic/elastic-agent-libs/mapstr"

kubernetes2 "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes"
"github.com/elastic/beats/v7/metricbeat/mb"
)

type kubernetesConfig struct {
Expand Down Expand Up @@ -281,8 +282,7 @@
}

func isNamespaced(resourceName string) bool {
if resourceName == NodeResource || resourceName == PersistentVolumeResource || resourceName == StorageClassResource ||
resourceName == NamespaceResource {
if resourceName == NodeResource || resourceName == PersistentVolumeResource || resourceName == StorageClassResource {
return false
}
return true
Expand Down Expand Up @@ -590,8 +590,8 @@
return &nilEnricher{}
}

var specificMetaGen metadata.MetaGen

Check failure on line 593 in metricbeat/module/kubernetes/util/kubernetes.go

View workflow job for this annotation

GitHub Actions / lint (darwin) 

specificMetaGen declared and not used (typecheck)
var generalMetaGen *metadata.Resource

Check failure on line 594 in metricbeat/module/kubernetes/util/kubernetes.go

View workflow job for this annotation

GitHub Actions / lint (darwin) 

generalMetaGen declared and not used (typecheck)
// Create the metadata generator to be used in the watcher's event handler.
// Both specificMetaGen and generalMetaGen implement Generate method for metadata collection.
if resourceName == ServiceResource || resourceName == PodResource {
Expand All @@ -604,11 +604,11 @@
return &nilEnricher{}
}

// updateFunc to be used as the resource watcher's add and update handler.
// updateFunc to be used as the resource watchers add and update handler.
// The handler function is executed when a watcher is triggered(i.e. new/updated resource).
// It is responsible for generating the metadata for a detected resource by executing the metadata generator's Generate method.
// It is responsible for generating the metadata for a detected resource by executing the metadata generators Generate method.
// It is a common handler for all resource watchers. The kind of resource(e.g. pod or deployment) is checked inside the function.
// It returns a map of a resourse identifier(i.e namespace-resource_name) as key and the metadata as value.
// It returns a map of a resource identifier(i.e. namespace-resource_name) as key and the metadata as value.
updateFunc := func(r kubernetes.Resource) map[string]mapstr.M {
accessor, _ := meta.Accessor(r)
id := accessor.GetName()
Expand Down Expand Up @@ -691,7 +691,7 @@
indexFunc := func(e mapstr.M) string {
name := getString(e, "name")
namespace := getString(e, mb.ModuleDataKey+".namespace")
id := ""
var id string
if name != "" && namespace != "" {
id = join(namespace, name)
} else if namespace != "" {
Expand Down
Loading