From 6dedace7fbbaf0e46c9129b783716c88230cce1c Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 7 Dec 2023 16:26:06 +0200 Subject: [PATCH 01/20] first update for nodePodUpdater --- .../autodiscover/providers/kubernetes/pod.go | 2 +- .../providers/kubernetes/pod_test.go | 29 +- .../providers/kubernetes/test1.yml | 411 ++++++++++++++++++ .../providers/kubernetes/test2.yml | 411 ++++++++++++++++++ 4 files changed, 848 insertions(+), 5 deletions(-) create mode 100644 libbeat/autodiscover/providers/kubernetes/test1.yml create mode 100644 libbeat/autodiscover/providers/kubernetes/test2.yml diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 31c7297a106e..b18d259b4bed 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -153,7 +153,7 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu watcher.AddEventHandler(p) if nodeWatcher != nil && (config.Hints.Enabled() || metaConf.Node.Enabled()) { - updater := kubernetes.NewNodePodUpdater(p.unlockedUpdate, watcher.Store(), &p.crossUpdate) + updater := kubernetes.NewNodePodUpdater(p.unlockedUpdate, watcher.Store(), p.nodeWatcher.Store(), &p.crossUpdate) nodeWatcher.AddEventHandler(updater) } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 4704dc6b8c75..c8076625dd78 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -2040,8 +2040,15 @@ func TestNodePodUpdater(t *testing.T) { } } + node := &kubernetes.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + cases := map[string]struct { pods []interface{} + nodes []interface{} expected []interface{} }{ "no pods": {}, @@ -2050,6 +2057,7 @@ func TestNodePodUpdater(t *testing.T) { pod("onepod", "foo"), pod("onepod", "bar"), }, + nodes: []interface{}{node}, expected: []interface{}{ pod("onepod", "foo"), }, @@ -2059,6 +2067,7 @@ func TestNodePodUpdater(t *testing.T) { pod("onepod", "bar"), pod("otherpod", "bar"), }, + nodes: []interface{}{node}, }, } @@ -2066,13 +2075,17 @@ func TestNodePodUpdater(t *testing.T) { t.Run(title, func(t *testing.T) { handler := &mockUpdaterHandler{} store := &mockUpdaterStore{objects: c.pods} - updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, &sync.Mutex{}) + storenode := &mockNodeStore{objects: c.nodes} + updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, storenode, &sync.Mutex{}) - node := &kubernetes.Node{ + //We simulate an update on the node with the addition of on label + node = &kubernetes.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", - }, - } + Labels: map[string]string{ + "beta.kubernetes.io/arch": "arm64", + }, + }} updater.OnUpdate(node) assert.EqualValues(t, c.expected, handler.objects) @@ -2096,6 +2109,14 @@ func (s *mockUpdaterStore) List() []interface{} { return s.objects } +type mockNodeStore struct { + objects []interface{} +} + +func (s *mockNodeStore) List() []interface{} { + return s.objects +} + func NewMockPodEventerManager(pod *pod) EventManager { em := &eventerManager{} em.eventer = pod diff --git a/libbeat/autodiscover/providers/kubernetes/test1.yml b/libbeat/autodiscover/providers/kubernetes/test1.yml new file mode 100644 index 000000000000..8f40f654f022 --- /dev/null +++ b/libbeat/autodiscover/providers/kubernetes/test1.yml @@ -0,0 +1,411 @@ +{ + "type":"MODIFIED", + "object":{ + "kind":"Node", + "apiVersion":"v1", + "metadata":{ + "name":"kind-control-plane", + "uid":"1d44438c-a5c6-4c62-a7a4-9e83723da205", + "resourceVersion":"27806", + "creationTimestamp":"2023-12-06T14:21:10Z", + "labels":{ + "beta.kubernetes.io/arch":"arm64", + "beta.kubernetes.io/os":"linux", + "katsoulis":"pasole", + "kubernetes.io/arch":"arm64", + "kubernetes.io/hostname":"kind-control-plane", + "kubernetes.io/os":"linux", + "node-role.kubernetes.io/control-plane":"", + "node.kubernetes.io/exclude-from-external-load-balancers":"" + }, + "annotations":{ + "kubeadm.alpha.kubernetes.io/cri-socket":"unix:///run/containerd/containerd.sock", + "node.alpha.kubernetes.io/ttl":"0", + "volumes.kubernetes.io/controller-managed-attach-detach":"true" + }, + "managedFields":[ + { + "manager":"kubelet", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-06T14:21:10Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:annotations":{ + ".":{ + + }, + "f:volumes.kubernetes.io/controller-managed-attach-detach":{ + + } + }, + "f:labels":{ + ".":{ + + }, + "f:beta.kubernetes.io/arch":{ + + }, + "f:beta.kubernetes.io/os":{ + + }, + "f:kubernetes.io/arch":{ + + }, + "f:kubernetes.io/hostname":{ + + }, + "f:kubernetes.io/os":{ + + } + } + }, + "f:spec":{ + "f:providerID":{ + + } + } + } + }, + { + "manager":"kubeadm", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-06T14:21:12Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:annotations":{ + "f:kubeadm.alpha.kubernetes.io/cri-socket":{ + + } + }, + "f:labels":{ + "f:node-role.kubernetes.io/control-plane":{ + + }, + "f:node.kubernetes.io/exclude-from-external-load-balancers":{ + + } + } + } + } + }, + { + "manager":"kube-controller-manager", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-06T14:21:25Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:annotations":{ + "f:node.alpha.kubernetes.io/ttl":{ + + } + } + }, + "f:spec":{ + "f:podCIDR":{ + + }, + "f:podCIDRs":{ + ".":{ + + }, + "v:\"10.244.0.0/24\"":{ + + } + } + } + } + }, + { + "manager":"kubectl-label", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-07T11:28:02Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:labels":{ + "f:katsoulis":{ + + } + } + } + } + }, + { + "manager":"kubelet", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-07T11:50:27Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:status":{ + "f:addresses":{ + "k:{\"type\":\"InternalIP\"}":{ + "f:address":{ + + } + } + }, + "f:allocatable":{ + "f:memory":{ + + } + }, + "f:capacity":{ + "f:memory":{ + + } + }, + "f:conditions":{ + "k:{\"type\":\"DiskPressure\"}":{ + "f:lastHeartbeatTime":{ + + } + }, + "k:{\"type\":\"MemoryPressure\"}":{ + "f:lastHeartbeatTime":{ + + } + }, + "k:{\"type\":\"PIDPressure\"}":{ + "f:lastHeartbeatTime":{ + + } + }, + "k:{\"type\":\"Ready\"}":{ + "f:lastHeartbeatTime":{ + + }, + "f:lastTransitionTime":{ + + }, + "f:message":{ + + }, + "f:reason":{ + + }, + "f:status":{ + + } + } + }, + "f:images":{ + + }, + "f:nodeInfo":{ + "f:bootID":{ + + }, + "f:machineID":{ + + }, + "f:systemUUID":{ + + } + } + } + }, + "subresource":"status" + } + ] + }, + "spec":{ + "podCIDR":"10.244.0.0/24", + "podCIDRs":[ + "10.244.0.0/24" + ], + "providerID":"kind://docker/kind/kind-control-plane" + }, + "status":{ + "capacity":{ + "cpu":"5", + "ephemeral-storage":"107077304Ki", + "hugepages-1Gi":"0", + "hugepages-2Mi":"0", + "hugepages-32Mi":"0", + "hugepages-64Ki":"0", + "memory":"10958488Ki", + "pods":"110" + }, + "allocatable":{ + "cpu":"5", + "ephemeral-storage":"107077304Ki", + "hugepages-1Gi":"0", + "hugepages-2Mi":"0", + "hugepages-32Mi":"0", + "hugepages-64Ki":"0", + "memory":"10958488Ki", + "pods":"110" + }, + "conditions":[ + { + "type":"MemoryPressure", + "status":"False", + "lastHeartbeatTime":"2023-12-07T11:50:27Z", + "lastTransitionTime":"2023-12-06T14:21:08Z", + "reason":"KubeletHasSufficientMemory", + "message":"kubelet has sufficient memory available" + }, + { + "type":"DiskPressure", + "status":"False", + "lastHeartbeatTime":"2023-12-07T11:50:27Z", + "lastTransitionTime":"2023-12-06T14:21:08Z", + "reason":"KubeletHasNoDiskPressure", + "message":"kubelet has no disk pressure" + }, + { + "type":"PIDPressure", + "status":"False", + "lastHeartbeatTime":"2023-12-07T11:50:27Z", + "lastTransitionTime":"2023-12-06T14:21:08Z", + "reason":"KubeletHasSufficientPID", + "message":"kubelet has sufficient PID available" + }, + { + "type":"Ready", + "status":"True", + "lastHeartbeatTime":"2023-12-07T11:50:27Z", + "lastTransitionTime":"2023-12-06T14:21:28Z", + "reason":"KubeletReady", + "message":"kubelet is posting ready status" + } + ], + "addresses":[ + { + "type":"InternalIP", + "address":"172.19.0.2" + }, + { + "type":"Hostname", + "address":"kind-control-plane" + } + ], + "daemonEndpoints":{ + "kubeletEndpoint":{ + "Port":10250 + } + }, + "nodeInfo":{ + "machineID":"3489e0d51f694e4385ee5dedac53736c", + "systemUUID":"3489e0d51f694e4385ee5dedac53736c", + "bootID":"a28a8c69-2fc4-4bda-afe3-54644dd3a089", + "kernelVersion":"5.10.104-linuxkit", + "osImage":"Debian GNU/Linux 11 (bullseye)", + "containerRuntimeVersion":"containerd://1.7.1", + "kubeletVersion":"v1.27.3", + "kubeProxyVersion":"v1.27.3", + "operatingSystem":"linux", + "architecture":"arm64" + }, + "images":[ + { + "names":[ + "docker.elastic.co/beats/metricbeat@sha256:9972dd91739655e20e49a0edb4e26811dca75f0740318a29ba26fc086d3d78af", + "docker.elastic.co/beats/metricbeat:8.12.0-SNAPSHOT" + ], + "sizeBytes":149012603 + }, + { + "names":[ + "docker.elastic.co/beats/filebeat@sha256:9c2e3c7b8c1a03fcdb2021734898e8ad33aefb77d2542b373bc3ba5e154d8be2", + "docker.elastic.co/beats/filebeat:8.12.0-SNAPSHOT" + ], + "sizeBytes":141949252 + }, + { + "names":[ + "docker.elastic.co/beats/metricbeat@sha256:40f8a6b2b6dfad26295112b144c83cfbe793ae667d7cce687d12b8d898b92937", + "docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT" + ], + "sizeBytes":103670593 + }, + { + "names":[ + "docker.elastic.co/beats/filebeat@sha256:9b3cec4fcbb3a1a8b6d72efcd6ebaa68eb17a942c8be48b20a7c6a11f1f3a6dc", + "docker.elastic.co/beats/filebeat:8.0.0-SNAPSHOT" + ], + "sizeBytes":95587869 + }, + { + "names":[ + "registry.k8s.io/etcd:3.5.7-0" + ], + "sizeBytes":80665728 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:024134bb4e61a4975c5c15db76a7e3571eea74eff90c12a730781bf9c9daedd0", + "registry.k8s.io/kube-apiserver:v1.27.3" + ], + "sizeBytes":79827455 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:336af6020e6377f586577425f1314d95067a9b4a1c8707e90d5062823ec03f43", + "registry.k8s.io/kube-controller-manager:v1.27.3" + ], + "sizeBytes":71504013 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:67c927df0c79fb7415d0fc925570687fbec9a36cf13145a6430f9da059965dad", + "registry.k8s.io/kube-proxy:v1.27.3" + ], + "sizeBytes":68097533 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:4b3d947dd06f4f5e4f417ec4da4bc83017f7cb5387d053ea2b6b906825011b7c", + "registry.k8s.io/kube-scheduler:v1.27.3" + ], + "sizeBytes":57610381 + }, + { + "names":[ + "docker.io/kindest/kindnetd:v20230511-dc714da8" + ], + "sizeBytes":25334607 + }, + { + "names":[ + "docker.io/kindest/local-path-provisioner:v20230511-dc714da8" + ], + "sizeBytes":17275248 + }, + { + "names":[ + "registry.k8s.io/coredns/coredns:v1.10.1" + ], + "sizeBytes":14557471 + }, + { + "names":[ + "registry.k8s.io/kube-state-metrics/kube-state-metrics@sha256:a15ca437f2309b769caa36a88240ce0266d67028f02da90f1c817c849b144273", + "registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.7.0" + ], + "sizeBytes":11371848 + }, + { + "names":[ + "docker.io/kindest/local-path-helper:v20230510-486859a6" + ], + "sizeBytes":2922287 + }, + { + "names":[ + "registry.k8s.io/pause:3.7" + ], + "sizeBytes":268400 + } + ] + } + } +} \ No newline at end of file diff --git a/libbeat/autodiscover/providers/kubernetes/test2.yml b/libbeat/autodiscover/providers/kubernetes/test2.yml new file mode 100644 index 000000000000..599491bd4524 --- /dev/null +++ b/libbeat/autodiscover/providers/kubernetes/test2.yml @@ -0,0 +1,411 @@ +{ + "type":"MODIFIED", + "object":{ + "kind":"Node", + "apiVersion":"v1", + "metadata":{ + "name":"kind-control-plane", + "uid":"1d44438c-a5c6-4c62-a7a4-9e83723da205", + "resourceVersion":"28202", + "creationTimestamp":"2023-12-06T14:21:10Z", + "labels":{ + "beta.kubernetes.io/arch":"arm64", + "beta.kubernetes.io/os":"linux", + "katsoulis":"pasole", + "kubernetes.io/arch":"arm64", + "kubernetes.io/hostname":"kind-control-plane", + "kubernetes.io/os":"linux", + "node-role.kubernetes.io/control-plane":"", + "node.kubernetes.io/exclude-from-external-load-balancers":"" + }, + "annotations":{ + "kubeadm.alpha.kubernetes.io/cri-socket":"unix:///run/containerd/containerd.sock", + "node.alpha.kubernetes.io/ttl":"0", + "volumes.kubernetes.io/controller-managed-attach-detach":"true" + }, + "managedFields":[ + { + "manager":"kubelet", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-06T14:21:10Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:annotations":{ + ".":{ + + }, + "f:volumes.kubernetes.io/controller-managed-attach-detach":{ + + } + }, + "f:labels":{ + ".":{ + + }, + "f:beta.kubernetes.io/arch":{ + + }, + "f:beta.kubernetes.io/os":{ + + }, + "f:kubernetes.io/arch":{ + + }, + "f:kubernetes.io/hostname":{ + + }, + "f:kubernetes.io/os":{ + + } + } + }, + "f:spec":{ + "f:providerID":{ + + } + } + } + }, + { + "manager":"kubeadm", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-06T14:21:12Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:annotations":{ + "f:kubeadm.alpha.kubernetes.io/cri-socket":{ + + } + }, + "f:labels":{ + "f:node-role.kubernetes.io/control-plane":{ + + }, + "f:node.kubernetes.io/exclude-from-external-load-balancers":{ + + } + } + } + } + }, + { + "manager":"kube-controller-manager", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-06T14:21:25Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:annotations":{ + "f:node.alpha.kubernetes.io/ttl":{ + + } + } + }, + "f:spec":{ + "f:podCIDR":{ + + }, + "f:podCIDRs":{ + ".":{ + + }, + "v:\"10.244.0.0/24\"":{ + + } + } + } + } + }, + { + "manager":"kubectl-label", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-07T11:28:02Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:metadata":{ + "f:labels":{ + "f:katsoulis":{ + + } + } + } + } + }, + { + "manager":"kubelet", + "operation":"Update", + "apiVersion":"v1", + "time":"2023-12-07T11:55:33Z", + "fieldsType":"FieldsV1", + "fieldsV1":{ + "f:status":{ + "f:addresses":{ + "k:{\"type\":\"InternalIP\"}":{ + "f:address":{ + + } + } + }, + "f:allocatable":{ + "f:memory":{ + + } + }, + "f:capacity":{ + "f:memory":{ + + } + }, + "f:conditions":{ + "k:{\"type\":\"DiskPressure\"}":{ + "f:lastHeartbeatTime":{ + + } + }, + "k:{\"type\":\"MemoryPressure\"}":{ + "f:lastHeartbeatTime":{ + + } + }, + "k:{\"type\":\"PIDPressure\"}":{ + "f:lastHeartbeatTime":{ + + } + }, + "k:{\"type\":\"Ready\"}":{ + "f:lastHeartbeatTime":{ + + }, + "f:lastTransitionTime":{ + + }, + "f:message":{ + + }, + "f:reason":{ + + }, + "f:status":{ + + } + } + }, + "f:images":{ + + }, + "f:nodeInfo":{ + "f:bootID":{ + + }, + "f:machineID":{ + + }, + "f:systemUUID":{ + + } + } + } + }, + "subresource":"status" + } + ] + }, + "spec":{ + "podCIDR":"10.244.0.0/24", + "podCIDRs":[ + "10.244.0.0/24" + ], + "providerID":"kind://docker/kind/kind-control-plane" + }, + "status":{ + "capacity":{ + "cpu":"5", + "ephemeral-storage":"107077304Ki", + "hugepages-1Gi":"0", + "hugepages-2Mi":"0", + "hugepages-32Mi":"0", + "hugepages-64Ki":"0", + "memory":"10958488Ki", + "pods":"110" + }, + "allocatable":{ + "cpu":"5", + "ephemeral-storage":"107077304Ki", + "hugepages-1Gi":"0", + "hugepages-2Mi":"0", + "hugepages-32Mi":"0", + "hugepages-64Ki":"0", + "memory":"10958488Ki", + "pods":"110" + }, + "conditions":[ + { + "type":"MemoryPressure", + "status":"False", + "lastHeartbeatTime":"2023-12-07T11:55:33Z", + "lastTransitionTime":"2023-12-06T14:21:08Z", + "reason":"KubeletHasSufficientMemory", + "message":"kubelet has sufficient memory available" + }, + { + "type":"DiskPressure", + "status":"False", + "lastHeartbeatTime":"2023-12-07T11:55:33Z", + "lastTransitionTime":"2023-12-06T14:21:08Z", + "reason":"KubeletHasNoDiskPressure", + "message":"kubelet has no disk pressure" + }, + { + "type":"PIDPressure", + "status":"False", + "lastHeartbeatTime":"2023-12-07T11:55:33Z", + "lastTransitionTime":"2023-12-06T14:21:08Z", + "reason":"KubeletHasSufficientPID", + "message":"kubelet has sufficient PID available" + }, + { + "type":"Ready", + "status":"True", + "lastHeartbeatTime":"2023-12-07T11:55:33Z", + "lastTransitionTime":"2023-12-06T14:21:28Z", + "reason":"KubeletReady", + "message":"kubelet is posting ready status" + } + ], + "addresses":[ + { + "type":"InternalIP", + "address":"172.19.0.2" + }, + { + "type":"Hostname", + "address":"kind-control-plane" + } + ], + "daemonEndpoints":{ + "kubeletEndpoint":{ + "Port":10250 + } + }, + "nodeInfo":{ + "machineID":"3489e0d51f694e4385ee5dedac53736c", + "systemUUID":"3489e0d51f694e4385ee5dedac53736c", + "bootID":"a28a8c69-2fc4-4bda-afe3-54644dd3a089", + "kernelVersion":"5.10.104-linuxkit", + "osImage":"Debian GNU/Linux 11 (bullseye)", + "containerRuntimeVersion":"containerd://1.7.1", + "kubeletVersion":"v1.27.3", + "kubeProxyVersion":"v1.27.3", + "operatingSystem":"linux", + "architecture":"arm64" + }, + "images":[ + { + "names":[ + "docker.elastic.co/beats/metricbeat@sha256:9972dd91739655e20e49a0edb4e26811dca75f0740318a29ba26fc086d3d78af", + "docker.elastic.co/beats/metricbeat:8.12.0-SNAPSHOT" + ], + "sizeBytes":149012603 + }, + { + "names":[ + "docker.elastic.co/beats/filebeat@sha256:9c2e3c7b8c1a03fcdb2021734898e8ad33aefb77d2542b373bc3ba5e154d8be2", + "docker.elastic.co/beats/filebeat:8.12.0-SNAPSHOT" + ], + "sizeBytes":141949252 + }, + { + "names":[ + "docker.elastic.co/beats/metricbeat@sha256:40f8a6b2b6dfad26295112b144c83cfbe793ae667d7cce687d12b8d898b92937", + "docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT" + ], + "sizeBytes":103670593 + }, + { + "names":[ + "docker.elastic.co/beats/filebeat@sha256:9b3cec4fcbb3a1a8b6d72efcd6ebaa68eb17a942c8be48b20a7c6a11f1f3a6dc", + "docker.elastic.co/beats/filebeat:8.0.0-SNAPSHOT" + ], + "sizeBytes":95587869 + }, + { + "names":[ + "registry.k8s.io/etcd:3.5.7-0" + ], + "sizeBytes":80665728 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:024134bb4e61a4975c5c15db76a7e3571eea74eff90c12a730781bf9c9daedd0", + "registry.k8s.io/kube-apiserver:v1.27.3" + ], + "sizeBytes":79827455 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:336af6020e6377f586577425f1314d95067a9b4a1c8707e90d5062823ec03f43", + "registry.k8s.io/kube-controller-manager:v1.27.3" + ], + "sizeBytes":71504013 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:67c927df0c79fb7415d0fc925570687fbec9a36cf13145a6430f9da059965dad", + "registry.k8s.io/kube-proxy:v1.27.3" + ], + "sizeBytes":68097533 + }, + { + "names":[ + "docker.io/library/import-2023-06-15@sha256:4b3d947dd06f4f5e4f417ec4da4bc83017f7cb5387d053ea2b6b906825011b7c", + "registry.k8s.io/kube-scheduler:v1.27.3" + ], + "sizeBytes":57610381 + }, + { + "names":[ + "docker.io/kindest/kindnetd:v20230511-dc714da8" + ], + "sizeBytes":25334607 + }, + { + "names":[ + "docker.io/kindest/local-path-provisioner:v20230511-dc714da8" + ], + "sizeBytes":17275248 + }, + { + "names":[ + "registry.k8s.io/coredns/coredns:v1.10.1" + ], + "sizeBytes":14557471 + }, + { + "names":[ + "registry.k8s.io/kube-state-metrics/kube-state-metrics@sha256:a15ca437f2309b769caa36a88240ce0266d67028f02da90f1c817c849b144273", + "registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.7.0" + ], + "sizeBytes":11371848 + }, + { + "names":[ + "docker.io/kindest/local-path-helper:v20230510-486859a6" + ], + "sizeBytes":2922287 + }, + { + "names":[ + "registry.k8s.io/pause:3.7" + ], + "sizeBytes":268400 + } + ] + } + } +} \ No newline at end of file From 314885f9f762ab2db65ca6be5a9224d54b99ff37 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 8 Dec 2023 14:28:19 +0200 Subject: [PATCH 02/20] first update for nodePodUpdater with key function --- .../providers/kubernetes/pod_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index c8076625dd78..dbe276caca97 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -2047,8 +2047,8 @@ func TestNodePodUpdater(t *testing.T) { } cases := map[string]struct { - pods []interface{} - nodes []interface{} + pods []interface{} + expected []interface{} }{ "no pods": {}, @@ -2057,7 +2057,6 @@ func TestNodePodUpdater(t *testing.T) { pod("onepod", "foo"), pod("onepod", "bar"), }, - nodes: []interface{}{node}, expected: []interface{}{ pod("onepod", "foo"), }, @@ -2067,7 +2066,6 @@ func TestNodePodUpdater(t *testing.T) { pod("onepod", "bar"), pod("otherpod", "bar"), }, - nodes: []interface{}{node}, }, } @@ -2075,18 +2073,18 @@ func TestNodePodUpdater(t *testing.T) { t.Run(title, func(t *testing.T) { handler := &mockUpdaterHandler{} store := &mockUpdaterStore{objects: c.pods} - storenode := &mockNodeStore{objects: c.nodes} + storenode := &mockNodeStore{objects: node, exist: true, err: nil} updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, storenode, &sync.Mutex{}) //We simulate an update on the node with the addition of on label - node = &kubernetes.Node{ + node1 := &kubernetes.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Labels: map[string]string{ "beta.kubernetes.io/arch": "arm64", }, }} - updater.OnUpdate(node) + updater.OnUpdate(node1) assert.EqualValues(t, c.expected, handler.objects) }) @@ -2110,11 +2108,13 @@ func (s *mockUpdaterStore) List() []interface{} { } type mockNodeStore struct { - objects []interface{} + objects interface{} + exist bool + err error } -func (s *mockNodeStore) List() []interface{} { - return s.objects +func (s *mockNodeStore) GetByKey(string) (interface{}, bool, error) { + return s.objects, s.exist, s.err } func NewMockPodEventerManager(pod *pod) EventManager { From a44a7b8e94baac19dda7c1e3a4e9001538a00467 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 8 Dec 2023 17:26:49 +0200 Subject: [PATCH 03/20] first update for namespacePodUpdater --- .../autodiscover/providers/kubernetes/pod.go | 2 +- .../providers/kubernetes/pod_test.go | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index b18d259b4bed..92d87474cb46 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -158,7 +158,7 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu } if namespaceWatcher != nil && (config.Hints.Enabled() || metaConf.Namespace.Enabled()) { - updater := kubernetes.NewNamespacePodUpdater(p.unlockedUpdate, watcher.Store(), &p.crossUpdate) + updater := kubernetes.NewNamespacePodUpdater(p.unlockedUpdate, watcher.Store(), p.namespaceWatcher.Store(), &p.crossUpdate) namespaceWatcher.AddEventHandler(updater) } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index dbe276caca97..cfde56572bfe 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -1988,6 +1988,11 @@ func TestNamespacePodUpdater(t *testing.T) { } } + namespace := &kubernetes.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }} + cases := map[string]struct { pods []interface{} expected []interface{} @@ -2014,14 +2019,19 @@ func TestNamespacePodUpdater(t *testing.T) { t.Run(title, func(t *testing.T) { handler := &mockUpdaterHandler{} store := &mockUpdaterStore{objects: c.pods} - updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, &sync.Mutex{}) + storenamespace := &mockNamespaceStore{objects: namespace, exist: true, err: nil} + updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, storenamespace, &sync.Mutex{}) - namespace := &kubernetes.Namespace{ + //We simulate an update on the namespace with the addition of one label + namespace1 := &kubernetes.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", - }, - } - updater.OnUpdate(namespace) + Labels: map[string]string{ + "beta.kubernetes.io/arch": "arm64", + }, + }} + + updater.OnUpdate(namespace1) assert.EqualValues(t, c.expected, handler.objects) }) @@ -2076,7 +2086,7 @@ func TestNodePodUpdater(t *testing.T) { storenode := &mockNodeStore{objects: node, exist: true, err: nil} updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, storenode, &sync.Mutex{}) - //We simulate an update on the node with the addition of on label + //We simulate an update on the node with the addition of one label node1 := &kubernetes.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", @@ -2107,6 +2117,16 @@ func (s *mockUpdaterStore) List() []interface{} { return s.objects } +type mockNamespaceStore struct { + objects interface{} + exist bool + err error +} + +func (s *mockNamespaceStore) GetByKey(string) (interface{}, bool, error) { + return s.objects, s.exist, s.err +} + type mockNodeStore struct { objects interface{} exist bool From 39de52d43643b32bbda06b27f7f51da158b4f8cc Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Tue, 12 Dec 2023 16:56:39 +0200 Subject: [PATCH 04/20] updating with delta --- .../autodiscover/providers/kubernetes/pod.go | 3 ++- .../providers/kubernetes/pod_test.go | 23 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 92d87474cb46..c69bf39efa85 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -158,7 +158,7 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu } if namespaceWatcher != nil && (config.Hints.Enabled() || metaConf.Namespace.Enabled()) { - updater := kubernetes.NewNamespacePodUpdater(p.unlockedUpdate, watcher.Store(), p.namespaceWatcher.Store(), &p.crossUpdate) + updater := kubernetes.NewNamespacePodUpdater(p.unlockedUpdate, watcher.Store(), p.namespaceWatcher.Store(), p.namespaceWatcher, &p.crossUpdate) namespaceWatcher.AddEventHandler(updater) } @@ -183,6 +183,7 @@ func (p *pod) OnUpdate(obj interface{}) { } func (p *pod) unlockedUpdate(obj interface{}) { + p.logger.Infof("Watcher Pod update: %+v", obj) p.logger.Debugf("Watcher Pod update: %+v", obj) p.emit(obj.(*kubernetes.Pod), "stop") p.emit(obj.(*kubernetes.Pod), "start") diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index cfde56572bfe..10f05352f871 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" k8sfake "k8s.io/client-go/kubernetes/fake" @@ -2020,8 +2021,6 @@ func TestNamespacePodUpdater(t *testing.T) { handler := &mockUpdaterHandler{} store := &mockUpdaterStore{objects: c.pods} storenamespace := &mockNamespaceStore{objects: namespace, exist: true, err: nil} - updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, storenamespace, &sync.Mutex{}) - //We simulate an update on the namespace with the addition of one label namespace1 := &kubernetes.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -2030,6 +2029,11 @@ func TestNamespacePodUpdater(t *testing.T) { "beta.kubernetes.io/arch": "arm64", }, }} + var deltaslice []runtime.Object + deltaslice = append(deltaslice, namespace) + deltaslice = append(deltaslice, namespace1) + watcher := &mockUpdaterNamespaceWatcher{objects: namespace, delta: deltaslice} + updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, storenamespace, watcher, &sync.Mutex{}) updater.OnUpdate(namespace1) @@ -2083,8 +2087,6 @@ func TestNodePodUpdater(t *testing.T) { t.Run(title, func(t *testing.T) { handler := &mockUpdaterHandler{} store := &mockUpdaterStore{objects: c.pods} - storenode := &mockNodeStore{objects: node, exist: true, err: nil} - updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, storenode, &sync.Mutex{}) //We simulate an update on the node with the addition of one label node1 := &kubernetes.Node{ @@ -2094,6 +2096,10 @@ func TestNodePodUpdater(t *testing.T) { "beta.kubernetes.io/arch": "arm64", }, }} + + storenode := &mockNodeStore{objects: node, exist: true, err: nil} + updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, storenode, &sync.Mutex{}) + updater.OnUpdate(node1) assert.EqualValues(t, c.expected, handler.objects) @@ -2113,6 +2119,15 @@ type mockUpdaterStore struct { objects []interface{} } +type mockUpdaterNamespaceWatcher struct { + objects interface{} + delta []runtime.Object +} + +func (s *mockUpdaterNamespaceWatcher) Deltaslice() []runtime.Object { + return s.delta +} + func (s *mockUpdaterStore) List() []interface{} { return s.objects } From 7511a0dcbf592e2a50abd47f67fee1809f981d3d Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 13 Dec 2023 10:25:10 +0200 Subject: [PATCH 05/20] fixing referneces only to one wather as an argument --- .../autodiscover/providers/kubernetes/pod.go | 4 ++-- .../providers/kubernetes/pod_test.go | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index c69bf39efa85..e994d1b8b229 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -153,12 +153,12 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu watcher.AddEventHandler(p) if nodeWatcher != nil && (config.Hints.Enabled() || metaConf.Node.Enabled()) { - updater := kubernetes.NewNodePodUpdater(p.unlockedUpdate, watcher.Store(), p.nodeWatcher.Store(), &p.crossUpdate) + updater := kubernetes.NewNodePodUpdater(p.unlockedUpdate, watcher.Store(), p.nodeWatcher, &p.crossUpdate) nodeWatcher.AddEventHandler(updater) } if namespaceWatcher != nil && (config.Hints.Enabled() || metaConf.Namespace.Enabled()) { - updater := kubernetes.NewNamespacePodUpdater(p.unlockedUpdate, watcher.Store(), p.namespaceWatcher.Store(), p.namespaceWatcher, &p.crossUpdate) + updater := kubernetes.NewNamespacePodUpdater(p.unlockedUpdate, watcher.Store(), p.namespaceWatcher, &p.crossUpdate) namespaceWatcher.AddEventHandler(updater) } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 10f05352f871..36ac9872021e 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -2020,7 +2020,6 @@ func TestNamespacePodUpdater(t *testing.T) { t.Run(title, func(t *testing.T) { handler := &mockUpdaterHandler{} store := &mockUpdaterStore{objects: c.pods} - storenamespace := &mockNamespaceStore{objects: namespace, exist: true, err: nil} //We simulate an update on the namespace with the addition of one label namespace1 := &kubernetes.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -2032,8 +2031,8 @@ func TestNamespacePodUpdater(t *testing.T) { var deltaslice []runtime.Object deltaslice = append(deltaslice, namespace) deltaslice = append(deltaslice, namespace1) - watcher := &mockUpdaterNamespaceWatcher{objects: namespace, delta: deltaslice} - updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, storenamespace, watcher, &sync.Mutex{}) + watcher := &mockUpdaterWatcher{delta: deltaslice} + updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, watcher, &sync.Mutex{}) updater.OnUpdate(namespace1) @@ -2092,14 +2091,18 @@ func TestNodePodUpdater(t *testing.T) { node1 := &kubernetes.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", - Labels: map[string]string{ + Annotations: map[string]string{ "beta.kubernetes.io/arch": "arm64", }, }} - storenode := &mockNodeStore{objects: node, exist: true, err: nil} - updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, storenode, &sync.Mutex{}) + var deltaslice []runtime.Object + deltaslice = append(deltaslice, node) + deltaslice = append(deltaslice, node1) + watcher := &mockUpdaterWatcher{delta: deltaslice} + updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, watcher, &sync.Mutex{}) + //This is when the update happens. updater.OnUpdate(node1) assert.EqualValues(t, c.expected, handler.objects) @@ -2119,12 +2122,11 @@ type mockUpdaterStore struct { objects []interface{} } -type mockUpdaterNamespaceWatcher struct { - objects interface{} - delta []runtime.Object +type mockUpdaterWatcher struct { + delta []runtime.Object } -func (s *mockUpdaterNamespaceWatcher) Deltaslice() []runtime.Object { +func (s *mockUpdaterWatcher) Deltaobjects() []runtime.Object { return s.delta } From 6d55935f194d84b9f8e84d9e47f45a40ba14e4d3 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 13 Dec 2023 11:21:31 +0200 Subject: [PATCH 06/20] removing comments --- libbeat/autodiscover/providers/kubernetes/pod.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index e994d1b8b229..b180adbc6d33 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -183,7 +183,6 @@ func (p *pod) OnUpdate(obj interface{}) { } func (p *pod) unlockedUpdate(obj interface{}) { - p.logger.Infof("Watcher Pod update: %+v", obj) p.logger.Debugf("Watcher Pod update: %+v", obj) p.emit(obj.(*kubernetes.Pod), "stop") p.emit(obj.(*kubernetes.Pod), "start") From 3f2132e6dfb3e10b4b09cee9d2f3fb13cfec5819 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 13 Dec 2023 11:38:57 +0200 Subject: [PATCH 07/20] removing test files --- .../providers/kubernetes/test1.yml | 411 ------------------ .../providers/kubernetes/test2.yml | 411 ------------------ 2 files changed, 822 deletions(-) delete mode 100644 libbeat/autodiscover/providers/kubernetes/test1.yml delete mode 100644 libbeat/autodiscover/providers/kubernetes/test2.yml diff --git a/libbeat/autodiscover/providers/kubernetes/test1.yml b/libbeat/autodiscover/providers/kubernetes/test1.yml deleted file mode 100644 index 8f40f654f022..000000000000 --- a/libbeat/autodiscover/providers/kubernetes/test1.yml +++ /dev/null @@ -1,411 +0,0 @@ -{ - "type":"MODIFIED", - "object":{ - "kind":"Node", - "apiVersion":"v1", - "metadata":{ - "name":"kind-control-plane", - "uid":"1d44438c-a5c6-4c62-a7a4-9e83723da205", - "resourceVersion":"27806", - "creationTimestamp":"2023-12-06T14:21:10Z", - "labels":{ - "beta.kubernetes.io/arch":"arm64", - "beta.kubernetes.io/os":"linux", - "katsoulis":"pasole", - "kubernetes.io/arch":"arm64", - "kubernetes.io/hostname":"kind-control-plane", - "kubernetes.io/os":"linux", - "node-role.kubernetes.io/control-plane":"", - "node.kubernetes.io/exclude-from-external-load-balancers":"" - }, - "annotations":{ - "kubeadm.alpha.kubernetes.io/cri-socket":"unix:///run/containerd/containerd.sock", - "node.alpha.kubernetes.io/ttl":"0", - "volumes.kubernetes.io/controller-managed-attach-detach":"true" - }, - "managedFields":[ - { - "manager":"kubelet", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-06T14:21:10Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:annotations":{ - ".":{ - - }, - "f:volumes.kubernetes.io/controller-managed-attach-detach":{ - - } - }, - "f:labels":{ - ".":{ - - }, - "f:beta.kubernetes.io/arch":{ - - }, - "f:beta.kubernetes.io/os":{ - - }, - "f:kubernetes.io/arch":{ - - }, - "f:kubernetes.io/hostname":{ - - }, - "f:kubernetes.io/os":{ - - } - } - }, - "f:spec":{ - "f:providerID":{ - - } - } - } - }, - { - "manager":"kubeadm", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-06T14:21:12Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:annotations":{ - "f:kubeadm.alpha.kubernetes.io/cri-socket":{ - - } - }, - "f:labels":{ - "f:node-role.kubernetes.io/control-plane":{ - - }, - "f:node.kubernetes.io/exclude-from-external-load-balancers":{ - - } - } - } - } - }, - { - "manager":"kube-controller-manager", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-06T14:21:25Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:annotations":{ - "f:node.alpha.kubernetes.io/ttl":{ - - } - } - }, - "f:spec":{ - "f:podCIDR":{ - - }, - "f:podCIDRs":{ - ".":{ - - }, - "v:\"10.244.0.0/24\"":{ - - } - } - } - } - }, - { - "manager":"kubectl-label", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-07T11:28:02Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:labels":{ - "f:katsoulis":{ - - } - } - } - } - }, - { - "manager":"kubelet", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-07T11:50:27Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:status":{ - "f:addresses":{ - "k:{\"type\":\"InternalIP\"}":{ - "f:address":{ - - } - } - }, - "f:allocatable":{ - "f:memory":{ - - } - }, - "f:capacity":{ - "f:memory":{ - - } - }, - "f:conditions":{ - "k:{\"type\":\"DiskPressure\"}":{ - "f:lastHeartbeatTime":{ - - } - }, - "k:{\"type\":\"MemoryPressure\"}":{ - "f:lastHeartbeatTime":{ - - } - }, - "k:{\"type\":\"PIDPressure\"}":{ - "f:lastHeartbeatTime":{ - - } - }, - "k:{\"type\":\"Ready\"}":{ - "f:lastHeartbeatTime":{ - - }, - "f:lastTransitionTime":{ - - }, - "f:message":{ - - }, - "f:reason":{ - - }, - "f:status":{ - - } - } - }, - "f:images":{ - - }, - "f:nodeInfo":{ - "f:bootID":{ - - }, - "f:machineID":{ - - }, - "f:systemUUID":{ - - } - } - } - }, - "subresource":"status" - } - ] - }, - "spec":{ - "podCIDR":"10.244.0.0/24", - "podCIDRs":[ - "10.244.0.0/24" - ], - "providerID":"kind://docker/kind/kind-control-plane" - }, - "status":{ - "capacity":{ - "cpu":"5", - "ephemeral-storage":"107077304Ki", - "hugepages-1Gi":"0", - "hugepages-2Mi":"0", - "hugepages-32Mi":"0", - "hugepages-64Ki":"0", - "memory":"10958488Ki", - "pods":"110" - }, - "allocatable":{ - "cpu":"5", - "ephemeral-storage":"107077304Ki", - "hugepages-1Gi":"0", - "hugepages-2Mi":"0", - "hugepages-32Mi":"0", - "hugepages-64Ki":"0", - "memory":"10958488Ki", - "pods":"110" - }, - "conditions":[ - { - "type":"MemoryPressure", - "status":"False", - "lastHeartbeatTime":"2023-12-07T11:50:27Z", - "lastTransitionTime":"2023-12-06T14:21:08Z", - "reason":"KubeletHasSufficientMemory", - "message":"kubelet has sufficient memory available" - }, - { - "type":"DiskPressure", - "status":"False", - "lastHeartbeatTime":"2023-12-07T11:50:27Z", - "lastTransitionTime":"2023-12-06T14:21:08Z", - "reason":"KubeletHasNoDiskPressure", - "message":"kubelet has no disk pressure" - }, - { - "type":"PIDPressure", - "status":"False", - "lastHeartbeatTime":"2023-12-07T11:50:27Z", - "lastTransitionTime":"2023-12-06T14:21:08Z", - "reason":"KubeletHasSufficientPID", - "message":"kubelet has sufficient PID available" - }, - { - "type":"Ready", - "status":"True", - "lastHeartbeatTime":"2023-12-07T11:50:27Z", - "lastTransitionTime":"2023-12-06T14:21:28Z", - "reason":"KubeletReady", - "message":"kubelet is posting ready status" - } - ], - "addresses":[ - { - "type":"InternalIP", - "address":"172.19.0.2" - }, - { - "type":"Hostname", - "address":"kind-control-plane" - } - ], - "daemonEndpoints":{ - "kubeletEndpoint":{ - "Port":10250 - } - }, - "nodeInfo":{ - "machineID":"3489e0d51f694e4385ee5dedac53736c", - "systemUUID":"3489e0d51f694e4385ee5dedac53736c", - "bootID":"a28a8c69-2fc4-4bda-afe3-54644dd3a089", - "kernelVersion":"5.10.104-linuxkit", - "osImage":"Debian GNU/Linux 11 (bullseye)", - "containerRuntimeVersion":"containerd://1.7.1", - "kubeletVersion":"v1.27.3", - "kubeProxyVersion":"v1.27.3", - "operatingSystem":"linux", - "architecture":"arm64" - }, - "images":[ - { - "names":[ - "docker.elastic.co/beats/metricbeat@sha256:9972dd91739655e20e49a0edb4e26811dca75f0740318a29ba26fc086d3d78af", - "docker.elastic.co/beats/metricbeat:8.12.0-SNAPSHOT" - ], - "sizeBytes":149012603 - }, - { - "names":[ - "docker.elastic.co/beats/filebeat@sha256:9c2e3c7b8c1a03fcdb2021734898e8ad33aefb77d2542b373bc3ba5e154d8be2", - "docker.elastic.co/beats/filebeat:8.12.0-SNAPSHOT" - ], - "sizeBytes":141949252 - }, - { - "names":[ - "docker.elastic.co/beats/metricbeat@sha256:40f8a6b2b6dfad26295112b144c83cfbe793ae667d7cce687d12b8d898b92937", - "docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT" - ], - "sizeBytes":103670593 - }, - { - "names":[ - "docker.elastic.co/beats/filebeat@sha256:9b3cec4fcbb3a1a8b6d72efcd6ebaa68eb17a942c8be48b20a7c6a11f1f3a6dc", - "docker.elastic.co/beats/filebeat:8.0.0-SNAPSHOT" - ], - "sizeBytes":95587869 - }, - { - "names":[ - "registry.k8s.io/etcd:3.5.7-0" - ], - "sizeBytes":80665728 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:024134bb4e61a4975c5c15db76a7e3571eea74eff90c12a730781bf9c9daedd0", - "registry.k8s.io/kube-apiserver:v1.27.3" - ], - "sizeBytes":79827455 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:336af6020e6377f586577425f1314d95067a9b4a1c8707e90d5062823ec03f43", - "registry.k8s.io/kube-controller-manager:v1.27.3" - ], - "sizeBytes":71504013 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:67c927df0c79fb7415d0fc925570687fbec9a36cf13145a6430f9da059965dad", - "registry.k8s.io/kube-proxy:v1.27.3" - ], - "sizeBytes":68097533 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:4b3d947dd06f4f5e4f417ec4da4bc83017f7cb5387d053ea2b6b906825011b7c", - "registry.k8s.io/kube-scheduler:v1.27.3" - ], - "sizeBytes":57610381 - }, - { - "names":[ - "docker.io/kindest/kindnetd:v20230511-dc714da8" - ], - "sizeBytes":25334607 - }, - { - "names":[ - "docker.io/kindest/local-path-provisioner:v20230511-dc714da8" - ], - "sizeBytes":17275248 - }, - { - "names":[ - "registry.k8s.io/coredns/coredns:v1.10.1" - ], - "sizeBytes":14557471 - }, - { - "names":[ - "registry.k8s.io/kube-state-metrics/kube-state-metrics@sha256:a15ca437f2309b769caa36a88240ce0266d67028f02da90f1c817c849b144273", - "registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.7.0" - ], - "sizeBytes":11371848 - }, - { - "names":[ - "docker.io/kindest/local-path-helper:v20230510-486859a6" - ], - "sizeBytes":2922287 - }, - { - "names":[ - "registry.k8s.io/pause:3.7" - ], - "sizeBytes":268400 - } - ] - } - } -} \ No newline at end of file diff --git a/libbeat/autodiscover/providers/kubernetes/test2.yml b/libbeat/autodiscover/providers/kubernetes/test2.yml deleted file mode 100644 index 599491bd4524..000000000000 --- a/libbeat/autodiscover/providers/kubernetes/test2.yml +++ /dev/null @@ -1,411 +0,0 @@ -{ - "type":"MODIFIED", - "object":{ - "kind":"Node", - "apiVersion":"v1", - "metadata":{ - "name":"kind-control-plane", - "uid":"1d44438c-a5c6-4c62-a7a4-9e83723da205", - "resourceVersion":"28202", - "creationTimestamp":"2023-12-06T14:21:10Z", - "labels":{ - "beta.kubernetes.io/arch":"arm64", - "beta.kubernetes.io/os":"linux", - "katsoulis":"pasole", - "kubernetes.io/arch":"arm64", - "kubernetes.io/hostname":"kind-control-plane", - "kubernetes.io/os":"linux", - "node-role.kubernetes.io/control-plane":"", - "node.kubernetes.io/exclude-from-external-load-balancers":"" - }, - "annotations":{ - "kubeadm.alpha.kubernetes.io/cri-socket":"unix:///run/containerd/containerd.sock", - "node.alpha.kubernetes.io/ttl":"0", - "volumes.kubernetes.io/controller-managed-attach-detach":"true" - }, - "managedFields":[ - { - "manager":"kubelet", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-06T14:21:10Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:annotations":{ - ".":{ - - }, - "f:volumes.kubernetes.io/controller-managed-attach-detach":{ - - } - }, - "f:labels":{ - ".":{ - - }, - "f:beta.kubernetes.io/arch":{ - - }, - "f:beta.kubernetes.io/os":{ - - }, - "f:kubernetes.io/arch":{ - - }, - "f:kubernetes.io/hostname":{ - - }, - "f:kubernetes.io/os":{ - - } - } - }, - "f:spec":{ - "f:providerID":{ - - } - } - } - }, - { - "manager":"kubeadm", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-06T14:21:12Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:annotations":{ - "f:kubeadm.alpha.kubernetes.io/cri-socket":{ - - } - }, - "f:labels":{ - "f:node-role.kubernetes.io/control-plane":{ - - }, - "f:node.kubernetes.io/exclude-from-external-load-balancers":{ - - } - } - } - } - }, - { - "manager":"kube-controller-manager", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-06T14:21:25Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:annotations":{ - "f:node.alpha.kubernetes.io/ttl":{ - - } - } - }, - "f:spec":{ - "f:podCIDR":{ - - }, - "f:podCIDRs":{ - ".":{ - - }, - "v:\"10.244.0.0/24\"":{ - - } - } - } - } - }, - { - "manager":"kubectl-label", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-07T11:28:02Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:metadata":{ - "f:labels":{ - "f:katsoulis":{ - - } - } - } - } - }, - { - "manager":"kubelet", - "operation":"Update", - "apiVersion":"v1", - "time":"2023-12-07T11:55:33Z", - "fieldsType":"FieldsV1", - "fieldsV1":{ - "f:status":{ - "f:addresses":{ - "k:{\"type\":\"InternalIP\"}":{ - "f:address":{ - - } - } - }, - "f:allocatable":{ - "f:memory":{ - - } - }, - "f:capacity":{ - "f:memory":{ - - } - }, - "f:conditions":{ - "k:{\"type\":\"DiskPressure\"}":{ - "f:lastHeartbeatTime":{ - - } - }, - "k:{\"type\":\"MemoryPressure\"}":{ - "f:lastHeartbeatTime":{ - - } - }, - "k:{\"type\":\"PIDPressure\"}":{ - "f:lastHeartbeatTime":{ - - } - }, - "k:{\"type\":\"Ready\"}":{ - "f:lastHeartbeatTime":{ - - }, - "f:lastTransitionTime":{ - - }, - "f:message":{ - - }, - "f:reason":{ - - }, - "f:status":{ - - } - } - }, - "f:images":{ - - }, - "f:nodeInfo":{ - "f:bootID":{ - - }, - "f:machineID":{ - - }, - "f:systemUUID":{ - - } - } - } - }, - "subresource":"status" - } - ] - }, - "spec":{ - "podCIDR":"10.244.0.0/24", - "podCIDRs":[ - "10.244.0.0/24" - ], - "providerID":"kind://docker/kind/kind-control-plane" - }, - "status":{ - "capacity":{ - "cpu":"5", - "ephemeral-storage":"107077304Ki", - "hugepages-1Gi":"0", - "hugepages-2Mi":"0", - "hugepages-32Mi":"0", - "hugepages-64Ki":"0", - "memory":"10958488Ki", - "pods":"110" - }, - "allocatable":{ - "cpu":"5", - "ephemeral-storage":"107077304Ki", - "hugepages-1Gi":"0", - "hugepages-2Mi":"0", - "hugepages-32Mi":"0", - "hugepages-64Ki":"0", - "memory":"10958488Ki", - "pods":"110" - }, - "conditions":[ - { - "type":"MemoryPressure", - "status":"False", - "lastHeartbeatTime":"2023-12-07T11:55:33Z", - "lastTransitionTime":"2023-12-06T14:21:08Z", - "reason":"KubeletHasSufficientMemory", - "message":"kubelet has sufficient memory available" - }, - { - "type":"DiskPressure", - "status":"False", - "lastHeartbeatTime":"2023-12-07T11:55:33Z", - "lastTransitionTime":"2023-12-06T14:21:08Z", - "reason":"KubeletHasNoDiskPressure", - "message":"kubelet has no disk pressure" - }, - { - "type":"PIDPressure", - "status":"False", - "lastHeartbeatTime":"2023-12-07T11:55:33Z", - "lastTransitionTime":"2023-12-06T14:21:08Z", - "reason":"KubeletHasSufficientPID", - "message":"kubelet has sufficient PID available" - }, - { - "type":"Ready", - "status":"True", - "lastHeartbeatTime":"2023-12-07T11:55:33Z", - "lastTransitionTime":"2023-12-06T14:21:28Z", - "reason":"KubeletReady", - "message":"kubelet is posting ready status" - } - ], - "addresses":[ - { - "type":"InternalIP", - "address":"172.19.0.2" - }, - { - "type":"Hostname", - "address":"kind-control-plane" - } - ], - "daemonEndpoints":{ - "kubeletEndpoint":{ - "Port":10250 - } - }, - "nodeInfo":{ - "machineID":"3489e0d51f694e4385ee5dedac53736c", - "systemUUID":"3489e0d51f694e4385ee5dedac53736c", - "bootID":"a28a8c69-2fc4-4bda-afe3-54644dd3a089", - "kernelVersion":"5.10.104-linuxkit", - "osImage":"Debian GNU/Linux 11 (bullseye)", - "containerRuntimeVersion":"containerd://1.7.1", - "kubeletVersion":"v1.27.3", - "kubeProxyVersion":"v1.27.3", - "operatingSystem":"linux", - "architecture":"arm64" - }, - "images":[ - { - "names":[ - "docker.elastic.co/beats/metricbeat@sha256:9972dd91739655e20e49a0edb4e26811dca75f0740318a29ba26fc086d3d78af", - "docker.elastic.co/beats/metricbeat:8.12.0-SNAPSHOT" - ], - "sizeBytes":149012603 - }, - { - "names":[ - "docker.elastic.co/beats/filebeat@sha256:9c2e3c7b8c1a03fcdb2021734898e8ad33aefb77d2542b373bc3ba5e154d8be2", - "docker.elastic.co/beats/filebeat:8.12.0-SNAPSHOT" - ], - "sizeBytes":141949252 - }, - { - "names":[ - "docker.elastic.co/beats/metricbeat@sha256:40f8a6b2b6dfad26295112b144c83cfbe793ae667d7cce687d12b8d898b92937", - "docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT" - ], - "sizeBytes":103670593 - }, - { - "names":[ - "docker.elastic.co/beats/filebeat@sha256:9b3cec4fcbb3a1a8b6d72efcd6ebaa68eb17a942c8be48b20a7c6a11f1f3a6dc", - "docker.elastic.co/beats/filebeat:8.0.0-SNAPSHOT" - ], - "sizeBytes":95587869 - }, - { - "names":[ - "registry.k8s.io/etcd:3.5.7-0" - ], - "sizeBytes":80665728 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:024134bb4e61a4975c5c15db76a7e3571eea74eff90c12a730781bf9c9daedd0", - "registry.k8s.io/kube-apiserver:v1.27.3" - ], - "sizeBytes":79827455 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:336af6020e6377f586577425f1314d95067a9b4a1c8707e90d5062823ec03f43", - "registry.k8s.io/kube-controller-manager:v1.27.3" - ], - "sizeBytes":71504013 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:67c927df0c79fb7415d0fc925570687fbec9a36cf13145a6430f9da059965dad", - "registry.k8s.io/kube-proxy:v1.27.3" - ], - "sizeBytes":68097533 - }, - { - "names":[ - "docker.io/library/import-2023-06-15@sha256:4b3d947dd06f4f5e4f417ec4da4bc83017f7cb5387d053ea2b6b906825011b7c", - "registry.k8s.io/kube-scheduler:v1.27.3" - ], - "sizeBytes":57610381 - }, - { - "names":[ - "docker.io/kindest/kindnetd:v20230511-dc714da8" - ], - "sizeBytes":25334607 - }, - { - "names":[ - "docker.io/kindest/local-path-provisioner:v20230511-dc714da8" - ], - "sizeBytes":17275248 - }, - { - "names":[ - "registry.k8s.io/coredns/coredns:v1.10.1" - ], - "sizeBytes":14557471 - }, - { - "names":[ - "registry.k8s.io/kube-state-metrics/kube-state-metrics@sha256:a15ca437f2309b769caa36a88240ce0266d67028f02da90f1c817c849b144273", - "registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.7.0" - ], - "sizeBytes":11371848 - }, - { - "names":[ - "docker.io/kindest/local-path-helper:v20230510-486859a6" - ], - "sizeBytes":2922287 - }, - { - "names":[ - "registry.k8s.io/pause:3.7" - ], - "sizeBytes":268400 - } - ] - } - } -} \ No newline at end of file From 4c4f45848812650d6f1f303b6d730d2f84533264 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 13 Dec 2023 11:49:43 +0200 Subject: [PATCH 08/20] updating changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8a81fe40002d..c6f3cf6d8494 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -155,6 +155,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor - Upgrade to Go 1.20.11. {pull}37123[37123] +- Make more selective the Pod autodiscovery upon node and namespace update events. {issue}3738[3738] {pull}37431[37431] *Auditbeat* From b2cf925f6bd3addb6ab1ec630af3f7aecef6ff6c Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Tue, 9 Jan 2024 14:31:17 +0200 Subject: [PATCH 09/20] updates for tests because we changed autodiscover watcher interface --- .../providers/kubernetes/pod_test.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 36ac9872021e..18e501997566 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -2028,10 +2028,8 @@ func TestNamespacePodUpdater(t *testing.T) { "beta.kubernetes.io/arch": "arm64", }, }} - var deltaslice []runtime.Object - deltaslice = append(deltaslice, namespace) - deltaslice = append(deltaslice, namespace1) - watcher := &mockUpdaterWatcher{delta: deltaslice} + + watcher := &mockUpdaterWatcher{oldobject: namespace} updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, watcher, &sync.Mutex{}) updater.OnUpdate(namespace1) @@ -2096,10 +2094,7 @@ func TestNodePodUpdater(t *testing.T) { }, }} - var deltaslice []runtime.Object - deltaslice = append(deltaslice, node) - deltaslice = append(deltaslice, node1) - watcher := &mockUpdaterWatcher{delta: deltaslice} + watcher := &mockUpdaterWatcher{oldobject: node} updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, watcher, &sync.Mutex{}) //This is when the update happens. @@ -2123,11 +2118,11 @@ type mockUpdaterStore struct { } type mockUpdaterWatcher struct { - delta []runtime.Object + oldobject runtime.Object } -func (s *mockUpdaterWatcher) Deltaobjects() []runtime.Object { - return s.delta +func (s *mockUpdaterWatcher) Oldobject() runtime.Object { + return s.oldobject } func (s *mockUpdaterStore) List() []interface{} { From 675a9fe75314c0b7c66aa8efba6d4c8026b14b22 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 10 Jan 2024 13:30:04 +0200 Subject: [PATCH 10/20] replacing oldobject with cachedobject --- libbeat/autodiscover/providers/kubernetes/pod_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 18e501997566..9b8e93bde9ea 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -2029,7 +2029,7 @@ func TestNamespacePodUpdater(t *testing.T) { }, }} - watcher := &mockUpdaterWatcher{oldobject: namespace} + watcher := &mockUpdaterWatcher{cachedObject: namespace} updater := kubernetes.NewNamespacePodUpdater(handler.OnUpdate, store, watcher, &sync.Mutex{}) updater.OnUpdate(namespace1) @@ -2094,7 +2094,7 @@ func TestNodePodUpdater(t *testing.T) { }, }} - watcher := &mockUpdaterWatcher{oldobject: node} + watcher := &mockUpdaterWatcher{cachedObject: node} updater := kubernetes.NewNodePodUpdater(handler.OnUpdate, store, watcher, &sync.Mutex{}) //This is when the update happens. @@ -2118,11 +2118,11 @@ type mockUpdaterStore struct { } type mockUpdaterWatcher struct { - oldobject runtime.Object + cachedObject runtime.Object } -func (s *mockUpdaterWatcher) Oldobject() runtime.Object { - return s.oldobject +func (s *mockUpdaterWatcher) CachedObject() runtime.Object { + return s.cachedObject } func (s *mockUpdaterStore) List() []interface{} { From da0b201c4220b2974cd52efb4325fe03c81f6945 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 11 Jan 2024 15:09:39 +0200 Subject: [PATCH 11/20] keeping only watcher struct --- .../providers/kubernetes/pod_test.go | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 9b8e93bde9ea..5e27df2aeaf9 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -30,6 +30,9 @@ import ( "k8s.io/apimachinery/pkg/types" k8sfake "k8s.io/client-go/kubernetes/fake" + interfaces "k8s.io/client-go/kubernetes" + caches "k8s.io/client-go/tools/cache" + "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-autodiscover/kubernetes" @@ -2117,6 +2120,10 @@ type mockUpdaterStore struct { objects []interface{} } +var store caches.Store +var client interfaces.Interface +var err error + type mockUpdaterWatcher struct { cachedObject runtime.Object } @@ -2125,6 +2132,24 @@ func (s *mockUpdaterWatcher) CachedObject() runtime.Object { return s.cachedObject } +func (s *mockUpdaterWatcher) Client() interfaces.Interface { + return client +} + +func (s *mockUpdaterWatcher) Start() error { + return err +} + +func (s *mockUpdaterWatcher) Stop() { +} + +func (s *mockUpdaterWatcher) Store() caches.Store { + return store +} + +func (s *mockUpdaterWatcher) AddEventHandler(kubernetes.ResourceEventHandler) { +} + func (s *mockUpdaterStore) List() []interface{} { return s.objects } From 99c949fecab52b4a6523dc0281b07c8bb7286c4c Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 12 Jan 2024 13:45:08 +0200 Subject: [PATCH 12/20] udpating changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1606718f5e9e..9249b4ad1744 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,11 +138,11 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor - Upgrade to Go 1.20.11. {pull}37123[37123] -- Make more selective the Pod autodiscovery upon node and namespace update events. {issue}3738[3738] {pull}37431[37431] - Upgrade to Go 1.21.5. {pull}37550[37550] - Upgrade to Go 1.21.6. {pull}37615[37615] - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259] - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544] +- Make more selective the Pod autodiscovery upon node and namespace update events. {issue}37338[37338] {pull}37431[37431] *Auditbeat* From f3900af4c7b79c97a7e81cc75368bed0664dfbdf Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Fri, 12 Jan 2024 13:49:52 +0200 Subject: [PATCH 13/20] Update CHANGELOG.next.asciidoc Co-authored-by: Michal Pristas --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9249b4ad1744..d4bd4643804e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,7 +138,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor - Upgrade to Go 1.20.11. {pull}37123[37123] -- Upgrade to Go 1.21.5. {pull}37550[37550] - Upgrade to Go 1.21.6. {pull}37615[37615] - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259] - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544] From e65f21d9b15f0a9024eb67fa7a2e726f4ef4204b Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Fri, 12 Jan 2024 13:49:58 +0200 Subject: [PATCH 14/20] Update CHANGELOG.next.asciidoc Co-authored-by: Michal Pristas --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d4bd4643804e..e56bba3d39e1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -137,7 +137,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] - Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor -- Upgrade to Go 1.20.11. {pull}37123[37123] - Upgrade to Go 1.21.6. {pull}37615[37615] - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259] - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544] From 109f1529eea3f32f775b991aa0eb06f4bafadd00 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 15 Jan 2024 11:39:16 +0200 Subject: [PATCH 15/20] updating elastic-agent-autodiscovery library to v0.6.7 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a9e019c82923..3f326b006a76 100644 --- a/go.mod +++ b/go.mod @@ -200,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.6.6 + github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.3 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.9.1 diff --git a/go.sum b/go.sum index 716e1a1b4117..3c6faaf866ad 100644 --- a/go.sum +++ b/go.sum @@ -658,8 +658,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= -github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= +github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= +github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM= github.com/elastic/elastic-agent-libs v0.7.3 h1:tc6JDXYR+2XFMHJVv+7+M0OwAbZPxm3caLJEd943dlE= From f7aec1231a4de1f712ed7406b0e4d668dcba3212 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 15 Jan 2024 11:46:23 +0200 Subject: [PATCH 16/20] updating NOTICE.txt --- NOTICE.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index d258999b356b..207dc8035f33 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12257,11 +12257,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.6 +Version: v0.6.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.7/LICENSE: Apache License Version 2.0, January 2004 From 9ef1313e08d0407266b8564f0c4d1e3b299d4760 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 15 Jan 2024 12:10:09 +0200 Subject: [PATCH 17/20] updating kubernetes_test.go interface functions --- metricbeat/module/kubernetes/util/kubernetes_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index 2d0d3e461139..e015e43c36ac 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -28,6 +28,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "github.com/elastic/elastic-agent-autodiscover/kubernetes" @@ -180,3 +181,7 @@ func (m *mockWatcher) Store() cache.Store { func (m *mockWatcher) Client() k8s.Interface { return nil } + +func (s *mockWatcher) CachedObject() runtime.Object { + return nil +} From 2383df8426486b61b20e558f2c901069abe0d342 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 15 Jan 2024 12:12:28 +0200 Subject: [PATCH 18/20] updating pod_test.go by removing unused functions --- .../providers/kubernetes/pod_test.go | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 5e27df2aeaf9..1718dbe07529 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -2154,26 +2154,6 @@ func (s *mockUpdaterStore) List() []interface{} { return s.objects } -type mockNamespaceStore struct { - objects interface{} - exist bool - err error -} - -func (s *mockNamespaceStore) GetByKey(string) (interface{}, bool, error) { - return s.objects, s.exist, s.err -} - -type mockNodeStore struct { - objects interface{} - exist bool - err error -} - -func (s *mockNodeStore) GetByKey(string) (interface{}, bool, error) { - return s.objects, s.exist, s.err -} - func NewMockPodEventerManager(pod *pod) EventManager { em := &eventerManager{} em.eventer = pod From 51295fcc07dd9053f6b1e63158829bddefea81ce Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 15 Jan 2024 12:43:11 +0200 Subject: [PATCH 19/20] updating receiver name in tests --- metricbeat/module/kubernetes/util/kubernetes_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index e015e43c36ac..92d60b28b2d2 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -182,6 +182,6 @@ func (m *mockWatcher) Client() k8s.Interface { return nil } -func (s *mockWatcher) CachedObject() runtime.Object { +func (m *mockWatcher) CachedObject() runtime.Object { return nil } From c22d63aeed1052c90ae29deaec5f0ab1c8e92ac1 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 15 Jan 2024 13:00:36 +0200 Subject: [PATCH 20/20] fixing lint events --- libbeat/autodiscover/providers/kubernetes/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index b180adbc6d33..d849039a66e4 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -407,7 +407,7 @@ func (p *pod) containerPodEvents(flag string, pod *kubernetes.Pod, c *kubernetes ports = []kubernetes.ContainerPort{{ContainerPort: 0}} } - var events []bus.Event + events := []bus.Event{} portsMap := mapstr.M{} ShouldPut(meta, "container", cmeta, p.logger)