From cf365a8d67e969f85e03996960f61e3e41816ccb Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 14 Sep 2023 09:54:37 +0300 Subject: [PATCH 01/14] Adding updates for code to introduces hints per container --- .../composable/providers/kubernetes/hints.go | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 751bab078be..ac7830a4daf 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -256,17 +256,47 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre composableMapping: mapstr.M{}, processors: []mapstr.M{}, } + hints := mapstr.M{} if ann, ok := k8sMapping["annotations"]; ok { annotations, _ := ann.(mapstr.M) - hints := utils.GenerateHints(annotations, "", prefix) - if len(hints) > 0 { - logger.Debugf("Extracted hints are :%v", hints) - hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) - logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) - - hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) - logger.Debugf("Generated Processors are :%v", hintData.processors) + + if containerEntries, err := annotations.GetValue(prefix + ".hints"); err == nil { + if entries, ok := containerEntries.(mapstr.M); ok { + if len(entries) > 0 { + for key := range entries { + parts := strings.Split(key, "/") + if con, ok := k8sMapping["container"]; ok { + containers, _ := con.(mapstr.M) + if cname, err := containers.GetValue("name"); err == nil { + if parts[0] == cname { + // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container + hints = utils.GenerateHints(annotations, parts[0], prefix) + } else { + // If there are top level hints like co.elastic.hints/ then just add the values after the / + hints = utils.GenerateHints(annotations, "", prefix) + } + if len(hints) > 0 { + logger.Debugf("Extracted hints are :%v", hints) + hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) + logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) + + hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) + //Processors for specific container + //We need to make an extra check if we have processors added only to the specific conatiners + containerProcessors := utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") + if len(containerProcessors) > 0 { + for _, value := range containerProcessors { + hintData.processors = append(hintData.processors, value) + } + } + logger.Debugf("Generated Processors are :%v", hintData.processors) + } + } + } + } + } + } } } From 4d324413e82503dd37c1ea8fdf290ba9e7388f11 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 14 Sep 2023 13:02:34 +0300 Subject: [PATCH 02/14] Fixing processors for specific container --- internal/pkg/composable/providers/kubernetes/hints.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index ac7830a4daf..7ea81c33bb7 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -257,6 +257,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre processors: []mapstr.M{}, } hints := mapstr.M{} + containerProcessors := []mapstr.M{} if ann, ok := k8sMapping["annotations"]; ok { annotations, _ := ann.(mapstr.M) @@ -272,6 +273,10 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre if parts[0] == cname { // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container hints = utils.GenerateHints(annotations, parts[0], prefix) + //Processors for specific container + //We need to make an extra check if we have processors added only to the specific conatiners + containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") + } else { // If there are top level hints like co.elastic.hints/ then just add the values after the / hints = utils.GenerateHints(annotations, "", prefix) @@ -282,9 +287,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) - //Processors for specific container - //We need to make an extra check if we have processors added only to the specific conatiners - containerProcessors := utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") + //Only if there are processors defined in a specific container we append the to the processors of the pod if len(containerProcessors) > 0 { for _, value := range containerProcessors { hintData.processors = append(hintData.processors, value) From ad54d8a693068bea4715d65f23cefb7e7d8a62d2 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 14 Sep 2023 13:43:48 +0300 Subject: [PATCH 03/14] Removing unwanted if --- .../composable/providers/kubernetes/hints.go | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 7ea81c33bb7..901f23359bd 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -263,43 +263,43 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre annotations, _ := ann.(mapstr.M) if containerEntries, err := annotations.GetValue(prefix + ".hints"); err == nil { - if entries, ok := containerEntries.(mapstr.M); ok { - if len(entries) > 0 { - for key := range entries { - parts := strings.Split(key, "/") - if con, ok := k8sMapping["container"]; ok { - containers, _ := con.(mapstr.M) - if cname, err := containers.GetValue("name"); err == nil { - if parts[0] == cname { - // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container - hints = utils.GenerateHints(annotations, parts[0], prefix) - //Processors for specific container - //We need to make an extra check if we have processors added only to the specific conatiners - containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") - - } else { - // If there are top level hints like co.elastic.hints/ then just add the values after the / - hints = utils.GenerateHints(annotations, "", prefix) - } - if len(hints) > 0 { - logger.Debugf("Extracted hints are :%v", hints) - hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) - logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) - - hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) - //Only if there are processors defined in a specific container we append the to the processors of the pod - if len(containerProcessors) > 0 { - for _, value := range containerProcessors { - hintData.processors = append(hintData.processors, value) - } + entries, _ := containerEntries.(mapstr.M) + if len(entries) > 0 { + for key := range entries { + parts := strings.Split(key, "/") + if con, ok := k8sMapping["container"]; ok { + containers, _ := con.(mapstr.M) + if cname, err := containers.GetValue("name"); err == nil { + if parts[0] == cname { + // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container + hints = utils.GenerateHints(annotations, parts[0], prefix) + //Processors for specific container + //We need to make an extra check if we have processors added only to the specific conatiners + containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") + + } else { + // If there are top level hints like co.elastic.hints/ then just add the values after the / + hints = utils.GenerateHints(annotations, "", prefix) + } + if len(hints) > 0 { + logger.Debugf("Extracted hints are :%v", hints) + hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) + logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) + + hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) + //Only if there are processors defined in a specific container we append the to the processors of the pod + if len(containerProcessors) > 0 { + for _, value := range containerProcessors { + hintData.processors = append(hintData.processors, value) } - logger.Debugf("Generated Processors are :%v", hintData.processors) } + logger.Debugf("Generated Processors are :%v", hintData.processors) } } } } } + } } From 6b9e836aad9c26fa387d7eb252fd31f3c37fb62c Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 14 Sep 2023 14:54:39 +0300 Subject: [PATCH 04/14] Adding changelog fragment file --- .../1694692246-hintspercontainer.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 changelog/fragments/1694692246-hintspercontainer.yaml diff --git a/changelog/fragments/1694692246-hintspercontainer.yaml b/changelog/fragments/1694692246-hintspercontainer.yaml new file mode 100644 index 00000000000..7defa8adea3 --- /dev/null +++ b/changelog/fragments/1694692246-hintspercontainer.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: enhancement + +# Change summary; a 80ish characters long description of the change. +summary: Hints Autodiscovery for Elastic Agent - Define configuration through annotations for specific containers inside a pod + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 From 4396a0bf83faccea04ea82f80c604bbe1cd21c4e Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 14 Sep 2023 15:08:51 +0300 Subject: [PATCH 05/14] Fixing lint errors --- internal/pkg/composable/providers/kubernetes/hints.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 901f23359bd..3d88c963c8e 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -256,8 +256,8 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre composableMapping: mapstr.M{}, processors: []mapstr.M{}, } - hints := mapstr.M{} - containerProcessors := []mapstr.M{} + var hints mapstr.M + var containerProcessors []mapstr.M if ann, ok := k8sMapping["annotations"]; ok { annotations, _ := ann.(mapstr.M) @@ -274,7 +274,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container hints = utils.GenerateHints(annotations, parts[0], prefix) //Processors for specific container - //We need to make an extra check if we have processors added only to the specific conatiners + //We need to make an extra check if we have processors added only to the specific containers containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") } else { @@ -289,9 +289,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) //Only if there are processors defined in a specific container we append the to the processors of the pod if len(containerProcessors) > 0 { - for _, value := range containerProcessors { - hintData.processors = append(hintData.processors, value) - } + hintData.processors = append(hintData.processors, containerProcessors...) } logger.Debugf("Generated Processors are :%v", hintData.processors) } From 5b4fffc941604469938422a16c17ee2aa082b730 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 14 Sep 2023 15:11:41 +0300 Subject: [PATCH 06/14] Fixing comments --- internal/pkg/composable/providers/kubernetes/hints.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 3d88c963c8e..6e95a40db98 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -273,8 +273,8 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre if parts[0] == cname { // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container hints = utils.GenerateHints(annotations, parts[0], prefix) - //Processors for specific container - //We need to make an extra check if we have processors added only to the specific containers + // Processors for specific container + // We need to make an extra check if we have processors added only to the specific containers containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") } else { @@ -287,7 +287,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) - //Only if there are processors defined in a specific container we append the to the processors of the pod + // Only if there are processors defined in a specific container we append them to the processors of the pod if len(containerProcessors) > 0 { hintData.processors = append(hintData.processors, containerProcessors...) } From 2e32781a1fb121d4f1980ffbda17e052a4658e3a Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Wed, 20 Sep 2023 11:08:45 +0300 Subject: [PATCH 07/14] Update internal/pkg/composable/providers/kubernetes/hints.go Co-authored-by: Chris Mark --- internal/pkg/composable/providers/kubernetes/hints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 6e95a40db98..2ef6c261de6 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -271,7 +271,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre containers, _ := con.(mapstr.M) if cname, err := containers.GetValue("name"); err == nil { if parts[0] == cname { - // If there are hints like co.elastic.hints./ then add add the values after the / to the corresponding container + // If there are hints like co.elastic.hints./ then add the values after the / to the corresponding container hints = utils.GenerateHints(annotations, parts[0], prefix) // Processors for specific container // We need to make an extra check if we have processors added only to the specific containers From 919fad9848463869542d1fe14fcf3190c6e350d8 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 20 Sep 2023 11:21:51 +0300 Subject: [PATCH 08/14] Fixing checks for return variables --- .../composable/providers/kubernetes/hints.go | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 2ef6c261de6..8f377b7d29e 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -263,38 +263,45 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre annotations, _ := ann.(mapstr.M) if containerEntries, err := annotations.GetValue(prefix + ".hints"); err == nil { - entries, _ := containerEntries.(mapstr.M) - if len(entries) > 0 { + entries, ok := containerEntries.(mapstr.M) + if ok && len(entries) > 0 { for key := range entries { parts := strings.Split(key, "/") - if con, ok := k8sMapping["container"]; ok { - containers, _ := con.(mapstr.M) - if cname, err := containers.GetValue("name"); err == nil { - if parts[0] == cname { - // If there are hints like co.elastic.hints./ then add the values after the / to the corresponding container - hints = utils.GenerateHints(annotations, parts[0], prefix) - // Processors for specific container - // We need to make an extra check if we have processors added only to the specific containers - containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") - - } else { - // If there are top level hints like co.elastic.hints/ then just add the values after the / - hints = utils.GenerateHints(annotations, "", prefix) - } - if len(hints) > 0 { - logger.Debugf("Extracted hints are :%v", hints) - hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) - logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) - - hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) - // Only if there are processors defined in a specific container we append them to the processors of the pod - if len(containerProcessors) > 0 { - hintData.processors = append(hintData.processors, containerProcessors...) + + if len(parts) > 0 { + if con, ok := k8sMapping["container"]; ok { + containers, ok := con.(mapstr.M) + if ok { + if cname, err := containers.GetValue("name"); err == nil { + if parts[0] == cname { + // If there are hints like co.elastic.hints./ then add the values after the / to the corresponding container + hints = utils.GenerateHints(annotations, parts[0], prefix) + // Processors for specific container + // We need to make an extra check if we have processors added only to the specific containers + containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") + + } else { + // If there are top level hints like co.elastic.hints/ then just add the values after the / + hints = utils.GenerateHints(annotations, "", prefix) + } + if len(hints) > 0 { + logger.Debugf("Extracted hints are :%v", hints) + hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) + logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) + + hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) + // Only if there are processors defined in a specific container we append them to the processors of the pod + if len(containerProcessors) > 0 { + hintData.processors = append(hintData.processors, containerProcessors...) + } + logger.Debugf("Generated Processors are :%v", hintData.processors) + } } - logger.Debugf("Generated Processors are :%v", hintData.processors) } + } } + } } From 419d3a952801fdb2e8fbeb56f3f5a630db3f3b9f Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 20 Sep 2023 12:06:06 +0300 Subject: [PATCH 09/14] Splitting functions for hints and processors --- .../composable/providers/kubernetes/hints.go | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 8f377b7d29e..86a3302398c 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -275,26 +275,24 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre if cname, err := containers.GetValue("name"); err == nil { if parts[0] == cname { // If there are hints like co.elastic.hints./ then add the values after the / to the corresponding container - hints = utils.GenerateHints(annotations, parts[0], prefix) - // Processors for specific container - // We need to make an extra check if we have processors added only to the specific containers - containerProcessors = utils.GetConfigs(annotations, prefix, "hints."+parts[0]+"/processors") - + // Eg Annotation "co.elastic.hints.nginx/stream: stderr" will create a hints entry for container nginx + //Eg co.elastic.hints.nginx/processors.add_fields.fields.name: "myproject", will add also a processor entry only for container nginx + hints, containerProcessors = GenerateHintsForContainer(annotations, parts[0], prefix) } else { // If there are top level hints like co.elastic.hints/ then just add the values after the / + // Eg. Annotation "co.elastic.hints/stream: stderr" will will create a hints entries for all containers in the pod hints = utils.GenerateHints(annotations, "", prefix) } + logger.Debugf("Extracted hints are :%v", hints) + if len(hints) > 0 { - logger.Debugf("Extracted hints are :%v", hints) - hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) - logger.Debugf("Generated hints mappings are :%v", hintData.composableMapping) + hintData = GenerateHintsResult(hints, k8sMapping, annotations, logger, prefix, cID) - hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) // Only if there are processors defined in a specific container we append them to the processors of the pod if len(containerProcessors) > 0 { hintData.processors = append(hintData.processors, containerProcessors...) } - logger.Debugf("Generated Processors are :%v", hintData.processors) + logger.Debugf("Generated Processors mapping :%v", hintData.processors) } } } @@ -310,3 +308,29 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre } return hintData } + +// Generates hints and processors list for specific containers +func GenerateHintsForContainer(annotations mapstr.M, parts string, prefix string) (mapstr.M, []mapstr.M) { + hints := utils.GenerateHints(annotations, parts, prefix) + // Processors for specific container + // We need to make an extra check if we have processors added only to the specific containers + containerProcessors := utils.GetConfigs(annotations, prefix, "hints."+parts+"/processors") + + return hints, containerProcessors +} + +// Generates the final hintData (hints and processors) struct that will be emmitted in pods. +func GenerateHintsResult(hints mapstr.M, k8sMapping map[string]interface{}, annotations mapstr.M, logger *logp.Logger, prefix string, cID string) hintsData { + hintData := hintsData{ + composableMapping: mapstr.M{}, + processors: []mapstr.M{}, + } + + hintData.composableMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID) + logger.Debugf("Generated hints mappings :%v", hintData.composableMapping) + + // Eg co.elastic.hints/processors.decode_json_fields.fields: "message" will add a processor in all containers of pod + hintData.processors = utils.GetConfigs(annotations, prefix, processorhints) + + return hintData +} From c38405bd1a7a502fdc0f1f2d721089477ae5ed1a Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 20 Sep 2023 17:15:14 +0300 Subject: [PATCH 10/14] Adding test for specific container hints --- .../composable/providers/kubernetes/hints.go | 36 +++-- .../providers/kubernetes/hints_test.go | 132 +++++++++++++++++- 2 files changed, 141 insertions(+), 27 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 86a3302398c..136e3f28970 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -268,7 +268,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre for key := range entries { parts := strings.Split(key, "/") - if len(parts) > 0 { + if len(parts) > 1 { if con, ok := k8sMapping["container"]; ok { containers, ok := con.(mapstr.M) if ok { @@ -276,36 +276,32 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre if parts[0] == cname { // If there are hints like co.elastic.hints./ then add the values after the / to the corresponding container // Eg Annotation "co.elastic.hints.nginx/stream: stderr" will create a hints entry for container nginx - //Eg co.elastic.hints.nginx/processors.add_fields.fields.name: "myproject", will add also a processor entry only for container nginx hints, containerProcessors = GenerateHintsForContainer(annotations, parts[0], prefix) - } else { - // If there are top level hints like co.elastic.hints/ then just add the values after the / - // Eg. Annotation "co.elastic.hints/stream: stderr" will will create a hints entries for all containers in the pod - hints = utils.GenerateHints(annotations, "", prefix) - } - logger.Debugf("Extracted hints are :%v", hints) - - if len(hints) > 0 { - hintData = GenerateHintsResult(hints, k8sMapping, annotations, logger, prefix, cID) - - // Only if there are processors defined in a specific container we append them to the processors of the pod - if len(containerProcessors) > 0 { - hintData.processors = append(hintData.processors, containerProcessors...) - } - logger.Debugf("Generated Processors mapping :%v", hintData.processors) } } } - } } - } } - + } else { + // If there are top level hints like co.elastic.hints/ then just add the values after the / + // Eg. Annotation "co.elastic.hints/stream: stderr" will will create a hints entries for all containers in the pod + hints = utils.GenerateHints(annotations, "", prefix) } + logger.Debugf("Extracted hints are :%v", hints) + + if len(hints) > 0 { + hintData = GenerateHintsResult(hints, k8sMapping, annotations, logger, prefix, cID) + // Only if there are processors defined in a specific container we append them to the processors of the top level + if len(containerProcessors) > 0 { + hintData.processors = append(hintData.processors, containerProcessors...) + } + logger.Debugf("Generated Processors mapping :%v", hintData.processors) + } } + return hintData } diff --git a/internal/pkg/composable/providers/kubernetes/hints_test.go b/internal/pkg/composable/providers/kubernetes/hints_test.go index 912b01b123a..1764d364f4b 100644 --- a/internal/pkg/composable/providers/kubernetes/hints_test.go +++ b/internal/pkg/composable/providers/kubernetes/hints_test.go @@ -433,11 +433,10 @@ func TestGenerateHintsMappingWithProcessors(t *testing.T) { }, }, }, - }, - }, + }}, } - expected_hints := mapstr.M{ + expectedhints := mapstr.M{ "container_id": "asdfghjkl", "apache": mapstr.M{ "container_logs": mapstr.M{ @@ -447,7 +446,7 @@ func TestGenerateHintsMappingWithProcessors(t *testing.T) { }, } - expected_procesors := []mapstr.M{ + expectedprocesors := []mapstr.M{ 0: { "rename": mapstr.M{ "fail_on_error": "false", @@ -467,9 +466,128 @@ func TestGenerateHintsMappingWithProcessors(t *testing.T) { hintData := GetHintsMapping(mapping, logger, "co.elastic", "asdfghjkl") - assert.Equal(t, expected_hints, hintData.composableMapping) + assert.Equal(t, expectedhints, hintData.composableMapping) //assert.Equal(t, expected_procesors, hintData.processors). We replace this assertion with assert.Contains in order to avoid flakiness in tests because map keys are not sorted - assert.Contains(t, expected_procesors, hintData.processors[0]) - assert.Contains(t, expected_procesors, hintData.processors[1]) + if len(hintData.processors) > 0 { + assert.Contains(t, expectedprocesors, hintData.processors[0]) + assert.Contains(t, expectedprocesors, hintData.processors[1]) + } +} + +// This test evaluates the hints Generation when you define specific container nginx +// Following will need to include all annotations after top level "co.elastic.hints/" plus those that defined for nginx with prefix "co.elastic.hints.nginx" +// mappings.container.name = nginx defines the container we want to emmit the new configuration. Annotations for other containers like co.elastic.hints.webapp should be excluded +func TestGenerateHintsMappingWithProcessorsForContainer(t *testing.T) { + logger := getLogger() + pod := &kubernetes.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + UID: types.UID(uid), + Namespace: "testns", + Labels: map[string]string{ + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", + }, + Annotations: map[string]string{ + "app": "production", + "co.elastic.hints/package": "apache", + "co.elastic.hints/processors.decode_json_fields.fields": "message", + "co.elastic.hints/processors.decode_json_fields.add_error_key": "true", + "co.elastic.hints/processors.decode_json_fields.overwrite_keys": "true", + "co.elastic.hints/processors.decode_json_fields.target": "team", + "co.elastic.hints.nginx/stream": "stderr", + "co.elastic.hints.nginx/processors.add_fields.fields.name": "myproject", + "co.elastic.hints.webapp/processors.add_fields.fields.name": "myproject2", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: kubernetes.PodSpec{ + NodeName: "testnode", + }, + Status: kubernetes.PodStatus{PodIP: "127.0.0.5"}, + } + + mapping := map[string]interface{}{ + "namespace": pod.GetNamespace(), + "pod": mapstr.M{ + "uid": string(pod.GetUID()), + "name": pod.GetName(), + "ip": pod.Status.PodIP, + }, + "namespace_annotations": mapstr.M{ + "nsa": "nsb", + }, + "labels": mapstr.M{ + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", + }, + "container": mapstr.M{ + "name": "nginx", + "id": "8863418215f5d6b1919db9b3b710615878f88b0773e2b098e714c8d696c3261f", + }, + "annotations": mapstr.M{ + "app": "production", + "co": mapstr.M{ + "elastic": mapstr.M{ + "hints/package": "apache", + "hints/processors": mapstr.M{ + "decode_json_fields": mapstr.M{ + "fields": "message", + "add_error_key": "true", + "overwrite_keys": "true", + "target": "team", + }}, + "hints": mapstr.M{ + "nginx/processors": mapstr.M{ + "add_fields": mapstr.M{ + "name": "myproject", + }, + }, + "nginx/stream": "stderr", + }, + }, + }, + }, + } + expectedhints := mapstr.M{ + "container_id": "asdfghjkl", + "apache": mapstr.M{ + "container_logs": mapstr.M{ + "enabled": true, + }, + "stream": "stderr", + "enabled": true, + }, + } + + expectedprocesors := []mapstr.M{ + 0: { + "decode_json_fields": mapstr.M{ + "fields": "message", + "add_error_key": "true", + "overwrite_keys": "true", + "target": "team", + }, + }, + 1: { + "add_fields": mapstr.M{ + "name": "myproject", + }, + }, + } + + hintData := GetHintsMapping(mapping, logger, "co.elastic", "asdfghjkl") + + assert.Equal(t, expectedhints, hintData.composableMapping) + //assert.Equal(t, expected_procesors, hintData.processors). We replace this assertion with assert.Contains in order to avoid flakiness in tests because map keys are not sorted + if len(hintData.processors) > 0 { + assert.Contains(t, expectedprocesors, hintData.processors[0]) + assert.Contains(t, expectedprocesors, hintData.processors[1]) + } } From e954e4f1043b86f1630d385d8bc8fb2d38f3ce90 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Tue, 3 Oct 2023 11:20:58 +0300 Subject: [PATCH 11/14] Updating hints test by commenting pod part as it is not needed for the test --- .../providers/kubernetes/hints_test.go | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints_test.go b/internal/pkg/composable/providers/kubernetes/hints_test.go index 1764d364f4b..17e36b0d7c7 100644 --- a/internal/pkg/composable/providers/kubernetes/hints_test.go +++ b/internal/pkg/composable/providers/kubernetes/hints_test.go @@ -479,44 +479,44 @@ func TestGenerateHintsMappingWithProcessors(t *testing.T) { // mappings.container.name = nginx defines the container we want to emmit the new configuration. Annotations for other containers like co.elastic.hints.webapp should be excluded func TestGenerateHintsMappingWithProcessorsForContainer(t *testing.T) { logger := getLogger() - pod := &kubernetes.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testpod", - UID: types.UID(uid), - Namespace: "testns", - Labels: map[string]string{ - "foo": "bar", - "with-dash": "dash-value", - "with/slash": "some/path", - }, - Annotations: map[string]string{ - "app": "production", - "co.elastic.hints/package": "apache", - "co.elastic.hints/processors.decode_json_fields.fields": "message", - "co.elastic.hints/processors.decode_json_fields.add_error_key": "true", - "co.elastic.hints/processors.decode_json_fields.overwrite_keys": "true", - "co.elastic.hints/processors.decode_json_fields.target": "team", - "co.elastic.hints.nginx/stream": "stderr", - "co.elastic.hints.nginx/processors.add_fields.fields.name": "myproject", - "co.elastic.hints.webapp/processors.add_fields.fields.name": "myproject2", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: kubernetes.PodSpec{ - NodeName: "testnode", - }, - Status: kubernetes.PodStatus{PodIP: "127.0.0.5"}, - } + // pod := &kubernetes.Pod{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: "testpod", + // UID: types.UID(uid), + // Namespace: "testns", + // Labels: map[string]string{ + // "foo": "bar", + // "with-dash": "dash-value", + // "with/slash": "some/path", + // }, + // Annotations: map[string]string{ + // "app": "production", + // "co.elastic.hints/package": "apache", + // "co.elastic.hints/processors.decode_json_fields.fields": "message", + // "co.elastic.hints/processors.decode_json_fields.add_error_key": "true", + // "co.elastic.hints/processors.decode_json_fields.overwrite_keys": "true", + // "co.elastic.hints/processors.decode_json_fields.target": "team", + // "co.elastic.hints.nginx/stream": "stderr", + // "co.elastic.hints.nginx/processors.add_fields.fields.name": "myproject", + // "co.elastic.hints.webapp/processors.add_fields.fields.name": "myproject2", + // }, + // }, + // TypeMeta: metav1.TypeMeta{ + // Kind: "Pod", + // APIVersion: "v1", + // }, + // Spec: kubernetes.PodSpec{ + // NodeName: "testnode", + // }, + // Status: kubernetes.PodStatus{PodIP: "127.0.0.5"}, + // } mapping := map[string]interface{}{ - "namespace": pod.GetNamespace(), + "namespace": "testns", "pod": mapstr.M{ - "uid": string(pod.GetUID()), - "name": pod.GetName(), - "ip": pod.Status.PodIP, + "uid": string(types.UID(uid)), + "name": "testpod", + "ip": "127.0.0.5", }, "namespace_annotations": mapstr.M{ "nsa": "nsb", From 866288b08aed6906e93b687aec7e26fc72149d71 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 4 Oct 2023 18:19:02 +0300 Subject: [PATCH 12/14] Fixing mispell --- internal/pkg/composable/providers/kubernetes/hints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 136e3f28970..2a32dc1532e 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -315,7 +315,7 @@ func GenerateHintsForContainer(annotations mapstr.M, parts string, prefix string return hints, containerProcessors } -// Generates the final hintData (hints and processors) struct that will be emmitted in pods. +// Generates the final hintData (hints and processors) struct that will be emitted in pods. func GenerateHintsResult(hints mapstr.M, k8sMapping map[string]interface{}, annotations mapstr.M, logger *logp.Logger, prefix string, cID string) hintsData { hintData := hintsData{ composableMapping: mapstr.M{}, From d594c6064d219a8e86cb197270f31c0ec420a156 Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Fri, 6 Oct 2023 17:38:01 +0300 Subject: [PATCH 13/14] Update internal/pkg/composable/providers/kubernetes/hints.go Co-authored-by: Aman Verma <38116245+devamanv@users.noreply.github.com> --- internal/pkg/composable/providers/kubernetes/hints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 2a32dc1532e..900fcd51e46 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -316,7 +316,7 @@ func GenerateHintsForContainer(annotations mapstr.M, parts string, prefix string } // Generates the final hintData (hints and processors) struct that will be emitted in pods. -func GenerateHintsResult(hints mapstr.M, k8sMapping map[string]interface{}, annotations mapstr.M, logger *logp.Logger, prefix string, cID string) hintsData { +func GenerateHintsResult(hints mapstr.M, k8sMapping map[string]interface{}, annotations mapstr.M, logger *logp.Logger, prefix, cID string) hintsData { hintData := hintsData{ composableMapping: mapstr.M{}, processors: []mapstr.M{}, From 463f5b72cb0e23faa282ef6c485fc3b94e949c43 Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Fri, 6 Oct 2023 17:38:17 +0300 Subject: [PATCH 14/14] Update internal/pkg/composable/providers/kubernetes/hints.go Co-authored-by: Aman Verma <38116245+devamanv@users.noreply.github.com> --- internal/pkg/composable/providers/kubernetes/hints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 900fcd51e46..0a439afd5da 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -306,7 +306,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre } // Generates hints and processors list for specific containers -func GenerateHintsForContainer(annotations mapstr.M, parts string, prefix string) (mapstr.M, []mapstr.M) { +func GenerateHintsForContainer(annotations mapstr.M, parts, prefix string) (mapstr.M, []mapstr.M) { hints := utils.GenerateHints(annotations, parts, prefix) // Processors for specific container // We need to make an extra check if we have processors added only to the specific containers