Merge remote-tracking branch 'origin/main' into helm/fix_fleet_k8s_mo…

…unts

# Conflicts:
#	deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml
#	deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml
#	deploy/helm/elastic-agent/templates/integrations/_kubernetes/_preset_pernode.tpl

changelog/fragments/1732656422-add-actionable-error-message-for-enroll-command.yaml

@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Elastic agent returns an actionable error message when a the use trying to execute the enroll command is not the same as the onwer of the elastic-agent program files
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6144
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/4889

changelog/fragments/1734512365-embed-hints-inputs-in-agent-container-image.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Embed hints-based inputs in the Elastic Agent container image.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodates a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: This change includes the addition of hints-based inputs directly within the Elastic Agent container image, enabling streamlined configurations for input discovery when deployed in containerized environments.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6381
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/5661

deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml

@@ -1142,8 +1142,6 @@ spec:
         - mountPath: /hostfs/var/lib
           name: var-lib
           readOnly: true
-        - mountPath: /usr/share/elastic-agent/state/inputs.d
-          name: external-inputs
         - mountPath: /usr/share/elastic-agent/state
           name: agent-data
         - mountPath: /etc/elastic-agent/agent.yml
@@ -1152,27 +1150,6 @@ spec:
           subPath: agent.yml
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
-      initContainers:
-      - args:
-        - -c
-        - mkdir -p /etc/elastic-agent/inputs.d && mkdir -p /etc/elastic-agent/inputs.d
-          && wget -O - https://github.com/elastic/elastic-agent/archive/v9.0.0.tar.gz
-          | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-9.0.0/deploy/kubernetes/elastic-agent-standalone/templates.d"
-        command:
-        - sh
-        image: busybox:1.36.1
-        name: k8s-templates-downloader
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsGroup: 1000
-          runAsUser: 1000
-        volumeMounts:
-        - mountPath: /etc/elastic-agent/inputs.d
-          name: external-inputs
       nodeSelector:
         kubernetes.io/os: linux
       serviceAccountName: agent-pernode-example
@@ -1195,8 +1172,6 @@ spec:
       - hostPath:
           path: /var/lib
         name: var-lib
-      - emptyDir: {}
-        name: external-inputs
       - hostPath:
           path: /etc/elastic-agent/default/agent-pernode-example/state
           type: DirectoryOrCreate

deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml

@@ -1158,8 +1158,6 @@ spec:
         - mountPath: /hostfs/var/lib
           name: var-lib
           readOnly: true
-        - mountPath: /usr/share/elastic-agent/state/inputs.d
-          name: external-inputs
         - mountPath: /usr/share/elastic-agent/state
           name: agent-data
         - mountPath: /etc/elastic-agent/agent.yml
@@ -1168,27 +1166,6 @@ spec:
           subPath: agent.yml
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
-      initContainers:
-      - args:
-        - -c
-        - mkdir -p /etc/elastic-agent/inputs.d && mkdir -p /etc/elastic-agent/inputs.d
-          && wget -O - https://github.com/elastic/elastic-agent/archive/v9.0.0.tar.gz
-          | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-9.0.0/deploy/kubernetes/elastic-agent-standalone/templates.d"
-        command:
-        - sh
-        image: busybox:1.36.1
-        name: k8s-templates-downloader
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsGroup: 1000
-          runAsUser: 1000
-        volumeMounts:
-        - mountPath: /etc/elastic-agent/inputs.d
-          name: external-inputs
       nodeSelector:
         kubernetes.io/os: linux
       serviceAccountName: agent-pernode-example
@@ -1211,8 +1188,6 @@ spec:
       - hostPath:
           path: /var/lib
         name: var-lib
-      - emptyDir: {}
-        name: external-inputs
       - hostPath:
           path: /etc/elastic-agent/default/agent-pernode-example/state
           type: DirectoryOrCreate

deploy/helm/elastic-agent/templates/integrations/_kubernetes/_kubernetes.tpl

@@ -17,7 +17,6 @@
 {{- include "elasticagent.kubernetes.config.state.statefulsets.init" $ -}}
 {{- include "elasticagent.kubernetes.config.state.storageclasses.init" $ -}}
 {{- include "elasticagent.kubernetes.config.kube_controller.init" $ -}}
-{{- include "elasticagent.kubernetes.config.hints.init" $ -}}
 {{- include "elasticagent.kubernetes.config.audit_logs.init" $ -}}
 {{- include "elasticagent.kubernetes.config.container_logs.init" $ -}}
 {{- include "elasticagent.kubernetes.config.kubelet.containers.init" $ -}}
@@ -28,4 +27,4 @@
 {{- include "elasticagent.kubernetes.config.kube_proxy.init" $ -}}
 {{- include "elasticagent.kubernetes.config.kube_scheduler.init" $ -}}
 {{- end -}}
-{{- end -}}
+{{- end -}}

deploy/helm/elastic-agent/templates/integrations/_kubernetes/_kubernetes_hints.tpl

@@ -1,6 +1,2 @@
 {{- define "elasticagent.kubernetes.config.hints.init" -}}
-{{- if eq $.Values.kubernetes.hints.enabled true -}}
-{{- $preset := $.Values.agent.presets.perNode -}}
-{{- include "elasticagent.preset.applyOnce" (list $ $preset "elasticagent.kubernetes.pernode.preset") -}}
-{{- end -}}
 {{- end -}}

deploy/helm/elastic-agent/templates/integrations/_kubernetes/_preset_pernode.tpl

@@ -3,7 +3,6 @@
 {{- include "elasticagent.preset.mutate.volumes" (list $ $.Values.agent.presets.perNode "elasticagent.kubernetes.pernode.preset.volumes") -}}
 {{- include "elasticagent.preset.mutate.outputs.byname" (list $ $.Values.agent.presets.perNode $.Values.kubernetes.output)}}
 {{- if and (eq $.Values.kubernetes.hints.enabled true) (eq $.Values.agent.fleet.enabled false) -}}
-{{- include "elasticagent.preset.mutate.initcontainers" (list $ $.Values.agent.presets.perNode "elasticagent.kubernetes.pernode.preset.initcontainers") -}}
 {{- include "elasticagent.preset.mutate.providers.kubernetes.hints" (list $ $.Values.agent.presets.perNode "elasticagent.kubernetes.pernode.preset.providers.kubernetes.hints") -}}
 {{- end -}}
 {{- if or (eq $.Values.kubernetes.scheduler.enabled true) (eq $.Values.kubernetes.controller_manager.enabled true) -}}
@@ -37,10 +36,6 @@ extraVolumeMounts:
 - name: var-lib
   mountPath: /hostfs/var/lib
   readOnly: true
-{{- if eq $.Values.kubernetes.hints.enabled true }}
-- name: external-inputs
-  mountPath: /usr/share/elastic-agent/state/inputs.d
-{{- end }}
 {{- end -}}
 
 {{- define "elasticagent.kubernetes.pernode.preset.volumes" -}}
@@ -63,34 +58,6 @@ extraVolumes:
 - name: var-lib
   hostPath:
     path: /var/lib
-{{- if eq $.Values.kubernetes.hints.enabled true }}
-- name: external-inputs
-  emptyDir: {}
-{{- end }}
-{{- end -}}
-
-{{- define "elasticagent.kubernetes.pernode.preset.initcontainers" -}}
-initContainers:
-- name: k8s-templates-downloader
-  image: busybox:1.36.1
-  securityContext:
-    allowPrivilegeEscalation: false
-    privileged: false
-    runAsUser: 1000
-    runAsGroup: 1000
-    capabilities:
-      drop:
-        - ALL
-  command: [ 'sh' ]
-  args:
-    - -c
-    - >-
-      mkdir -p /etc/elastic-agent/inputs.d &&
-      mkdir -p /etc/elastic-agent/inputs.d &&
-      wget -O - https://github.com/elastic/elastic-agent/archive/v{{$.Values.agent.version}}.tar.gz | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-{{$.Values.agent.version}}/deploy/kubernetes/elastic-agent-standalone/templates.d"
-  volumeMounts:
-    - name: external-inputs
-      mountPath: /etc/elastic-agent/inputs.d
 {{- end -}}
 
 {{- define "elasticagent.kubernetes.pernode.preset.providers.kubernetes.hints" -}}

deploy/kubernetes/elastic-agent-standalone/templates.d/activemq.yml

@@ -10,6 +10,8 @@ inputs:
             type: logs
           exclude_files:
             - .gz$
+          file_identity:
+            fingerprint: null
           parsers:
             - container:
                 format: auto
@@ -18,6 +20,8 @@ inputs:
             - /var/log/containers/*${kubernetes.hints.container_id}.log
           prospector:
             scanner:
+                fingerprint:
+                    enabled: true
                 symlinks: true
           tags:
             - forwarded
@@ -28,6 +32,8 @@ inputs:
             type: logs
           exclude_files:
             - .gz$
+          file_identity:
+            fingerprint: null
           multiline:
             match: after
             negate: true
@@ -40,6 +46,8 @@ inputs:
             - /var/log/containers/*${kubernetes.hints.container_id}.log
           prospector:
             scanner:
+                fingerprint:
+                    enabled: true
                 symlinks: true
           tags:
             - forwarded
@@ -62,7 +70,6 @@ inputs:
           path: /api/jolokia/?ignoreErrors=true&canonicalNaming=false
           period: ${kubernetes.hints.activemq.broker.period|kubernetes.hints.activemq.period|'10s'}
           tags:
-            - forwarded
             - activemq-broker
           username: ${kubernetes.hints.activemq.broker.username|kubernetes.hints.activemq.username|'admin'}
         - condition: ${kubernetes.hints.activemq.queue.enabled} == true or ${kubernetes.hints.activemq.enabled} == true
@@ -77,7 +84,6 @@ inputs:
           path: /api/jolokia/?ignoreErrors=true&canonicalNaming=false
           period: ${kubernetes.hints.activemq.queue.period|kubernetes.hints.activemq.period|'10s'}
           tags:
-            - forwarded
             - activemq-queue
           username: ${kubernetes.hints.activemq.queue.username|kubernetes.hints.activemq.username|'admin'}
         - condition: ${kubernetes.hints.activemq.topic.enabled} == true or ${kubernetes.hints.activemq.enabled} == true
@@ -92,7 +98,6 @@ inputs:
           path: /api/jolokia/?ignoreErrors=true&canonicalNaming=false
           period: ${kubernetes.hints.activemq.topic.period|kubernetes.hints.activemq.period|'10s'}
           tags:
-            - forwarded
             - activemq-topic
           username: ${kubernetes.hints.activemq.topic.username|kubernetes.hints.activemq.username|'admin'}
       data_stream.namespace: default

deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml

@@ -1,46 +1,4 @@
 inputs:
-    - name: filestream-apache
-      id: filestream-apache-${kubernetes.hints.container_id}
-      type: filestream
-      use_output: default
-      streams:
-        - condition: ${kubernetes.hints.apache.access.enabled} == true or ${kubernetes.hints.apache.enabled} == true
-          data_stream:
-            dataset: apache.access
-            type: logs
-          exclude_files:
-            - .gz$
-          parsers:
-            - container:
-                format: auto
-                stream: ${kubernetes.hints.apache.access.stream|'all'}
-          paths:
-            - /var/log/containers/*${kubernetes.hints.container_id}.log
-          prospector:
-            scanner:
-                symlinks: true
-          tags:
-            - apache-access
-        - condition: ${kubernetes.hints.apache.error.enabled} == true or ${kubernetes.hints.apache.enabled} == true
-          data_stream:
-            dataset: apache.error
-            type: logs
-          exclude_files:
-            - .gz$
-          parsers:
-            - container:
-                format: auto
-                stream: ${kubernetes.hints.apache.error.stream|'all'}
-          paths:
-            - /var/log/containers/*${kubernetes.hints.container_id}.log
-          processors:
-            - add_locale: null
-          prospector:
-            scanner:
-                symlinks: true
-          tags:
-            - apache-error
-      data_stream.namespace: default
     - name: httpjson-apache
       id: httpjson-apache-${kubernetes.hints.container_id}
       type: httpjson
@@ -139,3 +97,53 @@ inputs:
           period: ${kubernetes.hints.apache.status.period|kubernetes.hints.apache.period|'30s'}
           server_status_path: /server-status
       data_stream.namespace: default
+    - name: filestream-apache
+      id: filestream-apache-${kubernetes.hints.container_id}
+      type: filestream
+      use_output: default
+      streams:
+        - condition: ${kubernetes.hints.apache.access.enabled} == true or ${kubernetes.hints.apache.enabled} == true
+          data_stream:
+            dataset: apache.access
+            type: logs
+          exclude_files:
+            - .gz$
+          file_identity:
+            fingerprint: null
+          parsers:
+            - container:
+                format: auto
+                stream: ${kubernetes.hints.apache.access.stream|'all'}
+          paths:
+            - /var/log/containers/*${kubernetes.hints.container_id}.log
+          prospector:
+            scanner:
+                fingerprint:
+                    enabled: true
+                symlinks: true
+          tags:
+            - apache-access
+        - condition: ${kubernetes.hints.apache.error.enabled} == true or ${kubernetes.hints.apache.enabled} == true
+          data_stream:
+            dataset: apache.error
+            type: logs
+          exclude_files:
+            - .gz$
+          file_identity:
+            fingerprint: null
+          parsers:
+            - container:
+                format: auto
+                stream: ${kubernetes.hints.apache.error.stream|'all'}
+          paths:
+            - /var/log/containers/*${kubernetes.hints.container_id}.log
+          processors:
+            - add_locale: null
+          prospector:
+            scanner:
+                fingerprint:
+                    enabled: true
+                symlinks: true
+          tags:
+            - apache-error
+      data_stream.namespace: default