Merge branch 'main' into container_ubuntu_to_wolfi

changelog/fragments/1733862556-Log-warning-on-same-version-upgrade-attempts.yaml

@@ -0,0 +1,34 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Log warning on same version upgrade attempts
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Log a warning instead of reporting an error whan a same-version upgrade is
+  attempted. This prevents the agent from reporting a "failed" status.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6186

changelog/fragments/1734023789-add-loadbalancing-exporter-to-EDOT-collector.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: add loadbalancing exporter to EDOT collector
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1735137195-remove-deprecated-list-in-favor-of-items.yaml

@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: removes `list` from kibanaFetchToken in favor of `items` as the former is deprecated and will be removed from the api response
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6437
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6023

changelog/fragments/1735664420-remove-deprecated-path-install-cli-flag.yaml

@@ -0,0 +1,30 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: breaking-change
+
+# Change summary; a 80ish characters long description of the change.
+summary: Removing --path.install option
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6461/files
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/2489

deploy/helm/elastic-agent/examples/netflow-service/README.md

@@ -0,0 +1,28 @@
+# Example: Netflow Custom Integration
+
+In this example we define a `netflow` custom integration alongside a custom agent preset defined in [agent-netflow-values.yaml](agent-netflow-values.yaml). Also, we disable all `kubernetes` related providers and creation of cluster role and service account, as they are not required for this example.
+
+## Prerequisites:
+1. A k8s secret that contains the connection details to an Elasticsearch cluster such as the URL and the API key ([Kibana - Creating API Keys](https://www.elastic.co/guide/en/kibana/current/api-keys.html)):
+    ```console
+    kubectl create secret generic es-api-secret \
+       --from-literal=api_key=... \
+       --from-literal=url=...
+    ```
+
+2. `NetFlow Records` integration assets are installed through Kibana
+
+## Run:
+1. Install Helm chart
+    ```console
+    helm install elastic-agent ../../ -f ./agent-netflow-values.yaml
+    ```
+
+2. Run the netflow data generator deployment
+    ```console
+   kubectl run -it --rm netflow-generator --image=networkstatic/nflow-generator --restart=Never -- -t agent-netflow-elastic-agent.default.svc.cluster.local -p 2055
+    ```
+
+## Validate:
+
+1. The Kibana `netflow`-related dashboards should start showing netflow related data.

deploy/helm/elastic-agent/examples/netflow-service/agent-netflow-values.yaml

@@ -0,0 +1,62 @@
+outputs:
+  default:
+    type: ESSecretAuthAPI
+    secretName: es-api-secret
+
+extraIntegrations:
+  netflow:
+    id: netflow-netflow-60a9d5b2-c611-4749-90bf-5e2443936c1d
+    name: netflow-1
+    preset: netflow
+    revision: 1
+    type: netflow
+    use_output: default
+    meta:
+      package:
+        name: netflow
+        version: 2.19.1
+    data_stream:
+      namespace: default
+    package_policy_id: 60a9d5b2-c611-4749-90bf-5e2443936c1d
+    streams:
+      - id: netflow-netflow.log-60a9d5b2-c611-4749-90bf-5e2443936c1d
+        data_stream:
+          dataset: netflow.log
+          type: logs
+        protocols:
+          - v1
+          - v5
+          - v6
+          - v7
+          - v8
+          - v9
+          - ipfix
+        host: '0.0.0.0:2055'
+        max_message_size: 10KiB
+        expiration_timeout: 30m
+        queue_size: 8192
+        detect_sequence_reset: true
+        tags:
+          - netflow
+          - forwarded
+        publisher_pipeline.disable_host: true
+
+kubernetes:
+  enabled: false
+
+agent:
+  unprivileged: true
+  presets:
+    netflow:
+      automountServiceAccountToken: false
+      mode: deployment
+      service:
+        type: ClusterIP
+      ports:
+        - containerPort: 2055
+          servicePort: 2055
+          protocol: UDP
+      serviceAccount:
+        create: false
+      clusterRole:
+        create: false

deploy/helm/elastic-agent/examples/netflow-service/rendered/manifest.yaml

@@ -0,0 +1,168 @@
+---
+# Source: elastic-agent/templates/agent/k8s/secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: agent-netflow-example
+  namespace: "default"
+  labels:
+    helm.sh/chart: elastic-agent-9.0.0-beta
+    app.kubernetes.io/name: elastic-agent
+    app.kubernetes.io/instance: example
+    app.kubernetes.io/version: 9.0.0
+stringData:
+
+  agent.yml: |-
+    id: agent-netflow-example
+    outputs:
+      default:
+        api_key: ${OUTPUT_DEFAULT_API_KEY}
+        hosts:
+        - ${OUTPUT_DEFAULT_URL}
+        type: elasticsearch
+    secret_references: []
+    inputs:
+      - data_stream:
+          namespace: default
+        id: netflow-netflow-60a9d5b2-c611-4749-90bf-5e2443936c1d
+        meta:
+          package:
+            name: netflow
+            version: 2.19.1
+        name: netflow-1
+        package_policy_id: 60a9d5b2-c611-4749-90bf-5e2443936c1d
+        preset: netflow
+        revision: 1
+        streams:
+        - data_stream:
+            dataset: netflow.log
+            type: logs
+          detect_sequence_reset: true
+          expiration_timeout: 30m
+          host: 0.0.0.0:2055
+          id: netflow-netflow.log-60a9d5b2-c611-4749-90bf-5e2443936c1d
+          max_message_size: 10KiB
+          protocols:
+          - v1
+          - v5
+          - v6
+          - v7
+          - v8
+          - v9
+          - ipfix
+          publisher_pipeline.disable_host: true
+          queue_size: 8192
+          tags:
+          - netflow
+          - forwarded
+        type: netflow
+        use_output: default
+    providers:
+      kubernetes_leaderelection:
+        enabled: false
+        leader_lease: example-netflow
+---
+# Source: elastic-agent/templates/agent/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent-netflow-example
+  namespace: "default"
+  labels:
+    helm.sh/chart: elastic-agent-9.0.0-beta
+    app.kubernetes.io/name: elastic-agent
+    app.kubernetes.io/instance: example
+    app.kubernetes.io/version: 9.0.0
+spec:
+  type: ClusterIP
+  selector:
+    name: agent-netflow-example
+  ports:
+    - port: 2055
+      targetPort: 2055
+      protocol: UDP
+---
+# Source: elastic-agent/templates/agent/k8s/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: agent-netflow-example
+  namespace: "default"
+  labels:
+    helm.sh/chart: elastic-agent-9.0.0-beta
+    app.kubernetes.io/name: elastic-agent
+    app.kubernetes.io/instance: example
+    app.kubernetes.io/version: 9.0.0
+spec:
+  selector:
+    matchLabels:
+      name: agent-netflow-example
+  template:
+    metadata:
+      labels:
+        name: agent-netflow-example
+      annotations:
+        checksum/config: 4e9f48f0d6ae172f2f6aa5d526b0ca3af7dd28250e7c06c9d4e67ec0a2fc4573
+    spec:
+      automountServiceAccountToken: false
+      containers:
+      - args:
+        - -c
+        - /etc/elastic-agent/agent.yml
+        - -e
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: STATE_PATH
+          value: /usr/share/elastic-agent/state
+        - name: OUTPUT_DEFAULT_URL
+          valueFrom:
+            secretKeyRef:
+              key: url
+              name: es-api-secret
+        - name: OUTPUT_DEFAULT_API_KEY
+          valueFrom:
+            secretKeyRef:
+              key: api_key
+              name: es-api-secret
+        image: docker.elastic.co/beats/elastic-agent:9.0.0-SNAPSHOT
+        imagePullPolicy: IfNotPresent
+        name: agent
+        ports:
+        - containerPort: 2055
+          protocol: UDP
+        securityContext:
+          capabilities:
+            add:
+            - CHOWN
+            - SETPCAP
+            - DAC_READ_SEARCH
+            - SYS_PTRACE
+            drop:
+            - ALL
+          privileged: false
+          runAsGroup: 1000
+          runAsUser: 1000
+        volumeMounts:
+        - mountPath: /usr/share/elastic-agent/state
+          name: agent-data
+        - mountPath: /etc/elastic-agent/agent.yml
+          name: config
+          readOnly: true
+          subPath: agent.yml
+      dnsPolicy: ClusterFirstWithHostNet
+      volumes:
+      - hostPath:
+          path: /etc/elastic-agent/default/agent-netflow-example/state
+          type: DirectoryOrCreate
+        name: agent-data
+      - name: config
+        secret:
+          defaultMode: 292
+          secretName: agent-netflow-example