Merge branch 'main' of github.com:elastic/elastic-agent into test/log…

…-error

.buildkite/scripts/steps/integration_tests.sh

@@ -5,8 +5,8 @@ source .buildkite/scripts/common.sh
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
 # NOTE: use only after version bump when the new version is not yet available, for example:
-# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3"
-OVERRIDE_AGENT_PACKAGE_VERSION="8.10.2"
+# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
+OVERRIDE_AGENT_PACKAGE_VERSION=""
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"

.go-version

@@ -1 +1 @@
-1.20.9
+1.20.10

.golangci.yml

@@ -116,7 +116,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.9"
+    go: "1.20.10"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -136,17 +136,17 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.9"
+    go: "1.20.10"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.9"
+    go: "1.20.10"
     checks: ["all"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.9"
+    go: "1.20.10"
 
   gosec:
     excludes:

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.20.9
+ARG GO_VERSION=1.20.10
 FROM circleci/golang:${GO_VERSION}
 
 

Dockerfile.skaffold

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.20.9
+ARG GO_VERSION=1.20.10
 ARG crossbuild_image="docker.elastic.co/beats-dev/golang-crossbuild"
 ARG AGENT_VERSION=8.9.0-SNAPSHOT
 ARG AGENT_IMAGE="docker.elastic.co/beats/elastic-agent"

changelog/8.10.4.asciidoc

@@ -0,0 +1,55 @@
+// begin 8.10.4 relnotes
+
+[[release-notes-8.10.4]]
+==  8.10.4
+
+Review important information about the  8.10.4 release.
+
+
+
+[discrete]
+[[breaking-changes-8.10.4]]
+=== Breaking changes
+
+Breaking changes can prevent your application from optimal operation and
+performance. Before you upgrade, review the breaking changes, then mitigate the
+impact to your application.
+
+elastic-agent::
+
+[discrete]
+[[breaking-3591]]
+.`elastic-agent-autodiscover` library has been updated to version 0.6.4, disabling metadata For `kubernetes.deployment` and `kubernetes.cronjob` fields.
+[%collapsible]
+====
+*Details* +
+The `elastic-agent-autodiscover` Kubernetes library by default comes with `add_resource_metadata.deployment=false` and `add_resource_metadata.cronjob=false`.
+*Impact* +
+Pods that will be created from deployments or cronjobs will not have the extra metadata field for `kubernetes.deployment` or `kubernetes.cronjob`, respectively. This change was made to avoid the memory impact of keeping the feature enabled in big Kubernetes clusters.
+For more information, refer to {agent-pull}3591[#3591].
+====
+
+
+
+
+
+[discrete]
+[[new-features-8.10.4]]
+=== New features
+
+The 8.10.4 release adds the following new and notable features.
+
+
+elastic-agent::
+
+* Secondary Fallback For Package Signature Verification. {elastic-agent-pull}https://github.com/elastic/elastic-agent/pull/3453[#https://github.com/elastic/elastic-agent/pull/3453] {elastic-agent-issue}https://github.com/elastic/elastic-agent/issues/3264[#https://github.com/elastic/elastic-agent/issues/3264]
++
+Ability to upgrade securely in air-gapped environment where fleet server is the only reachable URI.
+
+
+
+
+
+
+
+// end 8.10.4 relnotes

changelog/8.10.4.yaml

@@ -0,0 +1,25 @@
+version: 8.10.4
+entries:
+    - kind: feature
+      summary: Secondary fallback for package signature verification
+      description: Ability to upgrade securely in air-gapped environment where fleet server is the only reachable URI.
+      component: elastic-agent
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/3453
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/3264
+      timestamp: 1695289867
+      file:
+        name: 1695289867-Secondary-fallback-for-package-signature-verification.yaml
+        checksum: 8f8c39d9eef2f5b6922353bcab9c4ee1b74b1378
+    - kind: breaking-change
+      summary: Elastic-agent-autodiscover to v0.6.4. Disables metadata for deployment and cronjob
+      description: Elastic-agent-autodiscover library by default comes with add_resource_metadata.deployment=false and add_resource_metadata.cronjob=false. Pods that will be created from deployments or cronjobs will not have the extra metadata field for kubernetes.deployment or kubernetes.cronjob respectively.
+      component: elastic-agent
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/3591
+      issue: []
+      timestamp: 1697102363
+      file:
+        name: 1697102363-updating_agentautodiscovery_810.yaml
+        checksum: fe9015185dc4d3fe85f9c2ebf9f47e64e26fc67d

changelog/fragments/1695289867-Secondary-fallback-for-package-signature-verification.yaml → changelog/fragments/1689328899-Elastic-Agent-container-runs-on-Azure-Container-Instances-.yaml

@@ -8,14 +8,14 @@
 # - security: impacts on the security of a product or a user’s deployment.
 # - upgrade: important information for someone upgrading from a prior version
 # - other: does not fit into any of the other categories
-kind: feature
+kind: bug
 
 # Change summary; a 80ish characters long description of the change.
-summary: Secondary fallback for package signature verification
+summary: Elastic-Agent container runs on Azure Container Instances 
 
 # Long description; in case the summary is not enough to describe the change
 # this field accommodate a description without length limits.
-description: Ability to upgrade securely in Air gapped environment where fleet server is the only reachable URI.
+#description: 
 
 # Affected component; a word indicating the component this changeset affects.
 component: elastic-agent
@@ -24,8 +24,8 @@ component: elastic-agent
 # If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
 # NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
 # Please provide it if you are adding a fragment for a different PR.
-pr: https://github.com/elastic/elastic-agent/pull/3453
+pr: 3576
 
 # Issue number; optional; the GitHub issue related to this changeset (either closes or is part of).
 # If not present is automatically filled by the tooling with the issue linked to the PR number.
-issue: https://github.com/elastic/elastic-agent/issues/3264
+issue: 82

changelog/fragments/1693403216-Surface-errors-during-Agent's-enroll.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Surface errors during Agent's enroll process, failing if any happens.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: install/enroll
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/3207
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1697111928-upgrade-to-go-1.20.9.yaml → changelog/fragments/1697229987-Upgrade-to-Go-1.20.10.yaml

@@ -11,21 +11,21 @@
 kind: security
 
 # Change summary; a 80ish characters long description of the change.
-summary: Upgrade to Go 1.20.9
+summary: Upgrade to Go 1.20.10.
 
 # Long description; in case the summary is not enough to describe the change
 # this field accommodate a description without length limits.
 # NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
 #description:
 
 # Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
-component: elastic-agent
+component: "elastic-agent"
 
 # PR URL; optional; the PR number that added the changeset.
 # If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
 # NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
 # Please provide it if you are adding a fragment for a different PR.
-pr: https://github.com/elastic/elastic-agent/pull/3393
+pr: https://github.com/elastic/elastic-agent/pull/3601
 
 # Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
 # If not present is automatically filled by the tooling with the issue linked to the PR number.

dev-tools/mage/godaemon.go

@@ -21,7 +21,7 @@ var (
 	}
 )
 
-// BuildGoDaemon builds the go-deamon binary.
+// BuildGoDaemon builds the go-daemon binary.
 func BuildGoDaemon() error {
 	if GOOS != "linux" {
 		return errors.New("go-daemon only builds for linux")

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -9,7 +9,6 @@ FROM {{ .buildFrom }} AS home
 COPY beat {{ $beatHome }}
 
 RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/logs && \
-    chown -R root:root {{ $beatHome }} && \
     find {{ $beatHome }} -type d -exec chmod 0755 {} \; && \
     find {{ $beatHome }} -type f -exec chmod 0644 {} \; && \
     find {{ $beatHome }}/data -type d -exec chmod 0770 {} \; && \
@@ -127,25 +126,16 @@ COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 COPY --from=home /opt /opt
 {{- end }}
 
-
-RUN setcap cap_net_raw,cap_setuid+p {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/components/heartbeat && \
-{{- if .linux_capabilities }}
-# Since the beat is stored at the other end of a symlink we must follow the symlink first
-# For security reasons setcap does not support symlinks. This is smart in the general case
-# but in our specific case since we're building a trusted image from trusted binaries this is
-# fine. Thus, we use readlink to follow the link and setcap on the actual binary
-    readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} && \
-{{- end }}
-true
-
 {{- if eq .user "root" }}
 {{- if contains .image_name "-cloud" }}
 # Generate folder for a stub command that will be overwritten at runtime
 RUN mkdir /app
 {{- end }}
 {{- else }}
-RUN groupadd --gid 1000 {{ .BeatName }}
-RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}
+RUN groupadd --gid 1000 {{ .BeatName }} && \
+    useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} && \
+    chown -R {{ .user }}:{{ .user }} {{ $beatHome }} && \
+    true
 
 {{- if contains .image_name "-cloud" }}
 # Generate folder for a stub command that will be overwritten at runtime
@@ -154,6 +144,17 @@ RUN chown {{ .user }} /app
 {{- end }}
 {{- end }}
 
+# Keep this after any chown command, chown resets any applied capabilities
+RUN setcap cap_net_raw,cap_setuid+p {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/components/heartbeat && \
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+   setcap {{ .linux_capabilities }}  $(readlink -f {{ $beatBinary }}) && \
+{{- end }}
+true
+
 {{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal")))  }}
 USER root
 ENV NODE_PATH={{ $beatHome }}/.node

internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go

@@ -88,6 +88,11 @@ func snapshotConfig(config *artifact.Config, versionOverride *agtversion.ParsedS
 }
 
 func snapshotURI(versionOverride *agtversion.ParsedSemVer, config *artifact.Config) (string, error) {
+	// Respect a non-default source URI even if the version is a snapshot.
+	if config.SourceURI != artifact.DefaultSourceURI {
+		return config.SourceURI, nil
+	}
+
 	// snapshot downloader is used also by the 'localremote' impl in case of agent currently running off a snapshot build:
 	// the 'localremote' downloader does not pass a specific version, implying that we should update to the latest snapshot
 	// build of the same <major>.<minor>.<patch>-SNAPSHOT version

internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go

@@ -0,0 +1,27 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package snapshot
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact"
+	"github.com/elastic/elastic-agent/pkg/version"
+)
+
+func TestNonDefaultSourceURI(t *testing.T) {
+	version, err := version.ParseVersion("8.12.0-SNAPSHOT")
+	require.NoError(t, err)
+
+	config := artifact.Config{
+		SourceURI: "localhost:1234",
+	}
+	sourceURI, err := snapshotURI(version, &config)
+	require.NoError(t, err)
+	require.Equal(t, config.SourceURI, sourceURI)
+
+}

internal/pkg/agent/cmd/enroll.go

@@ -351,7 +351,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command) error {
 	//  Error: failed to fix permissions: chown /Library/Elastic/Agent/data/elastic-agent-c13f91/elastic-agent.app: operation not permitted
 	// This is because we are fixing permissions twice, once during installation and again during the enrollment step.
 	// When we are enrolling as part of installation on MacOS, skip the second attempt to fix permissions.
-	var fixPermissions bool = fromInstall
+	fixPermissions := fromInstall
 	if runtime.GOOS == "darwin" {
 		fixPermissions = false
 	}