Extend agent container initialisation logic (#4925)

* feat: implement container initialisation that chowns related paths and raises capabilities

* doc: add changelog fragment

* fix: update NOTICE.txt

* fix: typo in container_init_test.go

* feat: add more unit-tests to increase coverage

* feat: utilise unix.Exec to re-exec elastic-agent when file capabilities are updated

* fix: handle err from utils.HasRoot

* fix: refactor skip-file-capabilities flag

* fix: add cap_setpcap at elastic-agent binary

* feat: make npm install work with elastic-agent user

* fix: remove --skip-file-capabilities flag

* fix: allow CAP_CHOWN in Ambient set

* fix: remove redundant comment from Dockerfile.elastic-agent.tmpl

* fix: remove IDE auto-corrections from NOTICE.txt

* chore: go mod tidy after merge

* feat: refactor to eliminate disrupting effects

* fix: correct import order

* fix: move logWarning inside container_init_linux.go

* fix: update NOTICE.txt

* fix: call initContainer before tryContainerLoadPaths for elastic-agent container

* fix: revert npm installation changes in agent container image

* fix: chown also symlinks

* fix: introduce support for inside kubernetes tests

* fix: permission for configuration configmap mount to support rootless agent

* fix: code format

* feat: revisit the runAsUser and runAsGroup in k8s test

* fix: go mod tidy

* fix: wrap errors in fmt.Errorf

* fix: update NOTICE.txt

* feat: add comments in container_init_linux.go

* fix: adjust artifact_paths to extract all artifacts

* fix: permission for configuration configmap mount to support rootless agent at base k8s files

* fix: log stderr for k8s inner tests

* fix: disable CGO for k8s inner tests

* fix: merge conflicts

* fix: permission for configuration configmap mount to support rootless agent at base k8s files

* fix: improve logging the output of status command when elastic-agent doesn't report healthy

* fix: revert back to pre-1.22 golang for loop to make linter happy

* fix: don't set runAsGroup inside k8s tests for deployment with root and elastic-agent uid

* fix: add DAC_READ_SEARCH capability in k8s tests for deployment with random uid and gid

* fix: add comment to elaborate why call initContainer before tryContainerLoadPaths

.buildkite/pipeline.yml

@@ -206,6 +206,7 @@ steps:
         command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
         artifact_paths:
           - "build/k8s-logs*/*"
+          - "build/k8s-logs*/**/*"
           - "build/TEST-**"
           - "build/diagnostics/*"
         agents:

changelog/fragments/1718273218-container-init-capabilities.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: extend agent container initialisation to chown paths and raise capabilities for non-root
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4925
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml

@@ -66,7 +66,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
+            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
               value: "false"
@@ -127,7 +127,7 @@ spec:
       volumes:
         - name: datastreams
           configMap:
-            defaultMode: 0640
+            defaultMode: 0644
             name: agent-node-datastreams
         - name: proc
           hostPath:

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml

@@ -66,7 +66,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
+            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
               value: "false"
@@ -127,7 +127,7 @@ spec:
       volumes:
         - name: datastreams
           configMap:
-            defaultMode: 0640
+            defaultMode: 0644
             name: agent-node-datastreams
         - name: proc
           hostPath:

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml

@@ -66,7 +66,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
+            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
               value: "false"
@@ -127,7 +127,7 @@ spec:
       volumes:
         - name: datastreams
           configMap:
-            defaultMode: 0640
+            defaultMode: 0644
             name: agent-ksm-datastreams
         - name: proc
           hostPath:

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -735,7 +735,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
+            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
               value: "false"
@@ -796,7 +796,7 @@ spec:
       volumes:
         - name: datastreams
           configMap:
-            defaultMode: 0640
+            defaultMode: 0644
             name: agent-node-datastreams
         - name: proc
           hostPath:

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml

@@ -66,7 +66,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
+            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
               value: "false"
@@ -127,7 +127,7 @@ spec:
       volumes:
         - name: datastreams
           configMap:
-            defaultMode: 0640
+            defaultMode: 0644
             name: agent-node-datastreams
         - name: proc
           hostPath:

dev-tools/mage/gotest.go

@@ -176,6 +176,33 @@ func InstallGoTestTools() error {
 	)
 }
 
+func GoTestBuild(ctx context.Context, params GoTestArgs) error {
+	if params.OutputFile == "" {
+		return fmt.Errorf("missing output file")
+	}
+
+	fmt.Println(">> go test:", params.LogName, "Building Test Binary")
+
+	args := []string{"test", "-c", "-o", params.OutputFile}
+
+	if len(params.Tags) > 0 {
+		params := strings.Join(params.Tags, " ")
+		if params != "" {
+			args = append(args, "-tags", params)
+		}
+	}
+
+	args = append(args, params.Packages...)
+
+	goTestBuild := makeCommand(ctx, params.Env, "go", args...)
+
+	err := goTestBuild.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // GoTest invokes "go test" and reports the results to stdout. It returns an
 // error if there was any failure executing the tests or if there were any
 // test failures.

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -56,6 +56,7 @@ RUN true && \
     true
 
 # Keep this after any chown command, chown resets any applied capabilities
+RUN setcap =p {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent
 RUN setcap cap_net_raw,cap_setuid+p {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/components/agentbeat && \
 {{- if .linux_capabilities }}
 # Since the beat is stored at the other end of a symlink we must follow the symlink first

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2
 	github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e
 	github.com/cenkalti/backoff/v4 v4.3.0
+	github.com/docker/docker v26.1.4+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
 	github.com/elastic/elastic-agent-autodiscover v0.8.0
@@ -63,6 +64,7 @@ require (
 	go.elastic.co/go-licence-detector v0.6.1
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.25.0
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/sync v0.7.0
 	golang.org/x/sys v0.22.0
 	golang.org/x/term v0.22.0
@@ -80,6 +82,7 @@ require (
 	k8s.io/apimachinery v0.30.1
 	k8s.io/client-go v0.30.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	kernel.org/pub/linux/libs/security/libcap/cap v1.2.70
 	sigs.k8s.io/e2e-framework v0.4.0
 	sigs.k8s.io/kustomize/api v0.12.1
 	sigs.k8s.io/kustomize/kyaml v0.13.9
@@ -143,7 +146,6 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dnephin/pflag v1.0.7 // indirect
-	github.com/docker/docker v26.1.4+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/elastic/go-docappender/v2 v2.2.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
@@ -312,7 +314,6 @@ require (
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.starlark.net v0.0.0-20221205180719-3fd0dac74452 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
@@ -324,6 +325,7 @@ require (
 	howett.net/plist v1.0.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kubelet v0.29.3 // indirect
+	kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 // indirect
 	sigs.k8s.io/controller-runtime v0.18.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

go.sum

@@ -2471,6 +2471,10 @@ k8s.io/kubelet v0.29.3/go.mod h1:jDiGuTkFOUynyBKzOoC1xRSWlgAZ9UPcTYeFyjr6vas=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.70 h1:QnLPkuDWWbD5C+3DUA2IUXai5TK6w2zff+MAGccqdsw=
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.70/go.mod h1:/iBwcj9nbLejQitYvUm9caurITQ6WyNHibJk6Q9fiS4=
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 h1:HsB2G/rEQiYyo1bGoQqHZ/Bvd6x1rERQTNdPr1FyWjI=
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.70/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=

internal/pkg/agent/cmd/common.go

@@ -37,6 +37,14 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "elastic-agent [subcommand]",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if cmd.Name() == "container" {
+				// need to initialize container and try to chown agent-related paths
+				// before tryContainerLoadPaths as this will try to read/write from
+				// the agent state dir which might not have proper permissions when
+				// running inside a container
+				initContainer(streams)
+			}
+
 			return tryContainerLoadPaths()
 		},
 	}

internal/pkg/agent/cmd/container.go

@@ -43,9 +43,10 @@ import (
 const (
 	requestRetrySleepEnv     = "KIBANA_REQUEST_RETRY_SLEEP"
 	maxRequestRetriesEnv     = "KIBANA_REQUEST_RETRY_COUNT"
-	defaultRequestRetrySleep = "1s"                             // sleep 1 sec between retries for HTTP requests
-	defaultMaxRequestRetries = "30"                             // maximum number of retries for HTTP requests
-	defaultStateDirectory    = "/usr/share/elastic-agent/state" // directory that will hold the state data
+	defaultRequestRetrySleep = "1s"                          // sleep 1 sec between retries for HTTP requests
+	defaultMaxRequestRetries = "30"                          // maximum number of retries for HTTP requests
+	agentBaseDirectory       = "/usr/share/elastic-agent"    // directory that holds all elastic-agent related files
+	defaultStateDirectory    = agentBaseDirectory + "/state" // directory that will hold the state data
 
 	logsPathPerms = 0775
 )
@@ -153,6 +154,7 @@ occurs on every start of the container set FLEET_FORCE to 1.
 			}
 		},
 	}
+
 	return &cmd
 }