Decrypt client TLS certificate key for Elastic Defend

NOTICE.txt

@@ -1264,11 +1264,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.12.1-0.20241009140206-6634efed32ab
+Version: v0.12.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]-0.20241009140206-6634efed32ab/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Elastic Defend accepts passphrase protected client certificate key for mTLS
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+#component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/5490

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
 	github.com/elastic/elastic-agent-autodiscover v0.9.0
 	github.com/elastic/elastic-agent-client/v7 v7.16.0
-	github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab
+	github.com/elastic/elastic-agent-libs v0.12.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.3
 	github.com/elastic/elastic-transport-go/v8 v8.6.0
 	github.com/elastic/go-elasticsearch/v8 v8.15.0

go.sum

@@ -264,8 +264,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab h1:7XHtHd5w4BV48ToCKGE4NYA0kpI+hv+7S08kdiTjpac=
-github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0=
+github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A=
 github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=

internal/pkg/agent/application/application.go

@@ -172,9 +172,11 @@ func New(
 			log.Info("Parsed configuration and determined agent is managed by Fleet")
 
 			composableManaged = true
-			compModifiers = append(compModifiers, FleetServerComponentModifier(cfg.Fleet.Server),
+			compModifiers = append(compModifiers,
+				FleetServerComponentModifier(cfg.Fleet.Server),
 				InjectFleetConfigComponentModifier(cfg.Fleet, agentInfo),
 				EndpointSignedComponentModifier(),
+				EndpointTLSComponentModifier(log),
 				InjectProxyEndpointModifier(),
 			)
 

internal/pkg/agent/application/endpoint_component_modifier.go

@@ -0,0 +1,265 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package application
+
+import (
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/elastic/elastic-agent-client/v7/pkg/client"
+	"github.com/elastic/elastic-agent-libs/transport/tlscommon"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
+	"github.com/elastic/elastic-agent/pkg/component"
+	"github.com/elastic/elastic-agent/pkg/core/logger"
+)
+
+// tlsCache is used to cache the decrypted client certificate and key.
+// In environments with a high rate of config changes, such as in Kubernetes
+// using autodiscover, loading files and decrypting them might have a
+// non-negligible overhead.
+type tlsCache struct {
+	mu *sync.Mutex
+
+	CacheKey string
+
+	Certificate string
+	Key         string
+}
+
+func (tlsCache) MakeKey(keyPassPath, certPath, keyPath string) string {
+	return keyPassPath + certPath + keyPath
+}
+
+// EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input.
+// Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent).
+// Endpoint uses uninstall_token_hash in order to verify uninstall command token
+// and signing_key in order validate the action signature.
+// Example:
+//
+//	{
+//		....
+//		"signed": {
+//			"data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19",
+//			"signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="
+//		},
+//	    "revision": 1,
+//	    "type": "endpoint"
+//	}
+func EndpointSignedComponentModifier() coordinator.ComponentsModifier {
+	return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) {
+		const signedKey = "signed"
+
+		compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput)
+		if !ok {
+			return comps, nil
+		}
+
+		unit := comps[compIdx].Units[unitIdx]
+		unitCfgMap := unit.Config.Source.AsMap()
+		if signed, ok := cfg[signedKey]; ok {
+			unitCfgMap[signedKey] = signed
+		}
+
+		unitCfg, err := component.ExpectedConfig(unitCfgMap)
+		if err != nil {
+			return nil, err
+		}
+
+		unit.Config = unitCfg
+		comps[compIdx].Units[unitIdx] = unit
+
+		return comps, nil
+	}
+}
+
+// EndpointTLSComponentModifier decrypts the client TLS certificate key if it's
+// passphrase-protected. It replaces the content of 'fleet.ssl.key'
+// and 'certificate' with theirs decrypted version and removes
+// 'key_passphrase_path'.
+// It does so, ONLY for the client TLS configuration for mTLS used with
+// fleet-server.
+func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModifier {
+	return newEndpointTLSComponentModifier(log, &tlsCache{mu: &sync.Mutex{}})
+}
+
+func newEndpointTLSComponentModifier(log *logger.Logger, cache *tlsCache) func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) {
+	return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) {
+		compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput)
+		if !ok {
+			// endpoint not present, nothing to do
+			return comps, nil
+		}
+
+		unit := comps[compIdx].Units[unitIdx]
+		unitCfgMap := unit.Config.Source.AsMap()
+
+		// ensure the following config exists:
+		// fleet.ssl:
+		//   key_passphrase_path
+		//   certificate
+		//   key
+		fleetNode, ok := unitCfgMap["fleet"]
+		if !ok {
+			// if 'fleet' isn't, present nothing to do
+			return comps, nil
+		}
+		fleetMap, ok := fleetNode.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("EndpointTLSComponentModifier: 'fleet' node isn't a map, it is: %T", fleetNode)
+		}
+
+		sslNode, ok := fleetMap["ssl"]
+		if !ok {
+			// 'ssl' node not present isn't an issue
+			return comps, nil
+		}
+		sslMap, ok := sslNode.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("EndpointTLSComponentModifier: 'ssl' node isn't a map, it is: %T", sslNode)
+		}
+
+		keyPassPathI, ok := sslMap["key_passphrase_path"]
+		if !ok {
+			// if no key_passphrase_path, nothing to decrypt
+			return comps, nil
+		}
+		keyPassPath, ok := keyPassPathI.(string)
+		if !ok {
+			return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' isn't a string")
+		}
+		if keyPassPath == "" {
+			// key_passphrase_path shouldn't be empty, but if it's, nothing to decrypt
+			return comps, nil
+		}
+
+		keyI, ok := sslMap["key"]
+		if !ok {
+			// if there is a key_passphrase_path, the key must be present
+			return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'key' isn't present")
+		}
+		keyPath, ok := keyI.(string)
+		if !ok {
+			return nil, fmt.Errorf("EndpointTLSComponentModifier: 'key' isn't a string, it's %T", keyI)
+		}
+
+		certI, ok := sslMap["certificate"]
+		if !ok {
+			// if there is a key_passphrase_path, the certificate must be present
+			return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'certificate' isn't present")
+		}
+		certPath, ok := certI.(string)
+		if !ok {
+			return nil, errors.New("EndpointTLSComponentModifier: 'certificate' isn't a string")
+		}
+
+		// all TLS config exists and the certificate key is passphrase protected,
+		// now load and decrypt the key.
+		cert, key, err := loadCertificatesWithCache(log, cache, keyPassPath, certPath, keyPath)
+		if err != nil {
+			return nil, err
+		}
+
+		// remove 'key_passphrase_path' as the certificate key isn't encrypted
+		// anymore.
+		delete(sslMap, "key_passphrase_path")
+
+		// update the certificate and its key with their decrypted version.
+		sslMap["certificate"] = cert
+		sslMap["key"] = key
+
+		unitCfg, err := component.ExpectedConfig(unitCfgMap)
+		if err != nil {
+			return nil, fmt.Errorf("EndpointTLSComponentModifier: could not covert modified config to expected config: %w", err)
+		}
+
+		unit.Config = unitCfg
+		comps[compIdx].Units[unitIdx] = unit
+
+		return comps, nil
+	}
+}
+
+func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath string, certPath string, keyPath string) (string, string, error) {
+	cache.mu.Lock()
+	defer cache.mu.Unlock()
+
+	cacheKey := cache.MakeKey(keyPassPath, certPath, keyPath)
+
+	// cache hit
+	if cache.CacheKey == cacheKey {
+		return cache.Certificate, cache.Key, nil
+	}
+
+	cert, key, err := loadCertificates(log, keyPassPath, certPath, keyPath)
+	if err != nil {
+		return "", "", err
+	}
+
+	cache.CacheKey = cacheKey
+	cache.Certificate = cert
+	cache.Key = key
+
+	return cert, key, nil
+}
+
+func loadCertificates(log *logger.Logger, keyPassPathStr string, certStr string, keyStr string) (string, string, error) {
+	pass, err := os.ReadFile(keyPassPathStr)
+	if err != nil {
+		return "", "", fmt.Errorf("EndpointTLSComponentModifier: failed to read client certificate passphrase file: %w", err)
+	}
+
+	// we don't really support encrypted certificates, but it's how
+	// tlscommon.LoadCertificate does. Thus, let's keep the same behaviour.
+	// Also, tlscommon.LoadCertificate 'loses' the type of the private key.
+	// It stores they private key as an interface and there is no way to
+	// retrieve the type os the private key without a type assertion.
+	// Therefore, instead of manually checking the type, which would mean
+	// to check for all supported private key types and keep it up to date,
+	// better to load the certificate and its key directly from the PEM file.
+	cert, err := tlscommon.ReadPEMFile(log,
+		certStr, string(pass))
+	if err != nil {
+		return "", "", fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate: %w", err)
+	}
+	key, err := tlscommon.ReadPEMFile(log,
+		keyStr,
+		string(pass))
+	if err != nil {
+		return "", "", fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate key: %w", err)
+	}
+
+	// tlscommon.ReadPEMFile only removes the 'DEK-Info' header, not the
+	// 'Proc-Type', so remove it now. Create a pem.Block to avoid editing
+	// the PEM data manually:
+	keyBlock, _ := pem.Decode(key)
+	delete(keyBlock.Headers, "Proc-Type")
+	key = pem.EncodeToMemory(keyBlock)
+
+	return string(cert), string(key), nil
+}
+
+// findEndpointUnit finds the endpoint component and its unit of type 'unitType'.
+// It returns the component and unit index and true if found, if not, it returns
+// 0, 0, false.
+func findEndpointUnit(comps []component.Component, unitType client.UnitType) (int, int, bool) {
+	// find the endpoint component
+	for compIdx, comp := range comps {
+		if comp.InputSpec != nil && comp.InputSpec.InputType != endpoint {
+			continue
+		}
+
+		for unitIdx, unit := range comp.Units {
+			if unit.Type != unitType {
+				continue
+			}
+
+			return compIdx, unitIdx, true
+		}
+	}
+	return 0, 0, false
+}