Fix audit/unenroll calls when agent runs fleet-server (#6085)

* Fix audit/unenroll calls when agent runs fleet-server

* Fix integration test

* Increase output for failed test

* move notification attempt

* move notification attempt

* audit/uninstall will call localhost with certonly tls

* Fix linter

* remove tls setting

* Add PR link to changelog

* stop checking if we need to notify in an error is encountered

* Shorten non-fatal error messages to prevent output issues in terminals

* Don't load agentinfo, use ID value from config instead

* Add unprivilged tests to audit unenroll integration tests

* Add comments for setting hosts

changelog/fragments/1732043830-Fix-audit-unenroll-call-when-running-fleet-server.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix audit/unenroll call when running fleet-server
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: Fix the call to the audit/unenroll endpoint that occurs on uninstall when the fleet-server is running locally.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: fleet-server
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6085
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/5752

internal/pkg/agent/install/uninstall.go

@@ -20,7 +20,6 @@ import (
 	"github.com/schollz/progressbar/v3"
 
 	"github.com/elastic/elastic-agent-libs/logp"
-	"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/secret"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
@@ -48,6 +47,13 @@ var (
 	fleetAuditWaitMax  = time.Second * 10
 )
 
+// agentInfo is a custom type that implements the fleetapi.AgentInfo interface
+type agentInfo string
+
+func (a *agentInfo) AgentID() string {
+	return string(*a)
+}
+
 // Uninstall uninstalls persistently Elastic Agent on the system.
 func Uninstall(ctx context.Context, cfgFile, topPath, uninstallToken string, log *logp.Logger, pt *progressbar.ProgressBar) error {
 	cwd, err := os.Getwd()
@@ -59,6 +65,49 @@ func Uninstall(ctx context.Context, cfgFile, topPath, uninstallToken string, log
 		return fmt.Errorf("uninstall must be run from outside the installed path '%s'", topPath)
 	}
 
+	// check if the agent was installed using --unprivileged by checking the file vault for the agent secret (needed on darwin to correctly load the vault)
+	unprivileged, err := checkForUnprivilegedVault(ctx)
+	if err != nil {
+		return fmt.Errorf("error checking for unprivileged vault: %w", err)
+	}
+
+	// will only notify fleet of the uninstall command if it can gather config and agentinfo, and is not a stand-alone install
+	localFleet := false
+	notifyFleet := false
+	var agentID agentInfo
+	var cfg *configuration.Configuration
+	func() { // check if we need to notify in a func to allow us to return early if a (non-fatal) error is encountered.
+		// read local config
+		c, err := operations.LoadFullAgentConfig(ctx, log, cfgFile, false, unprivileged)
+		if err != nil {
+			pt.Describe("notify Fleet failed: unable to read config")
+			return
+		}
+		cfg, err = configuration.NewFromConfig(c)
+		if err != nil {
+			pt.Describe("notify Fleet failed: error transforming config")
+			return
+		}
+
+		if cfg != nil && !configuration.IsStandalone(cfg.Fleet) {
+			agentID = agentInfo(cfg.Settings.ID)
+			notifyFleet = true
+			if cfg.Fleet != nil && cfg.Fleet.Server != nil {
+				localFleet = true
+			}
+		}
+	}()
+
+	// Notify fleet-server while it is still running if it's running locally
+	if notifyFleet && localFleet {
+		// host is set in the agent/cmd/enroll_cmd.go by createFleetServerBootstrapConfig
+		// hosts is set in agent/application/actions/handlers/handler_action_policy_change.go by updateFleetConfig
+		// agents running the fleet-server integration should communicate over the internal API (defaults to localhost:8221)
+		// This may need to be fixed with https://github.com/elastic/elastic-agent/issues/4771
+		cfg.Fleet.Client.Hosts = []string{cfg.Fleet.Client.Host}
+		notifyFleetAuditUninstall(ctx, log, pt, cfg, &agentID) //nolint:errcheck // ignore the error as we can't act on it
+	}
+
 	// ensure service is stopped
 	status, err := EnsureStoppedService(topPath, pt)
 	if err != nil {
@@ -71,12 +120,6 @@ func Uninstall(ctx context.Context, cfgFile, topPath, uninstallToken string, log
 		return fmt.Errorf("failed trying to kill any running watcher: %w", err)
 	}
 
-	// check if the agent was installed using --unprivileged by checking the file vault for the agent secret (needed on darwin to correctly load the vault)
-	unprivileged, err := checkForUnprivilegedVault(ctx)
-	if err != nil {
-		return fmt.Errorf("error checking for unprivileged vault: %w", err)
-	}
-
 	// Uninstall components first
 	if err := uninstallComponents(ctx, cfgFile, uninstallToken, log, pt, unprivileged); err != nil {
 		// If service status was running it was stopped to uninstall the components.
@@ -111,27 +154,6 @@ func Uninstall(ctx context.Context, cfgFile, topPath, uninstallToken string, log
 		}
 	}
 
-	// will only notify fleet of the uninstall command if it can gather config and agentinfo, and is not a stand-alone install
-	notifyFleet := false
-	var ai *info.AgentInfo
-	c, err := operations.LoadFullAgentConfig(ctx, log, cfgFile, false, unprivileged)
-	if err != nil {
-		pt.Describe(fmt.Sprintf("unable to read agent config to determine if notifying Fleet is needed: %v", err))
-	}
-	cfg, err := configuration.NewFromConfig(c)
-	if err != nil {
-		pt.Describe(fmt.Sprintf("notify Fleet: unable to transform *config.Config to *configuration.Configuration: %v", err))
-	}
-
-	if cfg != nil && !configuration.IsStandalone(cfg.Fleet) {
-		ai, err = info.NewAgentInfo(ctx, false)
-		if err != nil {
-			pt.Describe(fmt.Sprintf("unable to read agent info, Fleet will not be notified of uninstall: %v", err))
-		} else {
-			notifyFleet = true
-		}
-	}
-
 	// remove existing directory
 	pt.Describe("Removing install directory")
 	err = RemovePath(topPath)
@@ -144,8 +166,8 @@ func Uninstall(ctx context.Context, cfgFile, topPath, uninstallToken string, log
 	}
 	pt.Describe("Removed install directory")
 
-	if notifyFleet {
-		notifyFleetAuditUninstall(ctx, log, pt, cfg, ai) //nolint:errcheck // ignore the error as we can't act on it
+	if notifyFleet && !localFleet {
+		notifyFleetAuditUninstall(ctx, log, pt, cfg, &agentID) //nolint:errcheck // ignore the error as we can't act on it
 	}
 
 	return nil
@@ -154,7 +176,7 @@ func Uninstall(ctx context.Context, cfgFile, topPath, uninstallToken string, log
 // notifyFleetAuditUninstall will attempt to notify fleet-server of the agent's uninstall.
 //
 // There are retries for the attempt after a 10s wait, but it is a best-effort approach.
-func notifyFleetAuditUninstall(ctx context.Context, log *logp.Logger, pt *progressbar.ProgressBar, cfg *configuration.Configuration, ai *info.AgentInfo) error {
+func notifyFleetAuditUninstall(ctx context.Context, log *logp.Logger, pt *progressbar.ProgressBar, cfg *configuration.Configuration, ai fleetapi.AgentInfo) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	pt.Describe("Attempting to notify Fleet of uninstall")

internal/pkg/agent/install/uninstall_test.go

@@ -22,7 +22,6 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/elastic/elastic-agent-libs/logp"
-	"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/secret"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
@@ -178,7 +177,7 @@ func TestNotifyFleetAuditUnenroll(t *testing.T) {
 
 	log, _ := logp.NewInMemory("test", zap.NewDevelopmentEncoderConfig())
 	pt := progressbar.NewOptions(-1, progressbar.OptionSetWriter(io.Discard))
-	ai := &info.AgentInfo{}
+	var agentID agentInfo = "testID"
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -194,7 +193,7 @@ func TestNotifyFleetAuditUnenroll(t *testing.T) {
 					},
 				},
 			}
-			err := notifyFleetAuditUninstall(context.Background(), log, pt, cfg, ai)
+			err := notifyFleetAuditUninstall(context.Background(), log, pt, cfg, &agentID)
 			if tc.err == nil {
 				assert.NoError(t, err)
 			} else {
@@ -222,7 +221,7 @@ func TestNotifyFleetAuditUnenroll(t *testing.T) {
 				},
 			},
 		}
-		err := notifyFleetAuditUninstall(context.Background(), log, pt, cfg, ai)
+		err := notifyFleetAuditUninstall(context.Background(), log, pt, cfg, &agentID)
 		assert.EqualError(t, err, "notify Fleet: failed")
 
 	})

internal/pkg/fleetapi/ack_cmd.go

@@ -23,7 +23,7 @@ const ackPath = "/api/fleet/agents/%s/acks"
 type AckEvent struct {
 	EventType string          `json:"type"`              //  'STATE' | 'ERROR' | 'ACTION_RESULT' | 'ACTION'
 	SubType   string          `json:"subtype"`           // 'RUNNING','STARTING','IN_PROGRESS','CONFIG','FAILED','STOPPING','STOPPED','DATA_DUMP','ACKNOWLEDGED','UNKNOWN';
-	Timestamp string          `json:"timestamp"`         // : '2019-01-05T14:32:03.36764-05:00',
+	Timestamp string          `json:"timestamp"`         // : '2019-01-05T14:32:03.36764-05:00'
 	ActionID  string          `json:"action_id"`         // : '48cebde1-c906-4893-b89f-595d943b72a2',
 	AgentID   string          `json:"agent_id"`          // : 'agent1',
 	Message   string          `json:"message,omitempty"` // : 'hello2',
@@ -84,11 +84,11 @@ func (e *AckResponse) Validate() error {
 // AckCmd is a fleet API command.
 type AckCmd struct {
 	client client.Sender
-	info   agentInfo
+	info   AgentInfo
 }
 
 // NewAckCmd creates a new api command.
-func NewAckCmd(info agentInfo, client client.Sender) *AckCmd {
+func NewAckCmd(info AgentInfo, client client.Sender) *AckCmd {
 	return &AckCmd{
 		client: client,
 		info:   info,

internal/pkg/fleetapi/audit_unenroll_cmd.go

@@ -58,10 +58,10 @@ func (e *AuditUnenrollRequest) Validate() error {
 
 type AuditUnenrollCmd struct {
 	client client.Sender
-	info   agentInfo
+	info   AgentInfo
 }
 
-func NewAuditUnenrollCmd(info agentInfo, client client.Sender) *AuditUnenrollCmd {
+func NewAuditUnenrollCmd(info AgentInfo, client client.Sender) *AuditUnenrollCmd {
 	return &AuditUnenrollCmd{
 		client: client,
 		info:   info,

internal/pkg/fleetapi/checkin_cmd.go

@@ -85,15 +85,15 @@ func (e *CheckinResponse) Validate() error {
 // CheckinCmd is a fleet API command.
 type CheckinCmd struct {
 	client client.Sender
-	info   agentInfo
+	info   AgentInfo
 }
 
-type agentInfo interface {
+type AgentInfo interface {
 	AgentID() string
 }
 
 // NewCheckinCmd creates a new api command.
-func NewCheckinCmd(info agentInfo, client client.Sender) *CheckinCmd {
+func NewCheckinCmd(info AgentInfo, client client.Sender) *CheckinCmd {
 	return &CheckinCmd{
 		client: client,
 		info:   info,