From b48f69593bdc8c98315ddd895147600f0268d075 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 27 May 2024 16:18:20 -0400
Subject: [PATCH 01/49] Define development mode, alter install path.

---
 internal/pkg/agent/application/paths/common.go | 17 ++++++++++++++++-
 internal/pkg/agent/cmd/install.go              |  9 +++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/agent/application/paths/common.go b/internal/pkg/agent/application/paths/common.go
index 871435e65a1..6c80a1caaed 100644
--- a/internal/pkg/agent/application/paths/common.go
+++ b/internal/pkg/agent/application/paths/common.go
@@ -61,6 +61,7 @@ var (
 	installPath       string
 	controlSocketPath string
 	unversionedHome   bool
+	isDevelopmentMode bool
 	tmpCreator        sync.Once
 )
 
@@ -316,9 +317,23 @@ func BinaryPath(baseDir, agentName string) string {
 	return filepath.Join(binaryDir(baseDir), agentName)
 }
 
+// SetIsDevelopmentMode sets whether the agent is installed in development mode or not.
+func SetIsDevelopmentMode(developmentMode bool) {
+	isDevelopmentMode = developmentMode
+}
+
+// IsDevelopmentMode returns true if the agent is installed in development mode.
+func IsDevelopmentMode() bool {
+	return isDevelopmentMode
+}
+
 // InstallPath returns the top level directory Agent will be installed into.
 func InstallPath(basePath string) string {
-	return filepath.Join(basePath, "Elastic", "Agent")
+	elasticPath := filepath.Join(basePath, "Elastic")
+	if isDevelopmentMode {
+		return filepath.Join(elasticPath, "DevelopmentAgent")
+	}
+	return filepath.Join(elasticPath, "Agent")
 }
 
 // TopBinaryPath returns the path to the Elastic Agent binary that is inside the Top directory.
diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index cb58f2a8f7c..a80914449b6 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -25,6 +25,7 @@ import (
 const (
 	flagInstallBasePath     = "base-path"
 	flagInstallUnprivileged = "unprivileged"
+	flagInstallDevelopment  = "development"
 )
 
 func newInstallCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
@@ -49,6 +50,8 @@ would like the Agent to operate.
 	cmd.Flags().String(flagInstallBasePath, paths.DefaultBasePath, "The path where the Elastic Agent will be installed. It must be an absolute path.")
 	cmd.Flags().Bool(flagInstallUnprivileged, false, "Installed Elastic Agent will create an 'elastic-agent' user and run as that user. (experimental)")
 	_ = cmd.Flags().MarkHidden(flagInstallUnprivileged) // Hidden until fully supported
+	cmd.Flags().Bool(flagInstallDevelopment, false, "Install Elastic Agent for development in an isolated base path. (experimental)")
+	_ = cmd.Flags().MarkHidden(flagInstallDevelopment) // For internal use only.
 	addEnrollFlags(cmd)
 
 	return cmd
@@ -80,6 +83,12 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 		fmt.Fprintln(streams.Out, "Unprivileged installation mode enabled; this is an experimental and currently unsupported feature.")
 	}
 
+	isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
+	if unprivileged {
+		fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental feature.")
+	}
+	paths.SetIsDevelopmentMode(isDevelopmentMode)
+
 	topPath := paths.InstallPath(basePath)
 
 	status, reason := install.Status(topPath)

From ceb17e208d54cbebcd7c90f16230c7102466d473 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 27 May 2024 16:28:14 -0400
Subject: [PATCH 02/49] Change service name in development mode.

---
 .../pkg/agent/application/paths/common.go     |  7 +++++++
 .../agent/application/paths/paths_darwin.go   |  5 +++--
 .../agent/application/paths/paths_linux.go    |  5 +++--
 .../agent/application/paths/paths_windows.go  |  5 +++--
 .../application/reexec/reexec_windows.go      |  2 +-
 .../upgrade/service_update_linux.go           |  2 +-
 .../upgrade/service_update_linux_test.go      |  2 +-
 internal/pkg/agent/install/install.go         | 20 +++++++++----------
 internal/pkg/agent/install/install_windows.go |  2 +-
 internal/pkg/agent/install/svc.go             |  6 +++---
 internal/pkg/agent/install/uninstall.go       | 14 ++++++-------
 11 files changed, 40 insertions(+), 30 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common.go b/internal/pkg/agent/application/paths/common.go
index 6c80a1caaed..6db06c2f17c 100644
--- a/internal/pkg/agent/application/paths/common.go
+++ b/internal/pkg/agent/application/paths/common.go
@@ -327,6 +327,13 @@ func IsDevelopmentMode() bool {
 	return isDevelopmentMode
 }
 
+func ServiceName() string {
+	if isDevelopmentMode {
+		return serviceNameDevelopmentMode
+	}
+	return serviceName
+}
+
 // InstallPath returns the top level directory Agent will be installed into.
 func InstallPath(basePath string) string {
 	elasticPath := filepath.Join(basePath, "Elastic")
diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index 6630176660e..dc5ed6601b1 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -18,8 +18,9 @@ const (
 	// created to the control socket when Elastic Agent is running with root.
 	ControlSocketRunSymlink = "/var/run/elastic-agent.sock"
 
-	// ServiceName is the service name when installed.
-	ServiceName = "co.elastic.elastic-agent"
+	// serviceName is the service name when installed.
+	serviceName                = "co.elastic.elastic-agent"
+	serviceNameDevelopmentMode = "co.elastic.elastic-agent-development"
 
 	// ShellWrapperPath is the path to the installed shell wrapper.
 	ShellWrapperPath = "/usr/local/bin/elastic-agent"
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index f5450142752..84c1926e4c1 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -14,8 +14,9 @@ const (
 	// for installing Elastic Agent's files.
 	DefaultBasePath = "/opt"
 
-	// ServiceName is the service name when installed.
-	ServiceName = "elastic-agent"
+	// serviceName is the service name when installed.
+	serviceName                = "elastic-agent"
+	serviceNameDevelopmentMode = "elastic-agent-development"
 
 	// ShellWrapperPath is the path to the installed shell wrapper.
 	ShellWrapperPath = "/usr/bin/elastic-agent"
diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index 5c8f7008d6d..ddf2de3d66e 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -23,8 +23,9 @@ const (
 	// ControlSocketRunSymlink is not created on Windows.
 	ControlSocketRunSymlink = ""
 
-	// ServiceName is the service name when installed.
-	ServiceName = "Elastic Agent"
+	// serviceName is the service name when installed.
+	serviceName                = "Elastic Agent"
+	serviceNameDevelopmentMode = "Elastic Agent (Development)"
 
 	// ShellWrapperPath is the path to the installed shell wrapper.
 	ShellWrapperPath = "" // no wrapper on Windows
diff --git a/internal/pkg/agent/application/reexec/reexec_windows.go b/internal/pkg/agent/application/reexec/reexec_windows.go
index cc9eeb4950b..530ab4dcdf4 100644
--- a/internal/pkg/agent/application/reexec/reexec_windows.go
+++ b/internal/pkg/agent/application/reexec/reexec_windows.go
@@ -46,7 +46,7 @@ func reexec(log *logger.Logger, executable string, argOverrides ...string) error
 			_ = t.Close()
 		}()
 
-		args := []string{filepath.Base(executable), "reexec_windows", paths.ServiceName, strconv.Itoa(os.Getpid())}
+		args := []string{filepath.Base(executable), "reexec_windows", paths.ServiceName(), strconv.Itoa(os.Getpid())}
 		args = append(args, argOverrides...)
 		cmd := exec.Cmd{
 			Path:        executable,
diff --git a/internal/pkg/agent/application/upgrade/service_update_linux.go b/internal/pkg/agent/application/upgrade/service_update_linux.go
index 618b632f93a..a09eddf4bdf 100644
--- a/internal/pkg/agent/application/upgrade/service_update_linux.go
+++ b/internal/pkg/agent/application/upgrade/service_update_linux.go
@@ -23,7 +23,7 @@ import (
 func EnsureServiceConfigUpToDate() error {
 	switch service.ChosenSystem().String() {
 	case "linux-systemd":
-		unitFilePath := "/etc/systemd/system/" + paths.ServiceName + ".service"
+		unitFilePath := "/etc/systemd/system/" + paths.ServiceName() + ".service"
 		updated, err := ensureSystemdServiceConfigUpToDate(unitFilePath)
 		if err != nil {
 			return err
diff --git a/internal/pkg/agent/application/upgrade/service_update_linux_test.go b/internal/pkg/agent/application/upgrade/service_update_linux_test.go
index fa7913a5e93..eb2563f71de 100644
--- a/internal/pkg/agent/application/upgrade/service_update_linux_test.go
+++ b/internal/pkg/agent/application/upgrade/service_update_linux_test.go
@@ -94,7 +94,7 @@ WantedBy=multi-user.target
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
-			unitFilePath := filepath.Join(t.TempDir(), paths.ServiceName+".service")
+			unitFilePath := filepath.Join(t.TempDir(), paths.ServiceName()+".service")
 			err := os.WriteFile(unitFilePath, []byte(test.unitFileInitialContents), 0644)
 			require.NoError(t, err)
 
diff --git a/internal/pkg/agent/install/install.go b/internal/pkg/agent/install/install.go
index 922bb85d8b6..ee53e5f18d8 100644
--- a/internal/pkg/agent/install/install.go
+++ b/internal/pkg/agent/install/install.go
@@ -206,8 +206,8 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 		pt.Describe("Failed to install service")
 		return ownership, errors.New(
 			err,
-			fmt.Sprintf("failed to install service (%s)", paths.ServiceName),
-			errors.M("service", paths.ServiceName))
+			fmt.Sprintf("failed to install service (%s)", paths.ServiceName()),
+			errors.M("service", paths.ServiceName()))
 	}
 	err = servicePostInstall(ownership)
 	if err != nil {
@@ -217,8 +217,8 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 		_ = svc.Uninstall()
 		return ownership, errors.New(
 			err,
-			fmt.Sprintf("failed to configure service (%s)", paths.ServiceName),
-			errors.M("service", paths.ServiceName))
+			fmt.Sprintf("failed to configure service (%s)", paths.ServiceName()),
+			errors.M("service", paths.ServiceName()))
 	}
 	pt.Describe("Installed service")
 
@@ -393,8 +393,8 @@ func StartService(topPath string) error {
 	if err != nil {
 		return errors.New(
 			err,
-			fmt.Sprintf("failed to start service (%s)", paths.ServiceName),
-			errors.M("service", paths.ServiceName))
+			fmt.Sprintf("failed to start service (%s)", paths.ServiceName()),
+			errors.M("service", paths.ServiceName()))
 	}
 	return nil
 }
@@ -410,8 +410,8 @@ func StopService(topPath string) error {
 	if err != nil {
 		return errors.New(
 			err,
-			fmt.Sprintf("failed to stop service (%s)", paths.ServiceName),
-			errors.M("service", paths.ServiceName))
+			fmt.Sprintf("failed to stop service (%s)", paths.ServiceName()),
+			errors.M("service", paths.ServiceName()))
 	}
 	return nil
 }
@@ -427,8 +427,8 @@ func RestartService(topPath string) error {
 	if err != nil {
 		return errors.New(
 			err,
-			fmt.Sprintf("failed to restart service (%s)", paths.ServiceName),
-			errors.M("service", paths.ServiceName))
+			fmt.Sprintf("failed to restart service (%s)", paths.ServiceName()),
+			errors.M("service", paths.ServiceName()))
 	}
 	return nil
 }
diff --git a/internal/pkg/agent/install/install_windows.go b/internal/pkg/agent/install/install_windows.go
index 3790532b7fa..08a729eeace 100644
--- a/internal/pkg/agent/install/install_windows.go
+++ b/internal/pkg/agent/install/install_windows.go
@@ -100,7 +100,7 @@ func servicePostInstall(ownership utils.FileOwner) error {
 	if err != nil {
 		return fmt.Errorf("failed to get DACL from security descriptor: %w", err)
 	}
-	err = windows.SetNamedSecurityInfo(paths.ServiceName, windows.SE_SERVICE, windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)
+	err = windows.SetNamedSecurityInfo(paths.ServiceName(), windows.SE_SERVICE, windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)
 	if err != nil {
 		return fmt.Errorf("failed to set DACL for service(%s): %w", paths.ServiceName, err)
 	}
diff --git a/internal/pkg/agent/install/svc.go b/internal/pkg/agent/install/svc.go
index 3fc599fb80a..dc2e2b6f223 100644
--- a/internal/pkg/agent/install/svc.go
+++ b/internal/pkg/agent/install/svc.go
@@ -75,7 +75,7 @@ func newService(topPath string, opt ...serviceOpt) (service.Service, error) {
 	}
 
 	cfg := &service.Config{
-		Name:             paths.ServiceName,
+		Name:             paths.ServiceName(),
 		DisplayName:      ServiceDisplayName,
 		Description:      ServiceDescription,
 		Executable:       ExecutablePath(topPath),
@@ -107,8 +107,8 @@ func newService(topPath string, opt ...serviceOpt) (service.Service, error) {
 
 		// Set the stdout and stderr logs to be inside the installation directory, ensures that the
 		// executing user for the service can write to the directory for the logs.
-		cfg.Option["StandardOutPath"] = filepath.Join(topPath, fmt.Sprintf("%s.out.log", paths.ServiceName))
-		cfg.Option["StandardErrorPath"] = filepath.Join(topPath, fmt.Sprintf("%s.err.log", paths.ServiceName))
+		cfg.Option["StandardOutPath"] = filepath.Join(topPath, fmt.Sprintf("%s.out.log", paths.ServiceName()))
+		cfg.Option["StandardErrorPath"] = filepath.Join(topPath, fmt.Sprintf("%s.err.log", paths.ServiceName()))
 	}
 
 	return service.New(nil, cfg)
diff --git a/internal/pkg/agent/install/uninstall.go b/internal/pkg/agent/install/uninstall.go
index f333e8c49a5..ebdd9a5c87b 100644
--- a/internal/pkg/agent/install/uninstall.go
+++ b/internal/pkg/agent/install/uninstall.go
@@ -60,20 +60,20 @@ func Uninstall(cfgFile, topPath, uninstallToken string, log *logp.Logger, pt *pr
 			pt.Describe("Failed to issue stop service")
 			return aerrors.New(
 				err,
-				fmt.Sprintf("failed to issue stop service (%s)", paths.ServiceName),
-				aerrors.M("service", paths.ServiceName))
+				fmt.Sprintf("failed to issue stop service (%s)", paths.ServiceName()),
+				aerrors.M("service", paths.ServiceName()))
 		}
 	}
 	// The kardianos service manager can't tell the difference
 	// between 'Stopped' and 'StopPending' on Windows, so make
 	// sure the service is stopped.
-	err = isStopped(30*time.Second, 250*time.Millisecond, paths.ServiceName)
+	err = isStopped(30*time.Second, 250*time.Millisecond, paths.ServiceName())
 	if err != nil {
 		pt.Describe("Failed to complete stop of service")
 		return aerrors.New(
 			err,
-			fmt.Sprintf("failed to complete stop service (%s)", paths.ServiceName),
-			aerrors.M("service", paths.ServiceName))
+			fmt.Sprintf("failed to complete stop service (%s)", paths.ServiceName()),
+			aerrors.M("service", paths.ServiceName()))
 	}
 
 	pt.Describe("Successfully stopped service")
@@ -99,8 +99,8 @@ func Uninstall(cfgFile, topPath, uninstallToken string, log *logp.Logger, pt *pr
 			if startErr := svc.Start(); startErr != nil {
 				return aerrors.New(
 					err,
-					fmt.Sprintf("failed to restart service (%s), after failed components uninstall: %v", paths.ServiceName, startErr),
-					aerrors.M("service", paths.ServiceName))
+					fmt.Sprintf("failed to restart service (%s), after failed components uninstall: %v", paths.ServiceName(), startErr),
+					aerrors.M("service", paths.ServiceName()))
 			}
 		}
 		return fmt.Errorf("error uninstalling components: %w", err)

From 4b2f1a777f288cc518e6caa8979e238be2e286c0 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 27 May 2024 16:33:05 -0400
Subject: [PATCH 03/49] Make shell wrapper path conditional on development
 mode.

---
 .../pkg/agent/application/paths/common.go     |  7 ++++
 .../agent/application/paths/paths_darwin.go   |  5 +--
 .../agent/application/paths/paths_linux.go    |  5 +--
 .../agent/application/paths/paths_windows.go  |  5 +--
 internal/pkg/agent/install/install.go         | 32 +++++++++----------
 internal/pkg/agent/install/svc.go             |  4 +--
 internal/pkg/agent/install/uninstall.go       |  8 ++---
 7 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common.go b/internal/pkg/agent/application/paths/common.go
index 6db06c2f17c..2b289f0fc5e 100644
--- a/internal/pkg/agent/application/paths/common.go
+++ b/internal/pkg/agent/application/paths/common.go
@@ -334,6 +334,13 @@ func ServiceName() string {
 	return serviceName
 }
 
+func ShellWrapperPath() string {
+	if isDevelopmentMode {
+		return shellWrapperPathDevelopmentMode
+	}
+	return shellWrapperPath
+}
+
 // InstallPath returns the top level directory Agent will be installed into.
 func InstallPath(basePath string) string {
 	elasticPath := filepath.Join(basePath, "Elastic")
diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index dc5ed6601b1..e69ecb9223c 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -22,8 +22,9 @@ const (
 	serviceName                = "co.elastic.elastic-agent"
 	serviceNameDevelopmentMode = "co.elastic.elastic-agent-development"
 
-	// ShellWrapperPath is the path to the installed shell wrapper.
-	ShellWrapperPath = "/usr/local/bin/elastic-agent"
+	// shellWrapperPath is the path to the installed shell wrapper.
+	shellWrapperPath                = "/usr/local/bin/elastic-agent"
+	shellWrapperPathDevelopmentMode = "/usr/local/bin/elastic-dev-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index 84c1926e4c1..db0c6b9a366 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -18,8 +18,9 @@ const (
 	serviceName                = "elastic-agent"
 	serviceNameDevelopmentMode = "elastic-agent-development"
 
-	// ShellWrapperPath is the path to the installed shell wrapper.
-	ShellWrapperPath = "/usr/bin/elastic-agent"
+	// shellWrapperPath is the path to the installed shell wrapper.
+	shellWrapperPath                = "/usr/bin/elastic-agent"
+	shellWrapperPathDevelopmentMode = "/usr/bin/elastic-dev-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index ddf2de3d66e..9d76f433ea4 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -27,8 +27,9 @@ const (
 	serviceName                = "Elastic Agent"
 	serviceNameDevelopmentMode = "Elastic Agent (Development)"
 
-	// ShellWrapperPath is the path to the installed shell wrapper.
-	ShellWrapperPath = "" // no wrapper on Windows
+	// shellWrapperPath is the path to the installed shell wrapper.
+	shellWrapperPath                = ""
+	shellWrapperPathDevelopmentMode = ""
 
 	// ShellWrapper is the wrapper that is installed.
 	ShellWrapper = "" // no wrapper on Windows
diff --git a/internal/pkg/agent/install/install.go b/internal/pkg/agent/install/install.go
index ee53e5f18d8..27aff10f003 100644
--- a/internal/pkg/agent/install/install.go
+++ b/internal/pkg/agent/install/install.go
@@ -126,13 +126,13 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 	pt.Describe("Successfully copied files")
 
 	// place shell wrapper, if present on platform
-	if paths.ShellWrapperPath != "" {
-		pathDir := filepath.Dir(paths.ShellWrapperPath)
+	if paths.ShellWrapperPath() != "" {
+		pathDir := filepath.Dir(paths.ShellWrapperPath())
 		err = os.MkdirAll(pathDir, 0755)
 		if err != nil {
 			return utils.FileOwner{}, errors.New(
 				err,
-				fmt.Sprintf("failed to create directory (%s) for shell wrapper (%s)", pathDir, paths.ShellWrapperPath),
+				fmt.Sprintf("failed to create directory (%s) for shell wrapper (%s)", pathDir, paths.ShellWrapperPath()),
 				errors.M("directory", pathDir))
 		}
 		// Install symlink for darwin instead of the wrapper script.
@@ -141,32 +141,32 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 		// This is specifically important for osquery FDA permissions at the moment.
 		if runtime.GOOS == darwin {
 			// Check if previous shell wrapper or symlink exists and remove it so it can be overwritten
-			if _, err := os.Lstat(paths.ShellWrapperPath); err == nil {
-				if err := os.Remove(paths.ShellWrapperPath); err != nil {
+			if _, err := os.Lstat(paths.ShellWrapperPath()); err == nil {
+				if err := os.Remove(paths.ShellWrapperPath()); err != nil {
 					return utils.FileOwner{}, errors.New(
 						err,
-						fmt.Sprintf("failed to remove (%s)", paths.ShellWrapperPath),
-						errors.M("destination", paths.ShellWrapperPath))
+						fmt.Sprintf("failed to remove (%s)", paths.ShellWrapperPath()),
+						errors.M("destination", paths.ShellWrapperPath()))
 				}
 			}
-			err = os.Symlink(filepath.Join(topPath, paths.BinaryName), paths.ShellWrapperPath)
+			err = os.Symlink(filepath.Join(topPath, paths.BinaryName), paths.ShellWrapperPath())
 			if err != nil {
 				return utils.FileOwner{}, errors.New(
 					err,
-					fmt.Sprintf("failed to create elastic-agent symlink (%s)", paths.ShellWrapperPath),
-					errors.M("destination", paths.ShellWrapperPath))
+					fmt.Sprintf("failed to create elastic-agent symlink (%s)", paths.ShellWrapperPath()),
+					errors.M("destination", paths.ShellWrapperPath()))
 			}
 		} else {
 			// We use strings.Replace instead of fmt.Sprintf here because, with the
 			// latter, govet throws a false positive error here: "fmt.Sprintf call has
 			// arguments but no formatting directives".
 			shellWrapper := strings.Replace(paths.ShellWrapper, "%s", topPath, -1)
-			err = os.WriteFile(paths.ShellWrapperPath, []byte(shellWrapper), 0755)
+			err = os.WriteFile(paths.ShellWrapperPath(), []byte(shellWrapper), 0755)
 			if err != nil {
 				return utils.FileOwner{}, errors.New(
 					err,
-					fmt.Sprintf("failed to write shell wrapper (%s)", paths.ShellWrapperPath),
-					errors.M("destination", paths.ShellWrapperPath))
+					fmt.Sprintf("failed to write shell wrapper (%s)", paths.ShellWrapperPath()),
+					errors.M("destination", paths.ShellWrapperPath()))
 			}
 		}
 	}
@@ -182,10 +182,10 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 	if err != nil {
 		return ownership, fmt.Errorf("failed to perform permission changes on path %s: %w", topPath, err)
 	}
-	if paths.ShellWrapperPath != "" {
-		err = perms.FixPermissions(paths.ShellWrapperPath, perms.WithOwnership(ownership))
+	if paths.ShellWrapperPath() != "" {
+		err = perms.FixPermissions(paths.ShellWrapperPath(), perms.WithOwnership(ownership))
 		if err != nil {
-			return ownership, fmt.Errorf("failed to perform permission changes on path %s: %w", paths.ShellWrapperPath, err)
+			return ownership, fmt.Errorf("failed to perform permission changes on path %s: %w", paths.ShellWrapperPath(), err)
 		}
 	}
 
diff --git a/internal/pkg/agent/install/svc.go b/internal/pkg/agent/install/svc.go
index dc2e2b6f223..09e755d4220 100644
--- a/internal/pkg/agent/install/svc.go
+++ b/internal/pkg/agent/install/svc.go
@@ -31,8 +31,8 @@ const (
 // ExecutablePath returns the path for the installed Agents executable.
 func ExecutablePath(topPath string) string {
 	exec := filepath.Join(topPath, paths.BinaryName)
-	if paths.ShellWrapperPath != "" {
-		exec = paths.ShellWrapperPath
+	if paths.ShellWrapperPath() != "" {
+		exec = paths.ShellWrapperPath()
 	}
 	return exec
 }
diff --git a/internal/pkg/agent/install/uninstall.go b/internal/pkg/agent/install/uninstall.go
index ebdd9a5c87b..03debd00bdb 100644
--- a/internal/pkg/agent/install/uninstall.go
+++ b/internal/pkg/agent/install/uninstall.go
@@ -117,13 +117,13 @@ func Uninstall(cfgFile, topPath, uninstallToken string, log *logp.Logger, pt *pr
 	}
 
 	// remove, if present on platform
-	if paths.ShellWrapperPath != "" {
-		err = os.Remove(paths.ShellWrapperPath)
+	if paths.ShellWrapperPath() != "" {
+		err = os.Remove(paths.ShellWrapperPath())
 		if !os.IsNotExist(err) && err != nil {
 			return aerrors.New(
 				err,
-				fmt.Sprintf("failed to remove shell wrapper (%s)", paths.ShellWrapperPath),
-				aerrors.M("destination", paths.ShellWrapperPath))
+				fmt.Sprintf("failed to remove shell wrapper (%s)", paths.ShellWrapperPath()),
+				aerrors.M("destination", paths.ShellWrapperPath()))
 		}
 	}
 

From 7ef8e1bf8af27a3a2bf7c5f7beec2aedff22709f Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 27 May 2024 16:38:40 -0400
Subject: [PATCH 04/49] Make control socket run path account for dev mode.

---
 .../pkg/agent/application/paths/common.go     | 12 ++++++++++-
 .../agent/application/paths/paths_darwin.go   |  5 +++--
 .../agent/application/paths/paths_linux.go    |  5 +++--
 .../agent/application/paths/paths_windows.go  |  5 +++--
 internal/pkg/agent/cmd/install.go             |  2 +-
 internal/pkg/agent/cmd/run.go                 | 20 +++++++++----------
 pkg/testing/fixture_install.go                |  2 +-
 7 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common.go b/internal/pkg/agent/application/paths/common.go
index 2b289f0fc5e..4ea9d30de2d 100644
--- a/internal/pkg/agent/application/paths/common.go
+++ b/internal/pkg/agent/application/paths/common.go
@@ -327,6 +327,7 @@ func IsDevelopmentMode() bool {
 	return isDevelopmentMode
 }
 
+// ServiceName returns the service name accounting for development mode.
 func ServiceName() string {
 	if isDevelopmentMode {
 		return serviceNameDevelopmentMode
@@ -334,6 +335,7 @@ func ServiceName() string {
 	return serviceName
 }
 
+// ShellWrapperPath returns the shell wrapper path accounting for development mode.
 func ShellWrapperPath() string {
 	if isDevelopmentMode {
 		return shellWrapperPathDevelopmentMode
@@ -341,7 +343,15 @@ func ShellWrapperPath() string {
 	return shellWrapperPath
 }
 
-// InstallPath returns the top level directory Agent will be installed into.
+// ControlSocketRunSymlink returns the shell wrapper path accounting for development mode.
+func ControlSocketRunSymlink() string {
+	if isDevelopmentMode {
+		return controlSocketRunSymlinkDevelopmentMode
+	}
+	return controlSocketRunSymlink
+}
+
+// InstallPath returns the top level directory Agent will be installed into, accounting for development mode.
 func InstallPath(basePath string) string {
 	elasticPath := filepath.Join(basePath, "Elastic")
 	if isDevelopmentMode {
diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index e69ecb9223c..5d13da28cba 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -14,9 +14,10 @@ const (
 	// for installing Elastic Agent's files.
 	DefaultBasePath = "/Library"
 
-	// ControlSocketRunSymlink is the path to the symlink that should be
+	// controlSocketRunSymlink is the path to the symlink that should be
 	// created to the control socket when Elastic Agent is running with root.
-	ControlSocketRunSymlink = "/var/run/elastic-agent.sock"
+	controlSocketRunSymlink                = "/var/run/elastic-agent.sock"
+	controlSocketRunSymlinkDevelopmentMode = "/var/run/elastic-agent-development.sock"
 
 	// serviceName is the service name when installed.
 	serviceName                = "co.elastic.elastic-agent"
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index db0c6b9a366..6782cd9020c 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -28,9 +28,10 @@ const (
 exec %s/elastic-agent $@
 `
 
-	// ControlSocketRunSymlink is the path to the symlink that should be
+	// controlSocketRunSymlink is the path to the symlink that should be
 	// created to the control socket when Elastic Agent is running with root.
-	ControlSocketRunSymlink = "/run/elastic-agent.sock"
+	controlSocketRunSymlink                = "/run/elastic-agent.sock"
+	controlSocketRunSymlinkDevelopmentMode = "/run/elastic-agent-development.sock"
 )
 
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index 9d76f433ea4..61e99fdfde8 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -20,8 +20,9 @@ const (
 	// for installing Elastic Agent's files.
 	DefaultBasePath = `C:\Program Files`
 
-	// ControlSocketRunSymlink is not created on Windows.
-	ControlSocketRunSymlink = ""
+	// controlSocketRunSymlink is not created on Windows.
+	controlSocketRunSymlink                = ""
+	controlSocketRunSymlinkDevelopmentMode = ""
 
 	// serviceName is the service name when installed.
 	serviceName                = "Elastic Agent"
diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index a80914449b6..5afaf0a2e59 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -50,7 +50,7 @@ would like the Agent to operate.
 	cmd.Flags().String(flagInstallBasePath, paths.DefaultBasePath, "The path where the Elastic Agent will be installed. It must be an absolute path.")
 	cmd.Flags().Bool(flagInstallUnprivileged, false, "Installed Elastic Agent will create an 'elastic-agent' user and run as that user. (experimental)")
 	_ = cmd.Flags().MarkHidden(flagInstallUnprivileged) // Hidden until fully supported
-	cmd.Flags().Bool(flagInstallDevelopment, false, "Install Elastic Agent for development in an isolated base path. (experimental)")
+	cmd.Flags().Bool(flagInstallDevelopment, false, "Install an isolated Elastic Agent for development. Allows a non-development agent to be installed already. (experimental)")
 	_ = cmd.Flags().MarkHidden(flagInstallDevelopment) // For internal use only.
 	addEnrollFlags(cmd)
 
diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go
index de181923cec..3cf1a159123 100644
--- a/internal/pkg/agent/cmd/run.go
+++ b/internal/pkg/agent/cmd/run.go
@@ -307,23 +307,23 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf
 	// this provides backwards compatibility as the control socket was moved with the addition of --unprivileged
 	// option during installation
 	//
-	// Windows `paths.ControlSocketRunSymlink` is `""` so this is always skipped on Windows.
-	if isRoot && paths.RunningInstalled() && paths.ControlSocketRunSymlink != "" {
+	// Windows `paths.ControlSocketRunSymlink()` is `""` so this is always skipped on Windows.
+	if isRoot && paths.RunningInstalled() && paths.ControlSocketRunSymlink() != "" {
 		socketPath := strings.TrimPrefix(paths.ControlSocket(), "unix://")
-		socketLog := controlLog.With("path", socketPath).With("link", paths.ControlSocketRunSymlink)
+		socketLog := controlLog.With("path", socketPath).With("link", paths.ControlSocketRunSymlink())
 		// ensure it doesn't exist before creating the symlink
-		if err := os.Remove(paths.ControlSocketRunSymlink); err != nil && !errors.Is(err, os.ErrNotExist) {
-			socketLog.Errorf("Failed to remove existing control socket symlink %s: %s", paths.ControlSocketRunSymlink, err)
+		if err := os.Remove(paths.ControlSocketRunSymlink()); err != nil && !errors.Is(err, os.ErrNotExist) {
+			socketLog.Errorf("Failed to remove existing control socket symlink %s: %s", paths.ControlSocketRunSymlink(), err)
 		}
-		if err := os.Symlink(socketPath, paths.ControlSocketRunSymlink); err != nil {
-			socketLog.Errorf("Failed to create control socket symlink %s -> %s: %s", paths.ControlSocketRunSymlink, socketPath, err)
+		if err := os.Symlink(socketPath, paths.ControlSocketRunSymlink()); err != nil {
+			socketLog.Errorf("Failed to create control socket symlink %s -> %s: %s", paths.ControlSocketRunSymlink(), socketPath, err)
 		} else {
-			socketLog.Infof("Created control socket symlink %s -> %s; allowing unix://%s connection", paths.ControlSocketRunSymlink, socketPath, paths.ControlSocketRunSymlink)
+			socketLog.Infof("Created control socket symlink %s -> %s; allowing unix://%s connection", paths.ControlSocketRunSymlink(), socketPath, paths.ControlSocketRunSymlink())
 		}
 		defer func() {
 			// delete the symlink on exit; ignore the error
-			if err := os.Remove(paths.ControlSocketRunSymlink); err != nil {
-				socketLog.Errorf("Failed to remove control socket symlink %s: %s", paths.ControlSocketRunSymlink, err)
+			if err := os.Remove(paths.ControlSocketRunSymlink()); err != nil {
+				socketLog.Errorf("Failed to remove control socket symlink %s: %s", paths.ControlSocketRunSymlink(), err)
 			}
 		}()
 	}
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index 05e40cd2b38..a7aed61638b 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -186,7 +186,7 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 	}
 
 	// we just installed agent, the control socket is at a well-known location
-	socketPath := fmt.Sprintf("unix://%s", paths.ControlSocketRunSymlink) // use symlink as that works for all versions
+	socketPath := fmt.Sprintf("unix://%s", paths.ControlSocketRunSymlink()) // use symlink as that works for all versions
 	if runtime.GOOS == "windows" {
 		// Windows uses a fixed named pipe, that is always the same.
 		// It is the same even running in unprivileged mode.

From 7d6f1530f555b1c2a8cc388961d4713a6ace60ff Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 27 May 2024 16:53:28 -0400
Subject: [PATCH 05/49] Development mode automatically binds to port 0.

---
 internal/pkg/agent/configuration/grpc.go | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/agent/configuration/grpc.go b/internal/pkg/agent/configuration/grpc.go
index 652aa04f4f3..6c2e25a5764 100644
--- a/internal/pkg/agent/configuration/grpc.go
+++ b/internal/pkg/agent/configuration/grpc.go
@@ -4,7 +4,11 @@
 
 package configuration
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
+)
 
 // GRPCConfig is a configuration of GRPC server.
 type GRPCConfig struct {
@@ -16,9 +20,17 @@ type GRPCConfig struct {
 
 // DefaultGRPCConfig creates a default server configuration.
 func DefaultGRPCConfig() *GRPCConfig {
+	// In development mode bind to port zero to select a random free port to avoid collisions with
+	// any already installed Elastic Agent. Ideally we'd always bind to port zero, but this would be
+	// breaking for users that had to manually whitelist the gRPC port in local firewall rules.
+	defaultPort := uint16(6789)
+	if paths.IsDevelopmentMode() {
+		defaultPort = 0
+	}
+
 	return &GRPCConfig{
 		Address:                 "localhost",
-		Port:                    6789,
+		Port:                    defaultPort,
 		MaxMsgSize:              1024 * 1024 * 100, // grpc default 4MB is unsufficient for diagnostics
 		CheckinChunkingDisabled: false,             // on by default
 	}

From 06c1335edf74e59b7a14de018fb6c43ee9b87a09 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 28 May 2024 14:38:44 -0400
Subject: [PATCH 06/49] Refactor dev mode into separate file.

Auto-detect when we were installed in development mode at runtime.
---
 .../pkg/agent/application/paths/common.go     | 44 -------------
 .../paths/common_development_mode.go          | 63 +++++++++++++++++++
 2 files changed, 63 insertions(+), 44 deletions(-)
 create mode 100644 internal/pkg/agent/application/paths/common_development_mode.go

diff --git a/internal/pkg/agent/application/paths/common.go b/internal/pkg/agent/application/paths/common.go
index 4ea9d30de2d..eac97c1c94b 100644
--- a/internal/pkg/agent/application/paths/common.go
+++ b/internal/pkg/agent/application/paths/common.go
@@ -61,7 +61,6 @@ var (
 	installPath       string
 	controlSocketPath string
 	unversionedHome   bool
-	isDevelopmentMode bool
 	tmpCreator        sync.Once
 )
 
@@ -317,49 +316,6 @@ func BinaryPath(baseDir, agentName string) string {
 	return filepath.Join(binaryDir(baseDir), agentName)
 }
 
-// SetIsDevelopmentMode sets whether the agent is installed in development mode or not.
-func SetIsDevelopmentMode(developmentMode bool) {
-	isDevelopmentMode = developmentMode
-}
-
-// IsDevelopmentMode returns true if the agent is installed in development mode.
-func IsDevelopmentMode() bool {
-	return isDevelopmentMode
-}
-
-// ServiceName returns the service name accounting for development mode.
-func ServiceName() string {
-	if isDevelopmentMode {
-		return serviceNameDevelopmentMode
-	}
-	return serviceName
-}
-
-// ShellWrapperPath returns the shell wrapper path accounting for development mode.
-func ShellWrapperPath() string {
-	if isDevelopmentMode {
-		return shellWrapperPathDevelopmentMode
-	}
-	return shellWrapperPath
-}
-
-// ControlSocketRunSymlink returns the shell wrapper path accounting for development mode.
-func ControlSocketRunSymlink() string {
-	if isDevelopmentMode {
-		return controlSocketRunSymlinkDevelopmentMode
-	}
-	return controlSocketRunSymlink
-}
-
-// InstallPath returns the top level directory Agent will be installed into, accounting for development mode.
-func InstallPath(basePath string) string {
-	elasticPath := filepath.Join(basePath, "Elastic")
-	if isDevelopmentMode {
-		return filepath.Join(elasticPath, "DevelopmentAgent")
-	}
-	return filepath.Join(elasticPath, "Agent")
-}
-
 // TopBinaryPath returns the path to the Elastic Agent binary that is inside the Top directory.
 //
 // This always points to the symlink that points to the latest Elastic Agent version.
diff --git a/internal/pkg/agent/application/paths/common_development_mode.go b/internal/pkg/agent/application/paths/common_development_mode.go
new file mode 100644
index 00000000000..16cb2f8ccca
--- /dev/null
+++ b/internal/pkg/agent/application/paths/common_development_mode.go
@@ -0,0 +1,63 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package paths
+
+import "path/filepath"
+
+const developmentInstallPathSuffix string = "DevelopmentAgent"
+
+var isDevelopmentMode bool
+
+// SetIsDevelopmentMode sets whether the agent is installed in development mode or not.
+func SetIsDevelopmentMode(developmentMode bool) {
+	isDevelopmentMode = developmentMode
+}
+
+// IsDevelopmentMode returns true if the agent is installed in development mode.
+func IsDevelopmentMode() bool {
+	// The current process has explicitly been told it is in development mode.
+	if isDevelopmentMode {
+		return true
+	}
+
+	// We are installed in development mode and have to infer it from the path.
+	if RunningInstalled() && filepath.Base(Top()) == developmentInstallPathSuffix {
+		return true
+	}
+
+	return false
+}
+
+// InstallPath returns the top level directory Agent will be installed into, accounting for development mode.
+func InstallPath(basePath string) string {
+	if IsDevelopmentMode() {
+		return filepath.Join(basePath, "Elastic", developmentInstallPathSuffix)
+	}
+	return filepath.Join(basePath, "Elastic", "Agent")
+}
+
+// ServiceName returns the service name accounting for development mode.
+func ServiceName() string {
+	if IsDevelopmentMode() {
+		return serviceNameDevelopmentMode
+	}
+	return serviceName
+}
+
+// ShellWrapperPath returns the shell wrapper path accounting for development mode.
+func ShellWrapperPath() string {
+	if IsDevelopmentMode() {
+		return shellWrapperPathDevelopmentMode
+	}
+	return shellWrapperPath
+}
+
+// ControlSocketRunSymlink returns the shell wrapper path accounting for development mode.
+func ControlSocketRunSymlink() string {
+	if IsDevelopmentMode() {
+		return controlSocketRunSymlinkDevelopmentMode
+	}
+	return controlSocketRunSymlink
+}

From 8e1c177634aa6cfd17e882cdbb37d1528200dfaa Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 28 May 2024 15:09:32 -0400
Subject: [PATCH 07/49] Shorten command to --develop.

---
 internal/pkg/agent/cmd/install.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index 5afaf0a2e59..38fd3a70a0f 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -25,7 +25,7 @@ import (
 const (
 	flagInstallBasePath     = "base-path"
 	flagInstallUnprivileged = "unprivileged"
-	flagInstallDevelopment  = "development"
+	flagInstallDevelopment  = "develop"
 )
 
 func newInstallCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {

From f20c90921f3fd27aed93a0956a9a8c23c93606fe Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 28 May 2024 15:42:33 -0400
Subject: [PATCH 08/49] Fix windows build error.

---
 internal/pkg/agent/install/install_windows.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/install/install_windows.go b/internal/pkg/agent/install/install_windows.go
index 08a729eeace..ee2e5d8df18 100644
--- a/internal/pkg/agent/install/install_windows.go
+++ b/internal/pkg/agent/install/install_windows.go
@@ -102,7 +102,7 @@ func servicePostInstall(ownership utils.FileOwner) error {
 	}
 	err = windows.SetNamedSecurityInfo(paths.ServiceName(), windows.SE_SERVICE, windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)
 	if err != nil {
-		return fmt.Errorf("failed to set DACL for service(%s): %w", paths.ServiceName, err)
+		return fmt.Errorf("failed to set DACL for service(%s): %w", paths.ServiceName(), err)
 	}
 	return nil
 }

From 39bb905d90f47687457a72a0f9ee59f3b78f2cab Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 28 May 2024 15:42:44 -0400
Subject: [PATCH 09/49] Fix using wrong config option in install.

---
 internal/pkg/agent/cmd/install.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index 38fd3a70a0f..42a41fa8fc9 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -84,7 +84,7 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 	}
 
 	isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
-	if unprivileged {
+	if isDevelopmentMode {
 		fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental feature.")
 	}
 	paths.SetIsDevelopmentMode(isDevelopmentMode)

From 16ba1408e333a9c1ecc69578129510d4a7089a2b Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 29 May 2024 13:39:08 -0400
Subject: [PATCH 10/49] Add run --develop command.

---
 internal/pkg/agent/cmd/run.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go
index 3cf1a159123..778305b5f50 100644
--- a/internal/pkg/agent/cmd/run.go
+++ b/internal/pkg/agent/cmd/run.go
@@ -58,6 +58,7 @@ import (
 const (
 	agentName            = "elastic-agent"
 	fleetInitTimeoutName = "FLEET_SERVER_INIT_TIMEOUT"
+	flagRunDevelopment   = "develop"
 )
 
 type cfgOverrider func(cfg *configuration.Configuration)
@@ -69,9 +70,15 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 		Short: "Start the Elastic Agent",
 		Long:  "This command starts the Elastic Agent.",
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			// done very early so the encrypted store is never used
+			isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
+			if isDevelopmentMode {
+				fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental feature.")
+			}
+			paths.SetIsDevelopmentMode(isDevelopmentMode)
+
+			// done very early so the encrypted store is never used. Always done in development mode to remove the need to be root.
 			disableEncryptedStore, _ := cmd.Flags().GetBool("disable-encrypted-store")
-			if disableEncryptedStore {
+			if disableEncryptedStore || isDevelopmentMode {
 				storage.DisableEncryptionDarwin()
 			}
 			fleetInitTimeout, _ := cmd.Flags().GetDuration("fleet-init-timeout")
@@ -102,6 +109,9 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 	cmd.Flags().Duration("fleet-init-timeout", envTimeout(fleetInitTimeoutName), " Sets the initial timeout when starting up the fleet server under agent")
 	_ = cmd.Flags().MarkHidden("testing-mode")
 
+	cmd.Flags().Bool(flagRunDevelopment, false, "Run agent in development mode. Allows running when there is already and installed Elastic Agent. (experimental)")
+	_ = cmd.Flags().MarkHidden(flagRunDevelopment) // For internal use only.
+
 	return cmd
 }
 

From 1f3ee133b39054606e267f920185513aa2d28c37 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 29 May 2024 15:22:16 -0400
Subject: [PATCH 11/49] Add initial test for --develop.

---
 internal/pkg/agent/cmd/install.go             |  2 +-
 pkg/testing/fixture_install.go                |  4 +++
 testing/installtest/checks.go                 | 21 ++++++++++++---
 testing/installtest/checks_unix.go            | 13 ++++++---
 testing/installtest/checks_windows.go         |  4 +--
 .../integration/install_privileged_test.go    |  4 +--
 testing/integration/install_test.go           | 27 ++++++++++++++++---
 testing/integration/logs_ingestion_test.go    |  2 +-
 testing/upgradetest/upgrader.go               |  4 +--
 9 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index 42a41fa8fc9..9d895c99bc2 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -85,7 +85,7 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
 	if isDevelopmentMode {
-		fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental feature.")
+		fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental and currently unsupported feature.")
 	}
 	paths.SetIsDevelopmentMode(isDevelopmentMode)
 
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index a7aed61638b..7194640c57e 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -88,6 +88,7 @@ type InstallOpts struct {
 	NonInteractive bool   // --non-interactive
 	ProxyURL       string // --proxy-url
 	DelayEnroll    bool   // --delay-enroll
+	Develop        bool   // --develop, not supported for DEB and RPM.
 
 	Privileged bool // inverse of --unprivileged (as false is the default)
 
@@ -118,6 +119,9 @@ func (i InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
 	if !i.Privileged {
 		args = append(args, "--unprivileged")
 	}
+	if !i.Develop {
+		args = append(args, "--develop")
+	}
 
 	args = append(args, i.EnrollOpts.toCmdArgs()...)
 	args = append(args, i.FleetBootstrapOpts.toCmdArgs()...)
diff --git a/testing/installtest/checks.go b/testing/installtest/checks.go
index 66db06b4c19..d2183b44108 100644
--- a/testing/installtest/checks.go
+++ b/testing/installtest/checks.go
@@ -15,7 +15,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 )
 
-func DefaultTopPath() string {
+func defaultBasePath() string {
 	var defaultBasePath string
 	switch runtime.GOOS {
 	case "darwin":
@@ -25,10 +25,23 @@ func DefaultTopPath() string {
 	case "windows":
 		defaultBasePath = `C:\Program Files`
 	}
-	return filepath.Join(defaultBasePath, "Elastic", "Agent")
+	return defaultBasePath
+}
+
+func DefaultTopPath() string {
+	return filepath.Join(defaultBasePath(), "Elastic", "Agent")
+}
+
+func DevelopTopPath() string {
+	return filepath.Join(defaultBasePath(), "Elastic", "DevelopmentAgent")
+}
+
+type CheckOpts struct {
+	Unprivileged bool
+	Develop      bool
 }
 
-func CheckSuccess(ctx context.Context, f *atesting.Fixture, topPath string, unprivileged bool) error {
+func CheckSuccess(ctx context.Context, f *atesting.Fixture, topPath string, opts *CheckOpts) error {
 	// Use default topPath if one not defined.
 	if topPath == "" {
 		topPath = DefaultTopPath()
@@ -58,7 +71,7 @@ func CheckSuccess(ctx context.Context, f *atesting.Fixture, topPath string, unpr
 	}
 
 	// Specific checks depending on the platform.
-	return checkPlatform(ctx, f, topPath, unprivileged)
+	return checkPlatform(ctx, f, topPath, opts)
 }
 
 func exeOnWindows(filename string) string {
diff --git a/testing/installtest/checks_unix.go b/testing/installtest/checks_unix.go
index 841f3e97388..e4230df6c75 100644
--- a/testing/installtest/checks_unix.go
+++ b/testing/installtest/checks_unix.go
@@ -20,8 +20,8 @@ import (
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 )
 
-func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, unprivileged bool) error {
-	if unprivileged {
+func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, opts *CheckOpts) error {
+	if opts.Unprivileged {
 		// Check that the elastic-agent user/group exist.
 		uid, err := install.FindUID(install.ElasticUsername)
 		if err != nil {
@@ -64,10 +64,15 @@ func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, unp
 		}
 
 		// Executing `elastic-agent status` as the `elastic-agent-user` user should work.
+		shellWrapperName := "elastic-agent"
+		if opts.Develop {
+			shellWrapperName = "elastic-dev-agent"
+		}
+
 		var output []byte
 		err = waitForNoError(ctx, func(_ context.Context) error {
 			// #nosec G204 -- user cannot inject any parameters to this command
-			cmd := exec.Command("sudo", "-u", install.ElasticUsername, "elastic-agent", "status")
+			cmd := exec.Command("sudo", "-u", install.ElasticUsername, shellWrapperName, "status")
 			output, err = cmd.CombinedOutput()
 			if err != nil {
 				return fmt.Errorf("elastic-agent status failed: %w (output: %s)", err, output)
@@ -80,7 +85,7 @@ func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, unp
 		originalUser := os.Getenv("SUDO_USER")
 		if originalUser != "" {
 			// #nosec G204 -- user cannot inject any parameters to this command
-			cmd := exec.Command("sudo", "-u", originalUser, "elastic-agent", "status")
+			cmd := exec.Command("sudo", "-u", originalUser, shellWrapperName, "status")
 			output, err := cmd.CombinedOutput()
 			if err == nil {
 				return fmt.Errorf("sudo -u %s elastic-agent didn't fail: got output: %s", originalUser, output)
diff --git a/testing/installtest/checks_windows.go b/testing/installtest/checks_windows.go
index d16fbdf19c2..4a4dd0efc10 100644
--- a/testing/installtest/checks_windows.go
+++ b/testing/installtest/checks_windows.go
@@ -36,7 +36,7 @@ type accessAllowedAce struct {
 	SidStart   uint32
 }
 
-func checkPlatform(ctx context.Context, f *atesting.Fixture, topPath string, unprivileged bool) error {
+func checkPlatform(ctx context.Context, f *atesting.Fixture, topPath string, opts *CheckOpts) error {
 	secInfo, err := windows.GetNamedSecurityInfo(topPath, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION)
 	if err != nil {
 		return fmt.Errorf("GetNamedSecurityInfo failed for %s: %w", topPath, err)
@@ -52,7 +52,7 @@ func checkPlatform(ctx context.Context, f *atesting.Fixture, topPath string, unp
 	if err != nil {
 		return fmt.Errorf("failed to get allowed SID's for %s: %w", topPath, err)
 	}
-	if unprivileged {
+	if opts.Unprivileged {
 		// Check that the elastic-agent user/group exist.
 		uid, err := install.FindUID(install.ElasticUsername)
 		if err != nil {
diff --git a/testing/integration/install_privileged_test.go b/testing/integration/install_privileged_test.go
index 8b4f64a8da7..c4773c63aa7 100644
--- a/testing/integration/install_privileged_test.go
+++ b/testing/integration/install_privileged_test.go
@@ -54,7 +54,7 @@ func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
 	}
 
 	// Check that Agent was installed in default base path
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, false))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: false}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 }
 
@@ -101,6 +101,6 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, false))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: false}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 }
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index 8fc9df9d6fa..63bdd5f91e8 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -60,8 +60,28 @@ func TestInstallWithoutBasePath(t *testing.T) {
 
 	// Check that Agent was installed in default base path
 	topPath := installtest.DefaultTopPath()
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, true))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: true}))
+
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+
+	t.Run("check second agent installs with --develop", func(t *testing.T) {
+		// Get path to Elastic Agent executable
+		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+		require.NoError(t, err)
+
+		// Prepare the Elastic Agent so the binary is extracted and ready to use.
+		err = devFixture.Prepare(ctx)
+		require.NoError(t, err)
+
+		devOpts := &atesting.InstallOpts{Force: true, Privileged: false, Develop: true}
+		devOut, err := devFixture.Install(ctx, devOpts)
+		if err != nil {
+			t.Logf("install --develop output: %s", devOut)
+			require.NoError(t, err)
+		}
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: false, Develop: true}))
+	})
+
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
 		cwd, err := os.Getwd()
@@ -75,7 +95,6 @@ func TestInstallWithoutBasePath(t *testing.T) {
 		require.Error(t, err, "uninstall should have failed")
 		require.Containsf(t, string(out), "uninstall must be run from outside the installed path", "expected error string not found in: %s err: %s", out, err)
 	}
-
 }
 
 func TestInstallWithBasePath(t *testing.T) {
@@ -136,7 +155,7 @@ func TestInstallWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, true))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: true}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
@@ -196,7 +215,7 @@ func TestRepeatedInstallUninstall(t *testing.T) {
 			}
 
 			// Check that Agent was installed in default base path
-			require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, !opts.Privileged))
+			require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
 			t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 			out, err = fixture.Uninstall(ctx, &atesting.UninstallOpts{Force: true})
 			require.NoErrorf(t, err, "uninstall failed: %s", err)
diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go
index bf794dd1143..0b036a4e64c 100644
--- a/testing/integration/logs_ingestion_test.go
+++ b/testing/integration/logs_ingestion_test.go
@@ -93,7 +93,7 @@ func TestLogIngestionFleetManaged(t *testing.T) {
 	check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute)
 
 	// 3. Ensure installation is correct.
-	require.NoError(t, installtest.CheckSuccess(ctx, agentFixture, installOpts.BasePath, !installOpts.Privileged))
+	require.NoError(t, installtest.CheckSuccess(ctx, agentFixture, installOpts.BasePath, &installtest.CheckOpts{Unprivileged: !installOpts.Privileged}))
 
 	t.Run("Monitoring logs are shipped", func(t *testing.T) {
 		testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
diff --git a/testing/upgradetest/upgrader.go b/testing/upgradetest/upgrader.go
index 12996ebd7fa..54e1c517883 100644
--- a/testing/upgradetest/upgrader.go
+++ b/testing/upgradetest/upgrader.go
@@ -281,7 +281,7 @@ func PerformUpgrade(
 
 	// validate installation is correct
 	if InstallChecksAllowed(!installOpts.Privileged, startVersion) {
-		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, !installOpts.Privileged)
+		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, &installtest.CheckOpts{Unprivileged: !installOpts.Privileged})
 		if err != nil {
 			return fmt.Errorf("pre-upgrade installation checks failed: %w", err)
 		}
@@ -412,7 +412,7 @@ func PerformUpgrade(
 
 	// validate again that the installation is correct, upgrade should not have changed installation validation
 	if InstallChecksAllowed(!installOpts.Privileged, startVersion, endVersion) {
-		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, !installOpts.Privileged)
+		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, &installtest.CheckOpts{Unprivileged: !installOpts.Privileged})
 		if err != nil {
 			return fmt.Errorf("post-upgrade installation checks failed: %w", err)
 		}

From 1e389158e7b16851f0b9592177525f618ffe6b4c Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 29 May 2024 17:25:59 -0400
Subject: [PATCH 12/49] Initial version of an integration test for --develop
 mode.

---
 .../paths/common_development_mode.go          |  4 +-
 internal/pkg/agent/cmd/run.go                 | 19 ++++-----
 pkg/testing/fixture.go                        |  9 ++++-
 pkg/testing/fixture_install.go                | 40 +++++++++++++++----
 testing/integration/install_test.go           |  2 +-
 5 files changed, 53 insertions(+), 21 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_development_mode.go b/internal/pkg/agent/application/paths/common_development_mode.go
index 16cb2f8ccca..3f39ac89b1b 100644
--- a/internal/pkg/agent/application/paths/common_development_mode.go
+++ b/internal/pkg/agent/application/paths/common_development_mode.go
@@ -55,8 +55,8 @@ func ShellWrapperPath() string {
 }
 
 // ControlSocketRunSymlink returns the shell wrapper path accounting for development mode.
-func ControlSocketRunSymlink() string {
-	if IsDevelopmentMode() {
+func ControlSocketRunSymlink(isDevelopmentMode bool) string {
+	if isDevelopmentMode {
 		return controlSocketRunSymlinkDevelopmentMode
 	}
 	return controlSocketRunSymlink
diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go
index 778305b5f50..4f7686ec5ef 100644
--- a/internal/pkg/agent/cmd/run.go
+++ b/internal/pkg/agent/cmd/run.go
@@ -318,22 +318,23 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf
 	// option during installation
 	//
 	// Windows `paths.ControlSocketRunSymlink()` is `""` so this is always skipped on Windows.
-	if isRoot && paths.RunningInstalled() && paths.ControlSocketRunSymlink() != "" {
+	controlSocketRunSymlink := paths.ControlSocketRunSymlink(paths.IsDevelopmentMode())
+	if isRoot && paths.RunningInstalled() && controlSocketRunSymlink != "" {
 		socketPath := strings.TrimPrefix(paths.ControlSocket(), "unix://")
-		socketLog := controlLog.With("path", socketPath).With("link", paths.ControlSocketRunSymlink())
+		socketLog := controlLog.With("path", socketPath).With("link", controlSocketRunSymlink)
 		// ensure it doesn't exist before creating the symlink
-		if err := os.Remove(paths.ControlSocketRunSymlink()); err != nil && !errors.Is(err, os.ErrNotExist) {
-			socketLog.Errorf("Failed to remove existing control socket symlink %s: %s", paths.ControlSocketRunSymlink(), err)
+		if err := os.Remove(controlSocketRunSymlink); err != nil && !errors.Is(err, os.ErrNotExist) {
+			socketLog.Errorf("Failed to remove existing control socket symlink %s: %s", controlSocketRunSymlink, err)
 		}
-		if err := os.Symlink(socketPath, paths.ControlSocketRunSymlink()); err != nil {
-			socketLog.Errorf("Failed to create control socket symlink %s -> %s: %s", paths.ControlSocketRunSymlink(), socketPath, err)
+		if err := os.Symlink(socketPath, controlSocketRunSymlink); err != nil {
+			socketLog.Errorf("Failed to create control socket symlink %s -> %s: %s", controlSocketRunSymlink, socketPath, err)
 		} else {
-			socketLog.Infof("Created control socket symlink %s -> %s; allowing unix://%s connection", paths.ControlSocketRunSymlink(), socketPath, paths.ControlSocketRunSymlink())
+			socketLog.Infof("Created control socket symlink %s -> %s; allowing unix://%s connection", controlSocketRunSymlink, socketPath, controlSocketRunSymlink)
 		}
 		defer func() {
 			// delete the symlink on exit; ignore the error
-			if err := os.Remove(paths.ControlSocketRunSymlink()); err != nil {
-				socketLog.Errorf("Failed to remove control socket symlink %s: %s", paths.ControlSocketRunSymlink(), err)
+			if err := os.Remove(controlSocketRunSymlink); err != nil {
+				socketLog.Errorf("Failed to remove control socket symlink %s: %s", controlSocketRunSymlink, err)
 			}
 		}()
 	}
diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go
index 3472fcdc663..74dfe0a9c70 100644
--- a/pkg/testing/fixture.go
+++ b/pkg/testing/fixture.go
@@ -831,10 +831,15 @@ func (f *Fixture) EnsurePrepared(ctx context.Context) error {
 func (f *Fixture) binaryPath() string {
 	workDir := f.workDir
 	if f.installed {
+		installDir := "Agent"
+		if f.installOpts != nil && f.installOpts.Develop {
+			installDir = "DevelopmentAgent"
+		}
+
 		if f.installOpts != nil && f.installOpts.BasePath != "" {
-			workDir = filepath.Join(f.installOpts.BasePath, "Elastic", "Agent")
+			workDir = filepath.Join(f.installOpts.BasePath, "Elastic", installDir)
 		} else {
-			workDir = filepath.Join(paths.DefaultBasePath, "Elastic", "Agent")
+			workDir = filepath.Join(paths.DefaultBasePath, "Elastic", installDir)
 		}
 	}
 	if f.packageFormat == "deb" || f.packageFormat == "rpm" {
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index 7194640c57e..b583644e199 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -14,6 +14,7 @@ import (
 	"io/fs"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -119,7 +120,7 @@ func (i InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
 	if !i.Privileged {
 		args = append(args, "--unprivileged")
 	}
-	if !i.Develop {
+	if i.Develop {
 		args = append(args, "--develop")
 	}
 
@@ -139,8 +140,10 @@ func (i InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
 func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ...process.CmdOption) ([]byte, error) {
 	f.t.Logf("[test %s] Inside fixture install function", f.t.Name())
 
-	// check for running agents before installing, but proceed anyway
-	assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at beginning of Install()")
+	// check for running agents before installing, but only if not using --develop whose point is allowing two agents at once.
+	if installOpts != nil && !installOpts.Develop {
+		assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at beginning of Install()")
+	}
 
 	switch f.packageFormat {
 	case "targz", "zip":
@@ -183,14 +186,19 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 	f.installed = true
 	f.installOpts = installOpts
 
+	installDir := "Agent"
+	if installOpts.Develop {
+		installDir = "DevelopmentAgent"
+	}
+
 	if installOpts.BasePath == "" {
-		f.workDir = filepath.Join(paths.DefaultBasePath, "Elastic", "Agent")
+		f.workDir = filepath.Join(paths.DefaultBasePath, "Elastic", installDir)
 	} else {
-		f.workDir = filepath.Join(installOpts.BasePath, "Elastic", "Agent")
+		f.workDir = filepath.Join(installOpts.BasePath, "Elastic", installDir)
 	}
 
 	// we just installed agent, the control socket is at a well-known location
-	socketPath := fmt.Sprintf("unix://%s", paths.ControlSocketRunSymlink()) // use symlink as that works for all versions
+	socketPath := fmt.Sprintf("unix://%s", paths.ControlSocketRunSymlink(installOpts.Develop)) // use symlink as that works for all versions
 	if runtime.GOOS == "windows" {
 		// Windows uses a fixed named pipe, that is always the same.
 		// It is the same even running in unprivileged mode.
@@ -216,6 +224,12 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 			sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-")
 
 			filePath := filepath.Join(dir, "build", "diagnostics", fmt.Sprintf("TEST-%s-%s-%s-ProcessDump.json", sanitizedTestName, f.operatingSystem, f.architecture))
+			fileDir := path.Dir(filePath)
+			if err := os.MkdirAll(fileDir, 0777); err != nil {
+				f.t.Logf("failed to dump process; failed to create directory %s: %s", fileDir, err)
+				return
+			}
+
 			f.t.Logf("Dumping running processes in %s", filePath)
 			file, err := os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
 			if err != nil {
@@ -237,7 +251,19 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 
 	f.t.Cleanup(func() {
 		// check for running agents after uninstall had a chance to run
-		assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at the end of the test")
+		processes := getElasticAgentProcesses(f.t)
+
+		// there can be a single agent left when using --develop mode
+		if f.installOpts != nil && f.installOpts.Develop {
+			assert.LessOrEqual(f.t, len(processes), 1, "More than one agent left running at the end of the test when --develop was used: %v", processes)
+			// However as a convention the agent left running has to be the non-development agent. The development agent should be uninstalled first as a convention.
+			if len(processes) > 0 {
+				assert.NotContains(f.t, processes[0].Cmdline, "DevelopmentAgent", "The agent installed with --develop was left running at the end of the test or was not uninstalled first: %v", processes)
+			}
+			return
+		}
+
+		assert.Empty(f.t, processes, "there should be no running agent at the end of the test")
 	})
 
 	f.t.Cleanup(func() {
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index 63bdd5f91e8..7fb0c6614e5 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -79,7 +79,7 @@ func TestInstallWithoutBasePath(t *testing.T) {
 			t.Logf("install --develop output: %s", devOut)
 			require.NoError(t, err)
 		}
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: false, Develop: true}))
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: true, Develop: true}))
 	})
 
 	// Make sure uninstall from within the topPath fails on Windows

From 347dc6568effadb2d2e4998d577a1dd01b3ad5c2 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 10:36:26 -0400
Subject: [PATCH 13/49] Wait on the watcher instead of just releasing it.

---
 .../pkg/agent/application/upgrade/rollback.go    | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/internal/pkg/agent/application/upgrade/rollback.go b/internal/pkg/agent/application/upgrade/rollback.go
index 37b2f414717..71fdafaf24b 100644
--- a/internal/pkg/agent/application/upgrade/rollback.go
+++ b/internal/pkg/agent/application/upgrade/rollback.go
@@ -146,13 +146,6 @@ func InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error
 	}
 
 	cmd := invokeCmd(agentExecutable)
-	defer func() {
-		if cmd.Process != nil {
-			log.Infof("releasing watcher %v", cmd.Process.Pid)
-			_ = cmd.Process.Release()
-		}
-	}()
-
 	log.Infow("Starting upgrade watcher", "path", cmd.Path, "args", cmd.Args, "env", cmd.Env, "dir", cmd.Dir)
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("failed to start Upgrade Watcher: %w", err)
@@ -160,6 +153,15 @@ func InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error
 
 	upgradeWatcherPID := cmd.Process.Pid
 	agentPID := os.Getpid()
+
+	go func() {
+		// TODO: This should probably be associated with a context?
+		// TODO: What happens if the agent is sent SIGKILL?
+		if err := cmd.Wait(); err != nil {
+			log.Infow("Upgrade Watcher exited with error", "agent.upgrade.watcher.process.pid", "agent.process.pid", agentPID, upgradeWatcherPID, "error.message", err)
+		}
+	}()
+
 	log.Infow("Upgrade Watcher invoked", "agent.upgrade.watcher.process.pid", upgradeWatcherPID, "agent.process.pid", agentPID)
 
 	return cmd, nil

From 959725a19ab0ca64e1883af0200c1d7281ed47ac Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 13:24:21 -0400
Subject: [PATCH 14/49] Add --develop test with base path.

---
 .../paths/common_development_mode.go          |  8 ++++---
 pkg/testing/fixture.go                        |  2 +-
 pkg/testing/fixture_install.go                |  7 +++---
 testing/installtest/checks.go                 |  3 ++-
 testing/integration/install_test.go           | 23 +++++++++++++++++++
 5 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_development_mode.go b/internal/pkg/agent/application/paths/common_development_mode.go
index 3f39ac89b1b..c022c70aa0d 100644
--- a/internal/pkg/agent/application/paths/common_development_mode.go
+++ b/internal/pkg/agent/application/paths/common_development_mode.go
@@ -6,7 +6,9 @@ package paths
 
 import "path/filepath"
 
-const developmentInstallPathSuffix string = "DevelopmentAgent"
+// DevelopmentInstallDirName is the name of the directory agent will be installed to within the base path.
+// For example it is $BasePath/$DevelopmentInstallDirName, on MacOS it is /Library/Elastic/$DevelopmentInstallDirName.
+const DevelopmentInstallDirName string = "DevelopmentAgent"
 
 var isDevelopmentMode bool
 
@@ -23,7 +25,7 @@ func IsDevelopmentMode() bool {
 	}
 
 	// We are installed in development mode and have to infer it from the path.
-	if RunningInstalled() && filepath.Base(Top()) == developmentInstallPathSuffix {
+	if RunningInstalled() && filepath.Base(Top()) == DevelopmentInstallDirName {
 		return true
 	}
 
@@ -33,7 +35,7 @@ func IsDevelopmentMode() bool {
 // InstallPath returns the top level directory Agent will be installed into, accounting for development mode.
 func InstallPath(basePath string) string {
 	if IsDevelopmentMode() {
-		return filepath.Join(basePath, "Elastic", developmentInstallPathSuffix)
+		return filepath.Join(basePath, "Elastic", DevelopmentInstallDirName)
 	}
 	return filepath.Join(basePath, "Elastic", "Agent")
 }
diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go
index 74dfe0a9c70..5b3afe750a5 100644
--- a/pkg/testing/fixture.go
+++ b/pkg/testing/fixture.go
@@ -833,7 +833,7 @@ func (f *Fixture) binaryPath() string {
 	if f.installed {
 		installDir := "Agent"
 		if f.installOpts != nil && f.installOpts.Develop {
-			installDir = "DevelopmentAgent"
+			installDir = paths.DevelopmentInstallDirName
 		}
 
 		if f.installOpts != nil && f.installOpts.BasePath != "" {
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index b583644e199..df7bf877cab 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -188,7 +188,7 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 
 	installDir := "Agent"
 	if installOpts.Develop {
-		installDir = "DevelopmentAgent"
+		installDir = paths.DevelopmentInstallDirName
 	}
 
 	if installOpts.BasePath == "" {
@@ -256,9 +256,10 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 		// there can be a single agent left when using --develop mode
 		if f.installOpts != nil && f.installOpts.Develop {
 			assert.LessOrEqual(f.t, len(processes), 1, "More than one agent left running at the end of the test when --develop was used: %v", processes)
-			// However as a convention the agent left running has to be the non-development agent. The development agent should be uninstalled first as a convention.
+			// The agent left running has to be the non-development agent. The development agent should be uninstalled first as a convention.
 			if len(processes) > 0 {
-				assert.NotContains(f.t, processes[0].Cmdline, "DevelopmentAgent", "The agent installed with --develop was left running at the end of the test or was not uninstalled first: %v", processes)
+				assert.NotContains(f.t, processes[0].Cmdline, paths.DevelopmentInstallDirName,
+					"The agent installed with --develop was left running at the end of the test or was not uninstalled first: %v", processes)
 			}
 			return
 		}
diff --git a/testing/installtest/checks.go b/testing/installtest/checks.go
index d2183b44108..27e1202cd66 100644
--- a/testing/installtest/checks.go
+++ b/testing/installtest/checks.go
@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"runtime"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 )
@@ -33,7 +34,7 @@ func DefaultTopPath() string {
 }
 
 func DevelopTopPath() string {
-	return filepath.Join(defaultBasePath(), "Elastic", "DevelopmentAgent")
+	return filepath.Join(defaultBasePath(), "Elastic", paths.DevelopmentInstallDirName)
 }
 
 type CheckOpts struct {
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index 7fb0c6614e5..7ef36f4aef5 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -19,6 +19,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
@@ -156,7 +157,29 @@ func TestInstallWithBasePath(t *testing.T) {
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: true}))
+
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+
+	t.Run("check second agent installs with --develop", func(t *testing.T) {
+		// Get path to Elastic Agent executable
+		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+		require.NoError(t, err)
+
+		// Prepare the Elastic Agent so the binary is extracted and ready to use.
+		err = devFixture.Prepare(ctx)
+		require.NoError(t, err)
+
+		devOpts := &atesting.InstallOpts{Force: true, Privileged: false, Develop: true}
+		devOut, err := devFixture.Install(ctx, devOpts)
+		if err != nil {
+			t.Logf("install --develop output: %s", devOut)
+			require.NoError(t, err)
+		}
+
+		devTopPath := filepath.Join(basePath, "Elastic", paths.DevelopmentInstallDirName)
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, devTopPath, &installtest.CheckOpts{Unprivileged: true, Develop: true}))
+	})
+
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
 		cwd, err := os.Getwd()

From f8b1331f2d852c72594a8389cf29539b006048eb Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 14:01:49 -0400
Subject: [PATCH 15/49] Add privileged install tests with --develop

---
 .../integration/install_privileged_test.go    | 40 +++++++++++++++++++
 testing/integration/install_test.go           |  2 +-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/testing/integration/install_privileged_test.go b/testing/integration/install_privileged_test.go
index c4773c63aa7..1f6959ebeb0 100644
--- a/testing/integration/install_privileged_test.go
+++ b/testing/integration/install_privileged_test.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
@@ -55,7 +56,26 @@ func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
 
 	// Check that Agent was installed in default base path
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: false}))
+
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+
+	t.Run("check second agent installs with --develop", func(t *testing.T) {
+		// Get path to Elastic Agent executable
+		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+		require.NoError(t, err)
+
+		// Prepare the Elastic Agent so the binary is extracted and ready to use.
+		err = devFixture.Prepare(ctx)
+		require.NoError(t, err)
+
+		devOpts := &atesting.InstallOpts{Force: true, Privileged: true, Develop: true}
+		devOut, err := devFixture.Install(ctx, devOpts)
+		if err != nil {
+			t.Logf("install --develop output: %s", devOut)
+			require.NoError(t, err)
+		}
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: false, Develop: true}))
+	})
 }
 
 func TestInstallPrivilegedWithBasePath(t *testing.T) {
@@ -103,4 +123,24 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: false}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+
+	t.Run("check second agent installs with --develop", func(t *testing.T) {
+		// Get path to Elastic Agent executable
+		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+		require.NoError(t, err)
+
+		// Prepare the Elastic Agent so the binary is extracted and ready to use.
+		err = devFixture.Prepare(ctx)
+		require.NoError(t, err)
+
+		devOpts := &atesting.InstallOpts{BasePath: randomBasePath, Force: true, Privileged: true, Develop: true}
+		devOut, err := devFixture.Install(ctx, devOpts)
+		if err != nil {
+			t.Logf("install --develop output: %s", devOut)
+			require.NoError(t, err)
+		}
+
+		devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, devTopPath, &installtest.CheckOpts{Unprivileged: false, Develop: true}))
+	})
 }
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index 7ef36f4aef5..b38661ad808 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -169,7 +169,7 @@ func TestInstallWithBasePath(t *testing.T) {
 		err = devFixture.Prepare(ctx)
 		require.NoError(t, err)
 
-		devOpts := &atesting.InstallOpts{Force: true, Privileged: false, Develop: true}
+		devOpts := &atesting.InstallOpts{BasePath: basePath, Force: true, Privileged: false, Develop: true}
 		devOut, err := devFixture.Install(ctx, devOpts)
 		if err != nil {
 			t.Logf("install --develop output: %s", devOut)

From ea3999d423549a03db4a9f359d14e502000205c6 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 14:16:28 -0400
Subject: [PATCH 16/49] Move install tests to the same file.

---
 .../integration/install_privileged_test.go    | 146 ------------------
 testing/integration/install_test.go           | 123 +++++++++++++++
 2 files changed, 123 insertions(+), 146 deletions(-)
 delete mode 100644 testing/integration/install_privileged_test.go

diff --git a/testing/integration/install_privileged_test.go b/testing/integration/install_privileged_test.go
deleted file mode 100644
index 1f6959ebeb0..00000000000
--- a/testing/integration/install_privileged_test.go
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-//go:build integration
-
-package integration
-
-import (
-	"context"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
-	atesting "github.com/elastic/elastic-agent/pkg/testing"
-	"github.com/elastic/elastic-agent/pkg/testing/define"
-	"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
-	"github.com/elastic/elastic-agent/testing/installtest"
-)
-
-func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
-	define.Require(t, define.Requirements{
-		Group: Default,
-		// We require sudo for this test to run
-		// `elastic-agent install`.
-		Sudo: true,
-
-		// It's not safe to run this test locally as it
-		// installs Elastic Agent.
-		Local: false,
-	})
-
-	// Get path to Elastic Agent executable
-	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-	require.NoError(t, err)
-
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
-	defer cancel()
-
-	// Prepare the Elastic Agent so the binary is extracted and ready to use.
-	err = fixture.Prepare(ctx)
-	require.NoError(t, err)
-
-	// Run `elastic-agent install`.  We use `--force` to prevent interactive
-	// execution.
-	opts := &atesting.InstallOpts{Force: true, Privileged: true}
-	out, err := fixture.Install(ctx, opts)
-	if err != nil {
-		t.Logf("install output: %s", out)
-		require.NoError(t, err)
-	}
-
-	// Check that Agent was installed in default base path
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: false}))
-
-	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-
-	t.Run("check second agent installs with --develop", func(t *testing.T) {
-		// Get path to Elastic Agent executable
-		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-		require.NoError(t, err)
-
-		// Prepare the Elastic Agent so the binary is extracted and ready to use.
-		err = devFixture.Prepare(ctx)
-		require.NoError(t, err)
-
-		devOpts := &atesting.InstallOpts{Force: true, Privileged: true, Develop: true}
-		devOut, err := devFixture.Install(ctx, devOpts)
-		if err != nil {
-			t.Logf("install --develop output: %s", devOut)
-			require.NoError(t, err)
-		}
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: false, Develop: true}))
-	})
-}
-
-func TestInstallPrivilegedWithBasePath(t *testing.T) {
-	define.Require(t, define.Requirements{
-		Group: Default,
-		// We require sudo for this test to run
-		// `elastic-agent install`.
-		Sudo: true,
-
-		// It's not safe to run this test locally as it
-		// installs Elastic Agent.
-		Local: false,
-	})
-
-	// Get path to Elastic Agent executable
-	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-	require.NoError(t, err)
-
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
-	defer cancel()
-
-	// Prepare the Elastic Agent so the binary is extracted and ready to use.
-	err = fixture.Prepare(ctx)
-	require.NoError(t, err)
-
-	// Set up random temporary directory to serve as base path for Elastic Agent
-	// installation.
-	tmpDir := t.TempDir()
-	randomBasePath := filepath.Join(tmpDir, strings.ToLower(randStr(8)))
-
-	// Run `elastic-agent install`.  We use `--force` to prevent interactive
-	// execution.
-	opts := &atesting.InstallOpts{
-		BasePath:   randomBasePath,
-		Force:      true,
-		Privileged: true,
-	}
-	out, err := fixture.Install(ctx, opts)
-	if err != nil {
-		t.Logf("install output: %s", out)
-		require.NoError(t, err)
-	}
-
-	// Check that Agent was installed in the custom base path
-	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: false}))
-	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-
-	t.Run("check second agent installs with --develop", func(t *testing.T) {
-		// Get path to Elastic Agent executable
-		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-		require.NoError(t, err)
-
-		// Prepare the Elastic Agent so the binary is extracted and ready to use.
-		err = devFixture.Prepare(ctx)
-		require.NoError(t, err)
-
-		devOpts := &atesting.InstallOpts{BasePath: randomBasePath, Force: true, Privileged: true, Develop: true}
-		devOut, err := devFixture.Install(ctx, devOpts)
-		if err != nil {
-			t.Logf("install --develop output: %s", devOut)
-			require.NoError(t, err)
-		}
-
-		devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, devTopPath, &installtest.CheckOpts{Unprivileged: false, Develop: true}))
-	})
-}
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index b38661ad808..608e9d4aeac 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -195,6 +195,129 @@ func TestInstallWithBasePath(t *testing.T) {
 	}
 }
 
+func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
+	define.Require(t, define.Requirements{
+		Group: Default,
+		// We require sudo for this test to run
+		// `elastic-agent install`.
+		Sudo: true,
+
+		// It's not safe to run this test locally as it
+		// installs Elastic Agent.
+		Local: false,
+	})
+
+	// Get path to Elastic Agent executable
+	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	require.NoError(t, err)
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+
+	// Prepare the Elastic Agent so the binary is extracted and ready to use.
+	err = fixture.Prepare(ctx)
+	require.NoError(t, err)
+
+	// Run `elastic-agent install`.  We use `--force` to prevent interactive
+	// execution.
+	opts := &atesting.InstallOpts{Force: true, Privileged: true}
+	out, err := fixture.Install(ctx, opts)
+	if err != nil {
+		t.Logf("install output: %s", out)
+		require.NoError(t, err)
+	}
+
+	// Check that Agent was installed in default base path
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: false}))
+
+	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+
+	t.Run("check second agent installs with --develop", func(t *testing.T) {
+		// Get path to Elastic Agent executable
+		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+		require.NoError(t, err)
+
+		// Prepare the Elastic Agent so the binary is extracted and ready to use.
+		err = devFixture.Prepare(ctx)
+		require.NoError(t, err)
+
+		devOpts := &atesting.InstallOpts{Force: true, Privileged: true, Develop: true}
+		devOut, err := devFixture.Install(ctx, devOpts)
+		if err != nil {
+			t.Logf("install --develop output: %s", devOut)
+			require.NoError(t, err)
+		}
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: false, Develop: true}))
+	})
+}
+
+func TestInstallPrivilegedWithBasePath(t *testing.T) {
+	define.Require(t, define.Requirements{
+		Group: Default,
+		// We require sudo for this test to run
+		// `elastic-agent install`.
+		Sudo: true,
+
+		// It's not safe to run this test locally as it
+		// installs Elastic Agent.
+		Local: false,
+	})
+
+	// Get path to Elastic Agent executable
+	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	require.NoError(t, err)
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+
+	// Prepare the Elastic Agent so the binary is extracted and ready to use.
+	err = fixture.Prepare(ctx)
+	require.NoError(t, err)
+
+	// Set up random temporary directory to serve as base path for Elastic Agent
+	// installation.
+	tmpDir := t.TempDir()
+	randomBasePath := filepath.Join(tmpDir, strings.ToLower(randStr(8)))
+
+	// Run `elastic-agent install`.  We use `--force` to prevent interactive
+	// execution.
+	opts := &atesting.InstallOpts{
+		BasePath:   randomBasePath,
+		Force:      true,
+		Privileged: true,
+	}
+	out, err := fixture.Install(ctx, opts)
+	if err != nil {
+		t.Logf("install output: %s", out)
+		require.NoError(t, err)
+	}
+
+	// Check that Agent was installed in the custom base path
+	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: false}))
+	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+
+	t.Run("check second agent installs with --develop", func(t *testing.T) {
+		// Get path to Elastic Agent executable
+		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+		require.NoError(t, err)
+
+		// Prepare the Elastic Agent so the binary is extracted and ready to use.
+		err = devFixture.Prepare(ctx)
+		require.NoError(t, err)
+
+		devOpts := &atesting.InstallOpts{BasePath: randomBasePath, Force: true, Privileged: true, Develop: true}
+		devOut, err := devFixture.Install(ctx, devOpts)
+		if err != nil {
+			t.Logf("install --develop output: %s", devOut)
+			require.NoError(t, err)
+		}
+
+		devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, devTopPath, &installtest.CheckOpts{Unprivileged: false, Develop: true}))
+	})
+}
+
 // TestRepeatedInstallUninstall will install then uninstall the agent
 // repeatedly.  This test exists because of a number of race
 // conditions that have occurred in the uninstall process.  Current

From 3897f40c4efb7bcb104a8e3ea2d954c76dc1adfe Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 14:48:19 -0400
Subject: [PATCH 17/49] Refactor develop test into function.

---
 testing/integration/install_test.go | 104 ++++++++--------------------
 1 file changed, 29 insertions(+), 75 deletions(-)

diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index 608e9d4aeac..0539a4a2049 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -52,8 +52,8 @@ func TestInstallWithoutBasePath(t *testing.T) {
 
 	// Run `elastic-agent install`.  We use `--force` to prevent interactive
 	// execution.
-	opts := &atesting.InstallOpts{Force: true, Privileged: false}
-	out, err := fixture.Install(ctx, opts)
+	opts := atesting.InstallOpts{Force: true, Privileged: false}
+	out, err := fixture.Install(ctx, &opts)
 	if err != nil {
 		t.Logf("install output: %s", out)
 		require.NoError(t, err)
@@ -61,27 +61,10 @@ func TestInstallWithoutBasePath(t *testing.T) {
 
 	// Check that Agent was installed in default base path
 	topPath := installtest.DefaultTopPath()
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: true}))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-
-	t.Run("check second agent installs with --develop", func(t *testing.T) {
-		// Get path to Elastic Agent executable
-		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-		require.NoError(t, err)
-
-		// Prepare the Elastic Agent so the binary is extracted and ready to use.
-		err = devFixture.Prepare(ctx)
-		require.NoError(t, err)
-
-		devOpts := &atesting.InstallOpts{Force: true, Privileged: false, Develop: true}
-		devOut, err := devFixture.Install(ctx, devOpts)
-		if err != nil {
-			t.Logf("install --develop output: %s", devOut)
-			require.NoError(t, err)
-		}
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: true, Develop: true}))
-	})
+	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, installtest.DevelopTopPath(), opts))
 
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
@@ -143,12 +126,12 @@ func TestInstallWithBasePath(t *testing.T) {
 
 	// Run `elastic-agent install`.  We use `--force` to prevent interactive
 	// execution.
-	opts := &atesting.InstallOpts{
+	opts := atesting.InstallOpts{
 		BasePath:   basePath,
 		Force:      true,
 		Privileged: false,
 	}
-	out, err := fixture.Install(ctx, opts)
+	out, err := fixture.Install(ctx, &opts)
 	if err != nil {
 		t.Logf("install output: %s", out)
 		require.NoError(t, err)
@@ -156,29 +139,11 @@ func TestInstallWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: true}))
+	devTopPath := filepath.Join(basePath, "Elastic", paths.DevelopmentInstallDirName)
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-
-	t.Run("check second agent installs with --develop", func(t *testing.T) {
-		// Get path to Elastic Agent executable
-		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-		require.NoError(t, err)
-
-		// Prepare the Elastic Agent so the binary is extracted and ready to use.
-		err = devFixture.Prepare(ctx)
-		require.NoError(t, err)
-
-		devOpts := &atesting.InstallOpts{BasePath: basePath, Force: true, Privileged: false, Develop: true}
-		devOut, err := devFixture.Install(ctx, devOpts)
-		if err != nil {
-			t.Logf("install --develop output: %s", devOut)
-			require.NoError(t, err)
-		}
-
-		devTopPath := filepath.Join(basePath, "Elastic", paths.DevelopmentInstallDirName)
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, devTopPath, &installtest.CheckOpts{Unprivileged: true, Develop: true}))
-	})
+	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))
 
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
@@ -220,35 +185,18 @@ func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
 
 	// Run `elastic-agent install`.  We use `--force` to prevent interactive
 	// execution.
-	opts := &atesting.InstallOpts{Force: true, Privileged: true}
-	out, err := fixture.Install(ctx, opts)
+	opts := atesting.InstallOpts{Force: true, Privileged: true}
+	out, err := fixture.Install(ctx, &opts)
 	if err != nil {
 		t.Logf("install output: %s", out)
 		require.NoError(t, err)
 	}
 
 	// Check that Agent was installed in default base path
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: false}))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-
-	t.Run("check second agent installs with --develop", func(t *testing.T) {
-		// Get path to Elastic Agent executable
-		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
-		require.NoError(t, err)
-
-		// Prepare the Elastic Agent so the binary is extracted and ready to use.
-		err = devFixture.Prepare(ctx)
-		require.NoError(t, err)
-
-		devOpts := &atesting.InstallOpts{Force: true, Privileged: true, Develop: true}
-		devOut, err := devFixture.Install(ctx, devOpts)
-		if err != nil {
-			t.Logf("install --develop output: %s", devOut)
-			require.NoError(t, err)
-		}
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, installtest.DevelopTopPath(), &installtest.CheckOpts{Unprivileged: false, Develop: true}))
-	})
+	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, installtest.DevelopTopPath(), opts))
 }
 
 func TestInstallPrivilegedWithBasePath(t *testing.T) {
@@ -281,12 +229,12 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 
 	// Run `elastic-agent install`.  We use `--force` to prevent interactive
 	// execution.
-	opts := &atesting.InstallOpts{
+	opts := atesting.InstallOpts{
 		BasePath:   randomBasePath,
 		Force:      true,
 		Privileged: true,
 	}
-	out, err := fixture.Install(ctx, opts)
+	out, err := fixture.Install(ctx, &opts)
 	if err != nil {
 		t.Logf("install output: %s", out)
 		require.NoError(t, err)
@@ -294,10 +242,15 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: false}))
+	devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
+	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))
+}
 
-	t.Run("check second agent installs with --develop", func(t *testing.T) {
+// Tests that a second agent for development purposes can be installed alongside the first one with the --develop option.
+func testDevelopmentAgentCanInstall(ctx context.Context, fixture *atesting.Fixture, topPath string, installOpts atesting.InstallOpts) func(*testing.T) {
+	return func(t *testing.T) {
 		// Get path to Elastic Agent executable
 		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
 		require.NoError(t, err)
@@ -306,16 +259,17 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 		err = devFixture.Prepare(ctx)
 		require.NoError(t, err)
 
-		devOpts := &atesting.InstallOpts{BasePath: randomBasePath, Force: true, Privileged: true, Develop: true}
-		devOut, err := devFixture.Install(ctx, devOpts)
+		installOpts.Develop = true
+		devOut, err := devFixture.Install(ctx, &installOpts)
 		if err != nil {
 			t.Logf("install --develop output: %s", devOut)
 			require.NoError(t, err)
 		}
-
-		devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
-		require.NoError(t, installtest.CheckSuccess(ctx, fixture, devTopPath, &installtest.CheckOpts{Unprivileged: false, Develop: true}))
-	})
+		require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{
+			Unprivileged: !installOpts.Privileged,
+			Develop:      installOpts.Develop,
+		}))
+	}
 }
 
 // TestRepeatedInstallUninstall will install then uninstall the agent

From d373536660888ba8cab2f9bd51a35574a28dc0ef Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 14:52:37 -0400
Subject: [PATCH 18/49] Invert condition to match installopts

---
 testing/installtest/checks.go              |  4 ++--
 testing/installtest/checks_unix.go         |  2 +-
 testing/installtest/checks_windows.go      |  2 +-
 testing/integration/install_test.go        | 14 +++++++-------
 testing/integration/logs_ingestion_test.go |  2 +-
 testing/upgradetest/upgrader.go            |  4 ++--
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/testing/installtest/checks.go b/testing/installtest/checks.go
index 27e1202cd66..7373a670ef7 100644
--- a/testing/installtest/checks.go
+++ b/testing/installtest/checks.go
@@ -38,8 +38,8 @@ func DevelopTopPath() string {
 }
 
 type CheckOpts struct {
-	Unprivileged bool
-	Develop      bool
+	Privileged bool
+	Develop    bool
 }
 
 func CheckSuccess(ctx context.Context, f *atesting.Fixture, topPath string, opts *CheckOpts) error {
diff --git a/testing/installtest/checks_unix.go b/testing/installtest/checks_unix.go
index e4230df6c75..0b2a5741431 100644
--- a/testing/installtest/checks_unix.go
+++ b/testing/installtest/checks_unix.go
@@ -21,7 +21,7 @@ import (
 )
 
 func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, opts *CheckOpts) error {
-	if opts.Unprivileged {
+	if !opts.Privileged {
 		// Check that the elastic-agent user/group exist.
 		uid, err := install.FindUID(install.ElasticUsername)
 		if err != nil {
diff --git a/testing/installtest/checks_windows.go b/testing/installtest/checks_windows.go
index 4a4dd0efc10..6a87f43f173 100644
--- a/testing/installtest/checks_windows.go
+++ b/testing/installtest/checks_windows.go
@@ -52,7 +52,7 @@ func checkPlatform(ctx context.Context, f *atesting.Fixture, topPath string, opt
 	if err != nil {
 		return fmt.Errorf("failed to get allowed SID's for %s: %w", topPath, err)
 	}
-	if opts.Unprivileged {
+	if !opts.Privileged {
 		// Check that the elastic-agent user/group exist.
 		uid, err := install.FindUID(install.ElasticUsername)
 		if err != nil {
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index 0539a4a2049..a71b918367d 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -61,7 +61,7 @@ func TestInstallWithoutBasePath(t *testing.T) {
 
 	// Check that Agent was installed in default base path
 	topPath := installtest.DefaultTopPath()
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, installtest.DevelopTopPath(), opts))
@@ -140,7 +140,7 @@ func TestInstallWithBasePath(t *testing.T) {
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
 	devTopPath := filepath.Join(basePath, "Elastic", paths.DevelopmentInstallDirName)
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))
@@ -193,7 +193,7 @@ func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
 	}
 
 	// Check that Agent was installed in default base path
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, installtest.DevelopTopPath(), opts))
@@ -243,7 +243,7 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
 	devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))
 }
@@ -266,8 +266,8 @@ func testDevelopmentAgentCanInstall(ctx context.Context, fixture *atesting.Fixtu
 			require.NoError(t, err)
 		}
 		require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{
-			Unprivileged: !installOpts.Privileged,
-			Develop:      installOpts.Develop,
+			Privileged: installOpts.Privileged,
+			Develop:    installOpts.Develop,
 		}))
 	}
 }
@@ -315,7 +315,7 @@ func TestRepeatedInstallUninstall(t *testing.T) {
 			}
 
 			// Check that Agent was installed in default base path
-			require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Unprivileged: !opts.Privileged}))
+			require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 			t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 			out, err = fixture.Uninstall(ctx, &atesting.UninstallOpts{Force: true})
 			require.NoErrorf(t, err, "uninstall failed: %s", err)
diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go
index 0b036a4e64c..8a6b6ecfd91 100644
--- a/testing/integration/logs_ingestion_test.go
+++ b/testing/integration/logs_ingestion_test.go
@@ -93,7 +93,7 @@ func TestLogIngestionFleetManaged(t *testing.T) {
 	check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute)
 
 	// 3. Ensure installation is correct.
-	require.NoError(t, installtest.CheckSuccess(ctx, agentFixture, installOpts.BasePath, &installtest.CheckOpts{Unprivileged: !installOpts.Privileged}))
+	require.NoError(t, installtest.CheckSuccess(ctx, agentFixture, installOpts.BasePath, &installtest.CheckOpts{Privileged: installOpts.Privileged}))
 
 	t.Run("Monitoring logs are shipped", func(t *testing.T) {
 		testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
diff --git a/testing/upgradetest/upgrader.go b/testing/upgradetest/upgrader.go
index 54e1c517883..b8e8f70feb3 100644
--- a/testing/upgradetest/upgrader.go
+++ b/testing/upgradetest/upgrader.go
@@ -281,7 +281,7 @@ func PerformUpgrade(
 
 	// validate installation is correct
 	if InstallChecksAllowed(!installOpts.Privileged, startVersion) {
-		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, &installtest.CheckOpts{Unprivileged: !installOpts.Privileged})
+		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, &installtest.CheckOpts{Privileged: installOpts.Privileged})
 		if err != nil {
 			return fmt.Errorf("pre-upgrade installation checks failed: %w", err)
 		}
@@ -412,7 +412,7 @@ func PerformUpgrade(
 
 	// validate again that the installation is correct, upgrade should not have changed installation validation
 	if InstallChecksAllowed(!installOpts.Privileged, startVersion, endVersion) {
-		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, &installtest.CheckOpts{Unprivileged: !installOpts.Privileged})
+		err = installtest.CheckSuccess(ctx, startFixture, installOpts.BasePath, &installtest.CheckOpts{Privileged: installOpts.Privileged})
 		if err != nil {
 			return fmt.Errorf("post-upgrade installation checks failed: %w", err)
 		}

From b780b9b3fdeb8ad0b5263c24f89aecd491091f83 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 15:15:36 -0400
Subject: [PATCH 19/49] Automatically add development tag on enroll.

---
 internal/pkg/agent/cmd/enroll_cmd.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/internal/pkg/agent/cmd/enroll_cmd.go b/internal/pkg/agent/cmd/enroll_cmd.go
index 863727d6879..5b2ec2c812f 100644
--- a/internal/pkg/agent/cmd/enroll_cmd.go
+++ b/internal/pkg/agent/cmd/enroll_cmd.go
@@ -556,6 +556,12 @@ func (c *enrollCmd) enroll(ctx context.Context, persistentConfig map[string]inte
 		return errors.New(err, "acquiring metadata failed")
 	}
 
+	// Automatically add a "development" tag when enrolling in development mode.
+	// Ensures the development agent is differentiated from others when on the same host.
+	if paths.IsDevelopmentMode() {
+		c.options.Tags = append(c.options.Tags, "development")
+	}
+
 	r := &fleetapi.EnrollRequest{
 		EnrollAPIKey: c.options.EnrollAPIKey,
 		Type:         fleetapi.PermanentEnroll,

From 2dbd0a4dd74be2a38d5344ff40f6b4adf7f92edb Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 30 May 2024 15:46:34 -0400
Subject: [PATCH 20/49] Change shell wrapper path to development.

---
 internal/pkg/agent/application/paths/paths_darwin.go | 2 +-
 internal/pkg/agent/application/paths/paths_linux.go  | 2 +-
 testing/installtest/checks_unix.go                   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index 5d13da28cba..5c324066759 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -25,7 +25,7 @@ const (
 
 	// shellWrapperPath is the path to the installed shell wrapper.
 	shellWrapperPath                = "/usr/local/bin/elastic-agent"
-	shellWrapperPathDevelopmentMode = "/usr/local/bin/elastic-dev-agent"
+	shellWrapperPathDevelopmentMode = "/usr/local/bin/elastic-development-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index 6782cd9020c..940609650e0 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -20,7 +20,7 @@ const (
 
 	// shellWrapperPath is the path to the installed shell wrapper.
 	shellWrapperPath                = "/usr/bin/elastic-agent"
-	shellWrapperPathDevelopmentMode = "/usr/bin/elastic-dev-agent"
+	shellWrapperPathDevelopmentMode = "/usr/bin/elastic-development-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
diff --git a/testing/installtest/checks_unix.go b/testing/installtest/checks_unix.go
index 0b2a5741431..e1503abd691 100644
--- a/testing/installtest/checks_unix.go
+++ b/testing/installtest/checks_unix.go
@@ -66,7 +66,7 @@ func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, opt
 		// Executing `elastic-agent status` as the `elastic-agent-user` user should work.
 		shellWrapperName := "elastic-agent"
 		if opts.Develop {
-			shellWrapperName = "elastic-dev-agent"
+			shellWrapperName = "elastic-development-agent"
 		}
 
 		var output []byte

From 7546d097937cbdd398e39d1807ceaaad57723d76 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Fri, 31 May 2024 11:41:42 -0400
Subject: [PATCH 21/49] Add documentation for --develop.

---
 README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 48 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 7dda9194a77..16335c586e9 100644
--- a/README.md
+++ b/README.md
@@ -3,22 +3,66 @@
 [![Build status](https://badge.buildkite.com/1d35bb40427cc6833979645b61ea214fc4b686a2ffe3a68bdf.svg)](https://buildkite.com/elastic/elastic-agent)
 [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=elastic_elastic-agent&metric=coverage)](https://sonarcloud.io/summary/new_code?id=elastic_elastic-agent)
 
-## Architecture / internal docs
+## Architecture and Internals
 
 - [Agent architecture](docs/architecture.md)
 - [Component spec files](docs/component-specs.md)
 - [Policy configuration](docs/agent-policy.md)
 
+## Official Documentation
+
+See https://www.elastic.co/guide/en/fleet/current/index.html.
+
+The source files for the offical Elastic Agent documentation are currently stored
+in the [ingest-docs](https://github.com/elastic/ingest-docs/tree/main/docs/en/ingest-management) repository.
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
 
-## Developer docs
+## Developing
 
-The source files for the general Elastic Agent documentation are currently stored
-in the [ingest-docs](https://github.com/elastic/ingest-docs/tree/main/docs/en/ingest-management) repository.
 The following docs are only focused on getting developers started building code for Elastic Agent.
 
+### Development Installations
+
+> :warning: Development installations are not officially supported and are intended for Elastic Agent developers.
+
+If you are an Elastic employee, you already have an Information Security managed Elastic Agent installed on your machine for endpoint protection.
+This prevents you from installing the Elastic Agent a second time for development without using a VM or Docker container. To eliminate this point
+of friction, Elastic Agent has a development mode that permits installing the Elastic Agent on your machine a second time:
+
+```sh
+# All other arguments to the install command are supported when --develop is specified.
+sudo ./elastic-agent install --develop
+# The run command also supports the --develop option to all running without installing when there is another agent on the machine.
+./elastic-agent run -e --develop
+```
+
+Using the `--develop` option will install the agent in an isolated `DevelopmentAgent` agent directory in the chosen base path.
+Development agents installed in Fleet will automatically have the `development` tag applied. Using the default base path on MacOS you will see:
+
+```sh
+sudo ls /Library/Elastic/
+Agent
+DevelopmentAgent
+```
+
+The `elastic-agent` command in the shell is replaced with `elastic-development-agent` to interact with the development agent:
+
+```sh
+# For a priveleged agent
+sudo elastic-development-agent status
+# For an unpriveleged agent
+sudo -u elastic-agent-user elastic-development-agent status
+```
+
+The primary restriction of `--develop` installations is that you cannot run Elastic Defend a second time on the same machine. Defend installations
+for development installations will fail with resource conflicts. All other integrations should be usable provided conflicting configurations are
+changed ahead of time. For example two agents cannot bind to the same `agent.monitoring.http.port` value to expose their monitoring servers.
+
+To follow the changes made to support development mode, search for the IsDevelopmentMode() function in the source code.
+
 ### Test Framework
 
 In addition to standard Go tests, changes to the Elastic Agent are always installed and tested on cross-platform virtual machines.

From 3b0d997516e64cb7798cb6b9b67a1843097508d7 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Fri, 31 May 2024 14:49:15 -0400
Subject: [PATCH 22/49] Use lowercase for consistency.

---
 internal/pkg/agent/application/paths/paths_windows.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index 61e99fdfde8..eec47dbe211 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -26,7 +26,7 @@ const (
 
 	// serviceName is the service name when installed.
 	serviceName                = "Elastic Agent"
-	serviceNameDevelopmentMode = "Elastic Agent (Development)"
+	serviceNameDevelopmentMode = "Elastic Agent (development)"
 
 	// shellWrapperPath is the path to the installed shell wrapper.
 	shellWrapperPath                = ""

From ce929fb79d5ef2663574ef7e513ba19137cdf5e2 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Fri, 31 May 2024 14:59:56 -0400
Subject: [PATCH 23/49] Remove TODO comments.

---
 internal/pkg/agent/application/upgrade/rollback.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/internal/pkg/agent/application/upgrade/rollback.go b/internal/pkg/agent/application/upgrade/rollback.go
index 71fdafaf24b..45648951298 100644
--- a/internal/pkg/agent/application/upgrade/rollback.go
+++ b/internal/pkg/agent/application/upgrade/rollback.go
@@ -155,8 +155,6 @@ func InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error
 	agentPID := os.Getpid()
 
 	go func() {
-		// TODO: This should probably be associated with a context?
-		// TODO: What happens if the agent is sent SIGKILL?
 		if err := cmd.Wait(); err != nil {
 			log.Infow("Upgrade Watcher exited with error", "agent.upgrade.watcher.process.pid", "agent.process.pid", agentPID, upgradeWatcherPID, "error.message", err)
 		}

From 03fe19caae01b6d343f4f61aac685ad16dd9ca62 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Fri, 31 May 2024 15:07:00 -0400
Subject: [PATCH 24/49] Fix README typos.

---
 README.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 16335c586e9..44083c5e99f 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ## Developing
 
-The following docs are only focused on getting developers started building code for Elastic Agent.
+The following are exclusively focused on getting developers started building code for Elastic Agent.
 
 ### Development Installations
 
@@ -33,14 +33,14 @@ This prevents you from installing the Elastic Agent a second time for developmen
 of friction, Elastic Agent has a development mode that permits installing the Elastic Agent on your machine a second time:
 
 ```sh
-# All other arguments to the install command are supported when --develop is specified.
+# All other arguments to the install command are still supported when --develop is specified.
 sudo ./elastic-agent install --develop
-# The run command also supports the --develop option to all running without installing when there is another agent on the machine.
+# The run command also supports the --develop option to allow running without installing when there is another agent on the machine.
 ./elastic-agent run -e --develop
 ```
 
 Using the `--develop` option will install the agent in an isolated `DevelopmentAgent` agent directory in the chosen base path.
-Development agents installed in Fleet will automatically have the `development` tag applied. Using the default base path on MacOS you will see:
+Development agents enrolled in Fleet will have the `development` tag added automatically. Using the default base path on MacOS you will see:
 
 ```sh
 sudo ls /Library/Elastic/
@@ -57,11 +57,11 @@ sudo elastic-development-agent status
 sudo -u elastic-agent-user elastic-development-agent status
 ```
 
-The primary restriction of `--develop` installations is that you cannot run Elastic Defend a second time on the same machine. Defend installations
-for development installations will fail with resource conflicts. All other integrations should be usable provided conflicting configurations are
-changed ahead of time. For example two agents cannot bind to the same `agent.monitoring.http.port` value to expose their monitoring servers.
+The primary restriction of `--develop` installations is that you cannot run Elastic Defend a second time on the same machine. Attempting to
+install Defend twice will fail with resource conflicts. All other integrations should be usable provided conflicting configurations are
+changed ahead of time. For example two agents cannot bind to the same `agent.monitoring.http.port` to expose their monitoring servers.
 
-To follow the changes made to support development mode, search for the IsDevelopmentMode() function in the source code.
+To follow the changes made to support development mode, search for the `IsDevelopmentMode()` function in the source code.
 
 ### Test Framework
 

From 3c1ed11bb27a75c9ecedfb771bc117a7fe22cb45 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Fri, 31 May 2024 15:34:42 -0400
Subject: [PATCH 25/49] Adjust comments.

---
 .../pkg/agent/application/paths/common_development_mode.go    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/agent/application/paths/common_development_mode.go b/internal/pkg/agent/application/paths/common_development_mode.go
index c022c70aa0d..eb0b091caf3 100644
--- a/internal/pkg/agent/application/paths/common_development_mode.go
+++ b/internal/pkg/agent/application/paths/common_development_mode.go
@@ -2,6 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
+// This file encapsulates the common paths that need to account for development installation.
 package paths
 
 import "path/filepath"
@@ -12,7 +13,7 @@ const DevelopmentInstallDirName string = "DevelopmentAgent"
 
 var isDevelopmentMode bool
 
-// SetIsDevelopmentMode sets whether the agent is installed in development mode or not.
+// SetIsDevelopmentMode sets whether the agent is currently in or is being installed in development mode.
 func SetIsDevelopmentMode(developmentMode bool) {
 	isDevelopmentMode = developmentMode
 }
@@ -57,6 +58,7 @@ func ShellWrapperPath() string {
 }
 
 // ControlSocketRunSymlink returns the shell wrapper path accounting for development mode.
+// Does not auto detect development mode because it is used outside of agent itself in the testing framework.
 func ControlSocketRunSymlink(isDevelopmentMode bool) string {
 	if isDevelopmentMode {
 		return controlSocketRunSymlinkDevelopmentMode

From 6e36b1a7bf02fa4464fc8891dc7787b0f8cc4c99 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Fri, 31 May 2024 15:52:15 -0400
Subject: [PATCH 26/49] More typo fixes.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 44083c5e99f..67896703ccf 100644
--- a/README.md
+++ b/README.md
@@ -51,9 +51,9 @@ DevelopmentAgent
 The `elastic-agent` command in the shell is replaced with `elastic-development-agent` to interact with the development agent:
 
 ```sh
-# For a priveleged agent
+# For a privileged agent
 sudo elastic-development-agent status
-# For an unpriveleged agent
+# For an unprivileged agent
 sudo -u elastic-agent-user elastic-development-agent status
 ```
 

From 58afd75611a12b52252a1f2720d270a4a65cae48 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 3 Jun 2024 12:41:28 -0400
Subject: [PATCH 27/49] Fix description not to mention beats.

---
 magefile.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magefile.go b/magefile.go
index 98637eb3191..aca74c8da0d 100644
--- a/magefile.go
+++ b/magefile.go
@@ -108,7 +108,7 @@ func init() {
 	common.RegisterCheckDeps(Update, Check.All)
 	test.RegisterDeps(UnitTest)
 	devtools.BeatLicense = "Elastic License"
-	devtools.BeatDescription = "Agent manages other beats based on configuration provided."
+	devtools.BeatDescription = "Elastic Agent - single, unified way to add monitoring for logs, metrics, and other types of data to a host."
 
 	devtools.Platforms = devtools.Platforms.Filter("!linux/386")
 	devtools.Platforms = devtools.Platforms.Filter("!windows/386")

From b6bd76a2c6ed9e62b33423d6b999cf78e9a41bb5 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 3 Jun 2024 12:46:01 -0400
Subject: [PATCH 28/49] Change windows service name to avoid collision.

---
 internal/pkg/agent/application/paths/paths_windows.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index eec47dbe211..683d110147a 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -26,7 +26,7 @@ const (
 
 	// serviceName is the service name when installed.
 	serviceName                = "Elastic Agent"
-	serviceNameDevelopmentMode = "Elastic Agent (development)"
+	serviceNameDevelopmentMode = "Elastic Development Agent"
 
 	// shellWrapperPath is the path to the installed shell wrapper.
 	shellWrapperPath                = ""

From 7b8e541528d27acee997bf43e7905a25cc8249fb Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 3 Jun 2024 16:14:15 -0400
Subject: [PATCH 29/49] Make service display name unique on Windows.

---
 .../application/paths/common_development_mode.go | 16 +++++++++++++++-
 internal/pkg/agent/install/svc.go                |  5 +----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_development_mode.go b/internal/pkg/agent/application/paths/common_development_mode.go
index eb0b091caf3..63c79579b11 100644
--- a/internal/pkg/agent/application/paths/common_development_mode.go
+++ b/internal/pkg/agent/application/paths/common_development_mode.go
@@ -9,7 +9,13 @@ import "path/filepath"
 
 // DevelopmentInstallDirName is the name of the directory agent will be installed to within the base path.
 // For example it is $BasePath/$DevelopmentInstallDirName, on MacOS it is /Library/Elastic/$DevelopmentInstallDirName.
-const DevelopmentInstallDirName string = "DevelopmentAgent"
+const (
+	DevelopmentInstallDirName = "DevelopmentAgent"
+
+	// Service display names. Must be different from the ServiceName() on Windows.
+	serviceDisplayName                = "Elastic Agent"
+	serviceDisplayNameDevelopmentMode = "Elastic Development Agent"
+)
 
 var isDevelopmentMode bool
 
@@ -49,6 +55,14 @@ func ServiceName() string {
 	return serviceName
 }
 
+// ServiceDisplayName returns the service display name accounting for development mode.
+func ServiceDisplayName() string {
+	if IsDevelopmentMode() {
+		return serviceDisplayNameDevelopmentMode
+	}
+	return serviceDisplayName
+}
+
 // ShellWrapperPath returns the shell wrapper path accounting for development mode.
 func ShellWrapperPath() string {
 	if IsDevelopmentMode() {
diff --git a/internal/pkg/agent/install/svc.go b/internal/pkg/agent/install/svc.go
index 09e755d4220..eacc39fc0a1 100644
--- a/internal/pkg/agent/install/svc.go
+++ b/internal/pkg/agent/install/svc.go
@@ -15,9 +15,6 @@ import (
 )
 
 const (
-	// ServiceDisplayName is the service display name for the service.
-	ServiceDisplayName = "Elastic Agent"
-
 	// ServiceDescription is the description for the service.
 	ServiceDescription = "Elastic Agent is a unified agent to observe, monitor and protect your system."
 
@@ -76,7 +73,7 @@ func newService(topPath string, opt ...serviceOpt) (service.Service, error) {
 
 	cfg := &service.Config{
 		Name:             paths.ServiceName(),
-		DisplayName:      ServiceDisplayName,
+		DisplayName:      paths.ServiceDisplayName(),
 		Description:      ServiceDescription,
 		Executable:       ExecutablePath(topPath),
 		WorkingDirectory: topPath,

From 18e7e5e1251c20e3b583c733590a2cdae8b412c1 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 10 Jun 2024 15:27:46 -0400
Subject: [PATCH 30/49] Add concept of an installation namespace.

Restrict use to only the well known development namespace.
---
 README.md                                     |   8 +-
 .../paths/common_development_mode.go          |  81 -----------
 .../application/paths/common_namespace.go     | 131 ++++++++++++++++++
 .../paths/common_namespace_test.go            |  65 +++++++++
 .../agent/application/paths/paths_darwin.go   |  14 +-
 .../agent/application/paths/paths_linux.go    |  14 +-
 .../agent/application/paths/paths_windows.go  |  14 +-
 internal/pkg/agent/cmd/enroll_cmd.go          |   6 +-
 internal/pkg/agent/cmd/install.go             |   3 +-
 internal/pkg/agent/cmd/run.go                 |   6 +-
 internal/pkg/agent/configuration/grpc.go      |  18 +--
 internal/pkg/agent/install/install.go         |   2 +-
 pkg/testing/fixture.go                        |   2 +-
 pkg/testing/fixture_install.go                |   8 +-
 testing/installtest/checks.go                 |   6 +-
 testing/integration/install_test.go           |  10 +-
 16 files changed, 255 insertions(+), 133 deletions(-)
 delete mode 100644 internal/pkg/agent/application/paths/common_development_mode.go
 create mode 100644 internal/pkg/agent/application/paths/common_namespace.go
 create mode 100644 internal/pkg/agent/application/paths/common_namespace_test.go

diff --git a/README.md b/README.md
index 67896703ccf..e289052b922 100644
--- a/README.md
+++ b/README.md
@@ -39,13 +39,13 @@ sudo ./elastic-agent install --develop
 ./elastic-agent run -e --develop
 ```
 
-Using the `--develop` option will install the agent in an isolated `DevelopmentAgent` agent directory in the chosen base path.
-Development agents enrolled in Fleet will have the `development` tag added automatically. Using the default base path on MacOS you will see:
+Using the `--develop` option will install the agent in an isolated `Agent-Development` agent directory in the chosen base path.
+Development agents enrolled in Fleet will have the `Development` tag added automatically. Using the default base path on MacOS you will see:
 
 ```sh
 sudo ls /Library/Elastic/
 Agent
-DevelopmentAgent
+Agent-Development
 ```
 
 The `elastic-agent` command in the shell is replaced with `elastic-development-agent` to interact with the development agent:
@@ -61,8 +61,6 @@ The primary restriction of `--develop` installations is that you cannot run Elas
 install Defend twice will fail with resource conflicts. All other integrations should be usable provided conflicting configurations are
 changed ahead of time. For example two agents cannot bind to the same `agent.monitoring.http.port` to expose their monitoring servers.
 
-To follow the changes made to support development mode, search for the `IsDevelopmentMode()` function in the source code.
-
 ### Test Framework
 
 In addition to standard Go tests, changes to the Elastic Agent are always installed and tested on cross-platform virtual machines.
diff --git a/internal/pkg/agent/application/paths/common_development_mode.go b/internal/pkg/agent/application/paths/common_development_mode.go
deleted file mode 100644
index 63c79579b11..00000000000
--- a/internal/pkg/agent/application/paths/common_development_mode.go
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-// This file encapsulates the common paths that need to account for development installation.
-package paths
-
-import "path/filepath"
-
-// DevelopmentInstallDirName is the name of the directory agent will be installed to within the base path.
-// For example it is $BasePath/$DevelopmentInstallDirName, on MacOS it is /Library/Elastic/$DevelopmentInstallDirName.
-const (
-	DevelopmentInstallDirName = "DevelopmentAgent"
-
-	// Service display names. Must be different from the ServiceName() on Windows.
-	serviceDisplayName                = "Elastic Agent"
-	serviceDisplayNameDevelopmentMode = "Elastic Development Agent"
-)
-
-var isDevelopmentMode bool
-
-// SetIsDevelopmentMode sets whether the agent is currently in or is being installed in development mode.
-func SetIsDevelopmentMode(developmentMode bool) {
-	isDevelopmentMode = developmentMode
-}
-
-// IsDevelopmentMode returns true if the agent is installed in development mode.
-func IsDevelopmentMode() bool {
-	// The current process has explicitly been told it is in development mode.
-	if isDevelopmentMode {
-		return true
-	}
-
-	// We are installed in development mode and have to infer it from the path.
-	if RunningInstalled() && filepath.Base(Top()) == DevelopmentInstallDirName {
-		return true
-	}
-
-	return false
-}
-
-// InstallPath returns the top level directory Agent will be installed into, accounting for development mode.
-func InstallPath(basePath string) string {
-	if IsDevelopmentMode() {
-		return filepath.Join(basePath, "Elastic", DevelopmentInstallDirName)
-	}
-	return filepath.Join(basePath, "Elastic", "Agent")
-}
-
-// ServiceName returns the service name accounting for development mode.
-func ServiceName() string {
-	if IsDevelopmentMode() {
-		return serviceNameDevelopmentMode
-	}
-	return serviceName
-}
-
-// ServiceDisplayName returns the service display name accounting for development mode.
-func ServiceDisplayName() string {
-	if IsDevelopmentMode() {
-		return serviceDisplayNameDevelopmentMode
-	}
-	return serviceDisplayName
-}
-
-// ShellWrapperPath returns the shell wrapper path accounting for development mode.
-func ShellWrapperPath() string {
-	if IsDevelopmentMode() {
-		return shellWrapperPathDevelopmentMode
-	}
-	return shellWrapperPath
-}
-
-// ControlSocketRunSymlink returns the shell wrapper path accounting for development mode.
-// Does not auto detect development mode because it is used outside of agent itself in the testing framework.
-func ControlSocketRunSymlink(isDevelopmentMode bool) string {
-	if isDevelopmentMode {
-		return controlSocketRunSymlinkDevelopmentMode
-	}
-	return controlSocketRunSymlink
-}
diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
new file mode 100644
index 00000000000..9e363678364
--- /dev/null
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -0,0 +1,131 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// This file encapsulates the common paths that need to account for installation namepsaces.
+// Installation namespaces allow multiple agents on the same machine.
+package paths
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// installNamespace is the name of the agent's current installation namepsace.
+var installNamespace string
+
+const (
+	// installDirNamespaceFmt is the format of the directory agent will be installed to within the base path when using an installation namepsace.
+	// For example it is $BasePath/$DevelopmentInstallDirName, on MacOS it is /Library/Elastic/$DevelopmentInstallDirName.
+	installDir             = "Agent"
+	installDirNamespaceFmt = "Agent-%s"
+
+	// DevelopmentNamespace defines the "well known" development namespace.
+	DevelopmentNamespace = "Development"
+
+	// Service display names. Must be different from the ServiceName() on Windows.
+	serviceDisplayName             = "Elastic Agent"
+	serviceDisplayNameNamespaceFmt = "Elastic Agent - %s"
+)
+
+// SetInstallNamespace sets whether the agent is currently in or is being installed in an installation namespace.
+func SetInstallNamespace(namespace string) {
+	installNamespace = namespace
+}
+
+// InstallNamespace returns the name of the current installation namespace. Returns the empty string
+// for the default namespace. For installed agents, the namespace is parsed from the installation
+// directory name, since a unique directory name is required to avoid collisions between installed
+// agents in the same base path. Before installation, the installation namespace must be configured
+// using SetInstallNamespace().
+func InstallNamespace() string {
+	if installNamespace != "" {
+		return installNamespace
+	}
+
+	if RunningInstalled() {
+		return parseNamespaceFromDir(filepath.Base(Top()))
+	}
+
+	return ""
+}
+
+func parseNamespaceFromDir(dir string) string {
+	parts := strings.SplitAfterN(dir, "-", 2)
+	if len(parts) <= 1 {
+		return ""
+	}
+
+	return parts[1]
+}
+
+// InInstallNamespace returns true if the agent is being installed in an installation namespace.
+func InInstallNamespace() bool {
+	return InstallNamespace() != ""
+}
+
+// InstallDirNameForNamespace returns the installation directory name for a given namespace.
+// The installation directory name with a namespace is $BasePath/InstallDirNameForNamespace().
+func InstallDirNameForNamespace(namespace string) string {
+	if namespace == "" {
+		return installDir
+	}
+
+	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
+	return strings.Replace(installDirNamespaceFmt, "%s", namespace, 1)
+}
+
+// InstallPath returns the top level directory Agent will be installed into, accounting for any namespace.
+func InstallPath(basePath string) string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return filepath.Join(basePath, "Elastic", installDir)
+	}
+
+	return filepath.Join(basePath, "Elastic", InstallDirNameForNamespace(namespace))
+}
+
+// ServiceName returns the service name accounting for any namespace.
+func ServiceName() string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return serviceName
+	}
+
+	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
+	return strings.Replace(serviceNameNamespaceFmt, "%s", namespace, 1)
+}
+
+// ServiceDisplayName returns the service display name accounting for any namespace.
+func ServiceDisplayName() string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return serviceDisplayName
+	}
+
+	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
+	return strings.Replace(serviceDisplayNameNamespaceFmt, "%s", namespace, 1)
+}
+
+// ShellWrapperPath returns the shell wrapper path accounting for any namespace.
+// The provided namespace is always lowercased for consistency.
+func ShellWrapperPath() string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return shellWrapperPath
+	}
+
+	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
+	return strings.Replace(shellWrapperPathNamespaceFmt, "%s", strings.ToLower(namespace), 1)
+}
+
+// ControlSocketRunSymlink returns the shell wrapper path accounting for any namespace.
+// Does not auto detect the namespace because it is used outside of agent itself in the testing framework.
+func ControlSocketRunSymlink(namespace string) string {
+	if namespace == "" {
+		return controlSocketRunSymlink
+	}
+
+	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
+	return strings.Replace(controlSocketRunSymlinkNamespaceFmt, "%s", namespace, 1)
+}
diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
new file mode 100644
index 00000000000..4efacee3aed
--- /dev/null
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -0,0 +1,65 @@
+package paths
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInstallNamespace(t *testing.T) {
+	namespace := "testing"
+	basePath := filepath.Join("base", "path")
+	SetInstallNamespace(namespace)
+
+	assert.Equal(t, namespace, InstallNamespace())
+	assert.True(t, InInstallNamespace())
+	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
+	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
+	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
+	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())
+	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace))
+}
+
+func TestInstallNoNamespace(t *testing.T) {
+	namespace := ""
+	basePath := filepath.Join("base", "path")
+	SetInstallNamespace(namespace)
+
+	assert.Equal(t, namespace, InstallNamespace())
+	assert.False(t, InInstallNamespace())
+	assert.Equal(t, filepath.Join(basePath, "Elastic", installDir), InstallPath(basePath))
+	assert.Equal(t, serviceName, ServiceName())
+	assert.Equal(t, serviceDisplayName, ServiceDisplayName())
+	assert.Equal(t, shellWrapperPath, ShellWrapperPath())
+	assert.Equal(t, controlSocketRunSymlink, ControlSocketRunSymlink(namespace))
+}
+
+func TestParseNamespaceFromDirName(t *testing.T) {
+	testcases := []struct {
+		name      string
+		dir       string
+		namespace string
+	}{
+		{name: "empty", dir: "", namespace: ""},
+		{name: "none", dir: "Agent", namespace: ""},
+		{name: "develop", dir: "Agent-Development", namespace: "Development"},
+		{name: "dashes", dir: "Agent-With-Dashes", namespace: "With-Dashes"},
+		{name: "special", dir: "Agent-@!$%^&*()-_+=", namespace: "@!$%^&*()-_+="},
+		{name: "format", dir: "Agent-%s%d%v%t", namespace: "%s%d%v%t"},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.namespace, parseNamespaceFromDir(tc.dir))
+
+			// Special case: if the directory is empty the install dir is the default "Agent" not "Agent-"
+			wantDir := tc.dir
+			if wantDir == "" {
+				wantDir = installDir
+			}
+			assert.Equal(t, wantDir, InstallDirNameForNamespace(tc.namespace))
+		})
+	}
+}
diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index 5c324066759..44ed4ba3fa1 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -16,20 +16,20 @@ const (
 
 	// controlSocketRunSymlink is the path to the symlink that should be
 	// created to the control socket when Elastic Agent is running with root.
-	controlSocketRunSymlink                = "/var/run/elastic-agent.sock"
-	controlSocketRunSymlinkDevelopmentMode = "/var/run/elastic-agent-development.sock"
+	controlSocketRunSymlink             = "/var/run/elastic-agent.sock"
+	controlSocketRunSymlinkNamespaceFmt = "/var/run/elastic-agent-%s.sock"
 
 	// serviceName is the service name when installed.
-	serviceName                = "co.elastic.elastic-agent"
-	serviceNameDevelopmentMode = "co.elastic.elastic-agent-development"
+	serviceName             = "co.elastic.elastic-agent"
+	serviceNameNamespaceFmt = "co.elastic.elastic-agent-%s"
 
 	// shellWrapperPath is the path to the installed shell wrapper.
-	shellWrapperPath                = "/usr/local/bin/elastic-agent"
-	shellWrapperPathDevelopmentMode = "/usr/local/bin/elastic-development-agent"
+	shellWrapperPath             = "/usr/local/bin/elastic-agent"
+	shellWrapperPathNamespaceFmt = "/usr/local/bin/elastic-%s-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
-	ShellWrapper = `#!/bin/sh
+	ShellWrapperFmt = `#!/bin/sh
 exec %s/elastic-agent $@
 `
 )
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index 940609650e0..5f46602f833 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -15,23 +15,23 @@ const (
 	DefaultBasePath = "/opt"
 
 	// serviceName is the service name when installed.
-	serviceName                = "elastic-agent"
-	serviceNameDevelopmentMode = "elastic-agent-development"
+	serviceName             = "elastic-agent"
+	serviceNameNamespaceFmt = "elastic-agent-%s"
 
 	// shellWrapperPath is the path to the installed shell wrapper.
-	shellWrapperPath                = "/usr/bin/elastic-agent"
-	shellWrapperPathDevelopmentMode = "/usr/bin/elastic-development-agent"
+	shellWrapperPath             = "/usr/bin/elastic-agent"
+	shellWrapperPathNamespaceFmt = "/usr/bin/elastic-%s-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
-	ShellWrapper = `#!/bin/sh
+	ShellWrapperFmt = `#!/bin/sh
 exec %s/elastic-agent $@
 `
 
 	// controlSocketRunSymlink is the path to the symlink that should be
 	// created to the control socket when Elastic Agent is running with root.
-	controlSocketRunSymlink                = "/run/elastic-agent.sock"
-	controlSocketRunSymlinkDevelopmentMode = "/run/elastic-agent-development.sock"
+	controlSocketRunSymlink             = "/run/elastic-agent.sock"
+	controlSocketRunSymlinkNamespaceFmt = "/run/elastic-agent-%s.sock"
 )
 
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index 683d110147a..7f93e2916a5 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -21,19 +21,19 @@ const (
 	DefaultBasePath = `C:\Program Files`
 
 	// controlSocketRunSymlink is not created on Windows.
-	controlSocketRunSymlink                = ""
-	controlSocketRunSymlinkDevelopmentMode = ""
+	controlSocketRunSymlink             = ""
+	controlSocketRunSymlinkNamespaceFmt = ""
 
 	// serviceName is the service name when installed.
-	serviceName                = "Elastic Agent"
-	serviceNameDevelopmentMode = "Elastic Development Agent"
+	serviceName             = "Elastic Agent"
+	serviceNameNamespaceFmt = "Elastic Agent - %s"
 
 	// shellWrapperPath is the path to the installed shell wrapper.
-	shellWrapperPath                = ""
-	shellWrapperPathDevelopmentMode = ""
+	shellWrapperPath             = ""
+	shellWrapperPathNamespaceFmt = ""
 
 	// ShellWrapper is the wrapper that is installed.
-	ShellWrapper = "" // no wrapper on Windows
+	ShellWrapperFmt = "" // no wrapper on Windows
 )
 
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
diff --git a/internal/pkg/agent/cmd/enroll_cmd.go b/internal/pkg/agent/cmd/enroll_cmd.go
index 5b2ec2c812f..ec0fb15bc87 100644
--- a/internal/pkg/agent/cmd/enroll_cmd.go
+++ b/internal/pkg/agent/cmd/enroll_cmd.go
@@ -556,10 +556,10 @@ func (c *enrollCmd) enroll(ctx context.Context, persistentConfig map[string]inte
 		return errors.New(err, "acquiring metadata failed")
 	}
 
-	// Automatically add a "development" tag when enrolling in development mode.
+	// Automatically add the namespace as a tag when installed into a namepsace.
 	// Ensures the development agent is differentiated from others when on the same host.
-	if paths.IsDevelopmentMode() {
-		c.options.Tags = append(c.options.Tags, "development")
+	if namespace := paths.InstallNamespace(); namespace != "" {
+		c.options.Tags = append(c.options.Tags, namespace)
 	}
 
 	r := &fleetapi.EnrollRequest{
diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index 9d895c99bc2..5f09bdce335 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -86,8 +86,9 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 	isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
 	if isDevelopmentMode {
 		fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental and currently unsupported feature.")
+		// For now, development mode only installs agent in a well known namespace to allow two agents on the same machine.
+		paths.SetInstallNamespace(paths.DevelopmentNamespace)
 	}
-	paths.SetIsDevelopmentMode(isDevelopmentMode)
 
 	topPath := paths.InstallPath(basePath)
 
diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go
index 4157cd558a2..ab007728630 100644
--- a/internal/pkg/agent/cmd/run.go
+++ b/internal/pkg/agent/cmd/run.go
@@ -73,8 +73,10 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 			isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
 			if isDevelopmentMode {
 				fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental feature.")
+				// For now, development mode only makes the agent behave as if it was running in a namespace to allow
+				// multiple agents on the same machine.
+				paths.SetInstallNamespace(paths.DevelopmentNamespace)
 			}
-			paths.SetIsDevelopmentMode(isDevelopmentMode)
 
 			// done very early so the encrypted store is never used. Always done in development mode to remove the need to be root.
 			disableEncryptedStore, _ := cmd.Flags().GetBool("disable-encrypted-store")
@@ -322,7 +324,7 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf
 	// option during installation
 	//
 	// Windows `paths.ControlSocketRunSymlink()` is `""` so this is always skipped on Windows.
-	controlSocketRunSymlink := paths.ControlSocketRunSymlink(paths.IsDevelopmentMode())
+	controlSocketRunSymlink := paths.ControlSocketRunSymlink(paths.InstallNamespace())
 	if isRoot && paths.RunningInstalled() && controlSocketRunSymlink != "" {
 		socketPath := strings.TrimPrefix(paths.ControlSocket(), "unix://")
 		socketLog := controlLog.With("path", socketPath).With("link", controlSocketRunSymlink)
diff --git a/internal/pkg/agent/configuration/grpc.go b/internal/pkg/agent/configuration/grpc.go
index 0b887b28717..28f78412726 100644
--- a/internal/pkg/agent/configuration/grpc.go
+++ b/internal/pkg/agent/configuration/grpc.go
@@ -20,21 +20,23 @@ type GRPCConfig struct {
 
 // DefaultGRPCConfig creates a default server configuration.
 func DefaultGRPCConfig() *GRPCConfig {
-	// In development mode bind to port zero to select a random free port to avoid collisions with
-	// any already installed Elastic Agent. Ideally we'd always bind to port zero, but this would be
-	// breaking for users that had to manually whitelist the gRPC port in local firewall rules.
+	// When in an installation namepspace, bind to port zero to select a random free port to avoid
+	// collisions with any already installed Elastic Agent. Ideally we'd always bind to port zero,
+	// but this would be breaking for users that had to manually whitelist the gRPC port in local
+	// firewall rules.
 	//
-	// Note: this uses local TCP by default. A port of -1 switches to unix domain sockets / named pipes.
-	// Using domain sockets by default is preferable but is currently blocked because the gRPC library
-	// endpoint security uses does not support Windows named pipes.
+	// Note: this uses local TCP by default. A port of -1 switches to unix domain sockets / named
+	// pipes. Using domain sockets by default is preferable but is currently blocked because the
+	// gRPC library endpoint security uses does not support Windows named pipes.
 	defaultPort := uint16(6789)
-	if paths.IsDevelopmentMode() {
+	if paths.InInstallNamespace() {
 		defaultPort = 0
 	}
 
 	return &GRPCConfig{
 		Port:                    defaultPort,
-		CheckinChunkingDisabled: false, // on by default
+		MaxMsgSize:              1024 * 1024 * 100, // grpc default 4MB is unsufficient for diagnostics
+		CheckinChunkingDisabled: false,             // on by default
 	}
 }
 
diff --git a/internal/pkg/agent/install/install.go b/internal/pkg/agent/install/install.go
index 27aff10f003..4d23ee724f9 100644
--- a/internal/pkg/agent/install/install.go
+++ b/internal/pkg/agent/install/install.go
@@ -160,7 +160,7 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 			// We use strings.Replace instead of fmt.Sprintf here because, with the
 			// latter, govet throws a false positive error here: "fmt.Sprintf call has
 			// arguments but no formatting directives".
-			shellWrapper := strings.Replace(paths.ShellWrapper, "%s", topPath, -1)
+			shellWrapper := strings.Replace(paths.ShellWrapperFmt, "%s", topPath, -1)
 			err = os.WriteFile(paths.ShellWrapperPath(), []byte(shellWrapper), 0755)
 			if err != nil {
 				return utils.FileOwner{}, errors.New(
diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go
index 5b3afe750a5..cbc98757f86 100644
--- a/pkg/testing/fixture.go
+++ b/pkg/testing/fixture.go
@@ -833,7 +833,7 @@ func (f *Fixture) binaryPath() string {
 	if f.installed {
 		installDir := "Agent"
 		if f.installOpts != nil && f.installOpts.Develop {
-			installDir = paths.DevelopmentInstallDirName
+			installDir = paths.InstallDirNameForNamespace(paths.DevelopmentNamespace)
 		}
 
 		if f.installOpts != nil && f.installOpts.BasePath != "" {
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index df7bf877cab..f6ecdcd947f 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -187,8 +187,10 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 	f.installOpts = installOpts
 
 	installDir := "Agent"
+	socketRunSymlink := paths.ControlSocketRunSymlink("")
 	if installOpts.Develop {
-		installDir = paths.DevelopmentInstallDirName
+		installDir = paths.InstallDirNameForNamespace(paths.DevelopmentNamespace)
+		socketRunSymlink = paths.ControlSocketRunSymlink(paths.DevelopmentNamespace)
 	}
 
 	if installOpts.BasePath == "" {
@@ -198,7 +200,7 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 	}
 
 	// we just installed agent, the control socket is at a well-known location
-	socketPath := fmt.Sprintf("unix://%s", paths.ControlSocketRunSymlink(installOpts.Develop)) // use symlink as that works for all versions
+	socketPath := fmt.Sprintf("unix://%s", socketRunSymlink) // use symlink as that works for all versions
 	if runtime.GOOS == "windows" {
 		// Windows uses a fixed named pipe, that is always the same.
 		// It is the same even running in unprivileged mode.
@@ -258,7 +260,7 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 			assert.LessOrEqual(f.t, len(processes), 1, "More than one agent left running at the end of the test when --develop was used: %v", processes)
 			// The agent left running has to be the non-development agent. The development agent should be uninstalled first as a convention.
 			if len(processes) > 0 {
-				assert.NotContains(f.t, processes[0].Cmdline, paths.DevelopmentInstallDirName,
+				assert.NotContains(f.t, processes[0].Cmdline, paths.InstallDirNameForNamespace(paths.DevelopmentNamespace),
 					"The agent installed with --develop was left running at the end of the test or was not uninstalled first: %v", processes)
 			}
 			return
diff --git a/testing/installtest/checks.go b/testing/installtest/checks.go
index 7373a670ef7..b4704ee85ab 100644
--- a/testing/installtest/checks.go
+++ b/testing/installtest/checks.go
@@ -33,8 +33,8 @@ func DefaultTopPath() string {
 	return filepath.Join(defaultBasePath(), "Elastic", "Agent")
 }
 
-func DevelopTopPath() string {
-	return filepath.Join(defaultBasePath(), "Elastic", paths.DevelopmentInstallDirName)
+func NamespaceTopPath(namespace string) string {
+	return filepath.Join(defaultBasePath(), "Elastic", paths.InstallDirNameForNamespace(namespace))
 }
 
 type CheckOpts struct {
@@ -56,7 +56,7 @@ func CheckSuccess(ctx context.Context, f *atesting.Fixture, topPath string, opts
 	// Check that a few expected installed files are present
 	installedBinPath := filepath.Join(topPath, exeOnWindows("elastic-agent"))
 	installedDataPath := filepath.Join(topPath, "data")
-	installMarkerPath := filepath.Join(topPath, ".installed")
+	installMarkerPath := filepath.Join(topPath, paths.MarkerFileName)
 
 	_, err = os.Stat(installedBinPath)
 	if err != nil {
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index a71b918367d..ae7ca1080ec 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -64,7 +64,8 @@ func TestInstallWithoutBasePath(t *testing.T) {
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, installtest.DevelopTopPath(), opts))
+	t.Run("check second agent installs with --develop",
+		testDevelopmentAgentCanInstall(ctx, fixture, installtest.NamespaceTopPath(paths.DevelopmentNamespace), opts))
 
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
@@ -139,7 +140,7 @@ func TestInstallWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
-	devTopPath := filepath.Join(basePath, "Elastic", paths.DevelopmentInstallDirName)
+	devTopPath := filepath.Join(basePath, "Elastic", paths.InstallDirNameForNamespace(paths.DevelopmentNamespace))
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
@@ -196,7 +197,8 @@ func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, installtest.DevelopTopPath(), opts))
+	t.Run("check second agent installs with --develop",
+		testDevelopmentAgentCanInstall(ctx, fixture, installtest.NamespaceTopPath(paths.DevelopmentNamespace), opts))
 }
 
 func TestInstallPrivilegedWithBasePath(t *testing.T) {
@@ -242,7 +244,7 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
-	devTopPath := filepath.Join(randomBasePath, "Elastic", paths.DevelopmentInstallDirName)
+	devTopPath := filepath.Join(randomBasePath, "Elastic", paths.InstallDirNameForNamespace(paths.DevelopmentNamespace))
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
 	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))

From d340e199a863f59fcbd8e6ab9a8e815649aaf95a Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 10 Jun 2024 17:21:40 -0400
Subject: [PATCH 31/49] Add nolint directives.

---
 internal/pkg/agent/application/paths/common_namespace_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index 4efacee3aed..6d28d017980 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -18,8 +18,8 @@ func TestInstallNamespace(t *testing.T) {
 	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
 	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
 	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
-	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())
-	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace))
+	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())                        //nolint:govet
+	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace)) //nolint:govet
 }
 
 func TestInstallNoNamespace(t *testing.T) {

From 3696791aaed8de0f587613ee42d23bc1d62cdccf Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 17:01:55 -0400
Subject: [PATCH 32/49] Switch from strings.Replace to fmt.Sprintf.

---
 .../agent/application/paths/common_namespace.go  | 16 ++++++----------
 .../application/paths/common_namespace_test.go   |  4 ++--
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index 9e363678364..c794b0f930f 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -7,6 +7,7 @@
 package paths
 
 import (
+	"fmt"
 	"path/filepath"
 	"strings"
 )
@@ -71,8 +72,7 @@ func InstallDirNameForNamespace(namespace string) string {
 		return installDir
 	}
 
-	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
-	return strings.Replace(installDirNamespaceFmt, "%s", namespace, 1)
+	return fmt.Sprintf(installDirNamespaceFmt, namespace)
 }
 
 // InstallPath returns the top level directory Agent will be installed into, accounting for any namespace.
@@ -92,8 +92,7 @@ func ServiceName() string {
 		return serviceName
 	}
 
-	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
-	return strings.Replace(serviceNameNamespaceFmt, "%s", namespace, 1)
+	return fmt.Sprintf(serviceNameNamespaceFmt, namespace)
 }
 
 // ServiceDisplayName returns the service display name accounting for any namespace.
@@ -103,8 +102,7 @@ func ServiceDisplayName() string {
 		return serviceDisplayName
 	}
 
-	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
-	return strings.Replace(serviceDisplayNameNamespaceFmt, "%s", namespace, 1)
+	return fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace)
 }
 
 // ShellWrapperPath returns the shell wrapper path accounting for any namespace.
@@ -115,8 +113,7 @@ func ShellWrapperPath() string {
 		return shellWrapperPath
 	}
 
-	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
-	return strings.Replace(shellWrapperPathNamespaceFmt, "%s", strings.ToLower(namespace), 1)
+	return fmt.Sprintf(shellWrapperPathNamespaceFmt, strings.ToLower(namespace)) //nolint:govet // empty format string on Windows
 }
 
 // ControlSocketRunSymlink returns the shell wrapper path accounting for any namespace.
@@ -126,6 +123,5 @@ func ControlSocketRunSymlink(namespace string) string {
 		return controlSocketRunSymlink
 	}
 
-	// Use strings.Replace() to avoid having to sanitize format specifiers in the namespace itself.
-	return strings.Replace(controlSocketRunSymlinkNamespaceFmt, "%s", namespace, 1)
+	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace) //nolint:govet // empty format string on Windows
 }
diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index 6d28d017980..14d100ba230 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -18,8 +18,8 @@ func TestInstallNamespace(t *testing.T) {
 	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
 	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
 	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
-	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())                        //nolint:govet
-	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace)) //nolint:govet
+	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())                        //nolint:govet // empty format string on Windows
+	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace)) //nolint:govet // empty format string on Windows
 }
 
 func TestInstallNoNamespace(t *testing.T) {

From 8d36a0a53c5bea629080268fa6ceb98702264818 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 17:05:22 -0400
Subject: [PATCH 33/49] Allow empty nolint directives.

Some nolints are platform dependent in cross-platform code.
---
 .golangci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.golangci.yml b/.golangci.yml
index 07598c3c89a..5c60a74412c 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -124,7 +124,7 @@ linters-settings:
 
   nolintlint:
     # Enable to ensure that nolint directives are all used. Default is true.
-    allow-unused: false
+    allow-unused: true
     # Disable to ensure that nolint directives don't have a leading space. Default is true.
     allow-leading-space: false
     # Exclude following linters from requiring an explanation.  Default is [].

From 891a9e94f2f344c0f27dd271aac5bee999592cd6 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 17:36:49 -0400
Subject: [PATCH 34/49] Enforce agent prefix. Add whitespace tests.

---
 .../application/paths/common_namespace.go     | 19 ++++++++++++-------
 .../paths/common_namespace_test.go            | 11 +++++++++--
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index c794b0f930f..dc7719faff8 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -12,14 +12,13 @@ import (
 	"strings"
 )
 
-// installNamespace is the name of the agent's current installation namepsace.
-var installNamespace string
-
 const (
 	// installDirNamespaceFmt is the format of the directory agent will be installed to within the base path when using an installation namepsace.
-	// For example it is $BasePath/$DevelopmentInstallDirName, on MacOS it is /Library/Elastic/$DevelopmentInstallDirName.
-	installDir             = "Agent"
-	installDirNamespaceFmt = "Agent-%s"
+	// It is $BasePath/Agent-$namespace.
+	installDir                = "Agent"
+	installDirNamespaceSep    = "-"
+	installDirNamespacePrefix = installDir + installDirNamespaceSep
+	installDirNamespaceFmt    = installDirNamespacePrefix + "%s"
 
 	// DevelopmentNamespace defines the "well known" development namespace.
 	DevelopmentNamespace = "Development"
@@ -29,9 +28,13 @@ const (
 	serviceDisplayNameNamespaceFmt = "Elastic Agent - %s"
 )
 
+// installNamespace is the name of the agent's current installation namepsace.
+var installNamespace string
+
 // SetInstallNamespace sets whether the agent is currently in or is being installed in an installation namespace.
+// Removes leading and trailing whitespace
 func SetInstallNamespace(namespace string) {
-	installNamespace = namespace
+	installNamespace = strings.TrimSpace(namespace)
 }
 
 // InstallNamespace returns the name of the current installation namespace. Returns the empty string
@@ -55,6 +58,8 @@ func parseNamespaceFromDir(dir string) string {
 	parts := strings.SplitAfterN(dir, "-", 2)
 	if len(parts) <= 1 {
 		return ""
+	} else if parts[0] != installDirNamespacePrefix {
+		return ""
 	}
 
 	return parts[1]
diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index 14d100ba230..9b67bf65dc2 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -11,7 +11,9 @@ import (
 func TestInstallNamespace(t *testing.T) {
 	namespace := "testing"
 	basePath := filepath.Join("base", "path")
-	SetInstallNamespace(namespace)
+
+	// Add whitespace to ensure it gets removed.
+	SetInstallNamespace(" " + namespace + "\t   ")
 
 	assert.Equal(t, namespace, InstallNamespace())
 	assert.True(t, InInstallNamespace())
@@ -48,11 +50,12 @@ func TestParseNamespaceFromDirName(t *testing.T) {
 		{name: "dashes", dir: "Agent-With-Dashes", namespace: "With-Dashes"},
 		{name: "special", dir: "Agent-@!$%^&*()-_+=", namespace: "@!$%^&*()-_+="},
 		{name: "format", dir: "Agent-%s%d%v%t", namespace: "%s%d%v%t"},
+		{name: "spaces", dir: "Agent- Development \t", namespace: " Development \t"},
 	}
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
-			assert.Equal(t, tc.namespace, parseNamespaceFromDir(tc.dir))
+			assert.Equalf(t, tc.namespace, parseNamespaceFromDir(tc.dir), "parsing %s", tc.dir)
 
 			// Special case: if the directory is empty the install dir is the default "Agent" not "Agent-"
 			wantDir := tc.dir
@@ -63,3 +66,7 @@ func TestParseNamespaceFromDirName(t *testing.T) {
 		})
 	}
 }
+
+func TestParseNamespaceFromDirNameWithoutAgentPrefix(t *testing.T) {
+	assert.Equal(t, "", parseNamespaceFromDir("Beats-Development"))
+}

From ad85971064f8ae345c9e7e402693aca921621bc6 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 17:37:43 -0400
Subject: [PATCH 35/49] Fix typo.

---
 internal/pkg/agent/configuration/grpc.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/configuration/grpc.go b/internal/pkg/agent/configuration/grpc.go
index 28f78412726..5938a591d77 100644
--- a/internal/pkg/agent/configuration/grpc.go
+++ b/internal/pkg/agent/configuration/grpc.go
@@ -20,7 +20,7 @@ type GRPCConfig struct {
 
 // DefaultGRPCConfig creates a default server configuration.
 func DefaultGRPCConfig() *GRPCConfig {
-	// When in an installation namepspace, bind to port zero to select a random free port to avoid
+	// When in an installation namespace, bind to port zero to select a random free port to avoid
 	// collisions with any already installed Elastic Agent. Ideally we'd always bind to port zero,
 	// but this would be breaking for users that had to manually whitelist the gRPC port in local
 	// firewall rules.

From fc8e7dd8d127e9f83bf6ea9ee41086137c0af5f7 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 17:39:23 -0400
Subject: [PATCH 36/49] Remove unnecessary conditional check.

---
 internal/pkg/agent/application/paths/common_namespace.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index dc7719faff8..b7e47fbf91a 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -83,10 +83,6 @@ func InstallDirNameForNamespace(namespace string) string {
 // InstallPath returns the top level directory Agent will be installed into, accounting for any namespace.
 func InstallPath(basePath string) string {
 	namespace := InstallNamespace()
-	if namespace == "" {
-		return filepath.Join(basePath, "Elastic", installDir)
-	}
-
 	return filepath.Join(basePath, "Elastic", InstallDirNameForNamespace(namespace))
 }
 

From 2d38aec05b72a1ac7e4d4ef23caee6de2f253d86 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 17:41:14 -0400
Subject: [PATCH 37/49] Read the namespace once at startup.

---
 internal/pkg/agent/application/paths/common_namespace.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index b7e47fbf91a..df36a255b95 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -48,7 +48,9 @@ func InstallNamespace() string {
 	}
 
 	if RunningInstalled() {
-		return parseNamespaceFromDir(filepath.Base(Top()))
+		// Parse the namespace from the directory once to ensure deterministic behavior from startup.
+		namespace := parseNamespaceFromDir(filepath.Base(Top()))
+		installNamespace = namespace
 	}
 
 	return ""

From 6e98115409cc43831e45d55143129eb3cb53ad06 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 19:55:37 -0400
Subject: [PATCH 38/49] Add missing license header.

---
 internal/pkg/agent/application/paths/common_namespace_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index 9b67bf65dc2..2eab16c8a05 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package paths
 
 import (

From 35bebe8a05e482a89d9b4e65d09f39819aef61f0 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Tue, 11 Jun 2024 19:56:55 -0400
Subject: [PATCH 39/49] Also disable staticcheck for Windows lint warnings.

---
 internal/pkg/agent/application/paths/common_namespace_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index 2eab16c8a05..e2d6146c0b3 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -24,8 +24,8 @@ func TestInstallNamespace(t *testing.T) {
 	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
 	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
 	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
-	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())                        //nolint:govet // empty format string on Windows
-	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace)) //nolint:govet // empty format string on Windows
+	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())                        //nolint:govet,staticcheck // empty format string on Windows
+	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace)) //nolint:govet,staticcheck // empty format string on Windows
 }
 
 func TestInstallNoNamespace(t *testing.T) {

From b318abd10b3bc103f302b1b1bab230c60efc8284 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 12 Jun 2024 10:10:54 -0400
Subject: [PATCH 40/49] Revert "Allow empty nolint directives."

This reverts commit 8d36a0a53c5bea629080268fa6ceb98702264818.
---
 .golangci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.golangci.yml b/.golangci.yml
index 5c60a74412c..07598c3c89a 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -124,7 +124,7 @@ linters-settings:
 
   nolintlint:
     # Enable to ensure that nolint directives are all used. Default is true.
-    allow-unused: true
+    allow-unused: false
     # Disable to ensure that nolint directives don't have a leading space. Default is true.
     allow-leading-space: false
     # Exclude following linters from requiring an explanation.  Default is [].

From 18904f8d515e3eade1b7d2c3adf8961fbb451365 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 12 Jun 2024 10:34:56 -0400
Subject: [PATCH 41/49] Better handling of empty fmt strings on windows.

---
 .../agent/application/paths/common_namespace.go  | 10 ++++++++--
 .../application/paths/common_namespace_test.go   | 16 ++++++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index df36a255b95..5be28d89444 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -114,9 +114,12 @@ func ShellWrapperPath() string {
 	namespace := InstallNamespace()
 	if namespace == "" {
 		return shellWrapperPath
+	} else if shellWrapperPathNamespaceFmt == "" {
+		// No shell wrapper path on Windows.
+		return ""
 	}
 
-	return fmt.Sprintf(shellWrapperPathNamespaceFmt, strings.ToLower(namespace)) //nolint:govet // empty format string on Windows
+	return fmt.Sprintf(shellWrapperPathNamespaceFmt, strings.ToLower(namespace))
 }
 
 // ControlSocketRunSymlink returns the shell wrapper path accounting for any namespace.
@@ -124,7 +127,10 @@ func ShellWrapperPath() string {
 func ControlSocketRunSymlink(namespace string) string {
 	if namespace == "" {
 		return controlSocketRunSymlink
+	} else if controlSocketRunSymlinkNamespaceFmt == "" {
+		// No control socket run symlink on Windows.
+		return ""
 	}
 
-	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace) //nolint:govet // empty format string on Windows
+	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
 }
diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index e2d6146c0b3..b907d0e69ac 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -24,8 +24,20 @@ func TestInstallNamespace(t *testing.T) {
 	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
 	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
 	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
-	assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())                        //nolint:govet,staticcheck // empty format string on Windows
-	assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace)) //nolint:govet,staticcheck // empty format string on Windows
+
+	// No shell wrapper path on Windows.
+	if shellWrapperPathNamespaceFmt == "" {
+		assert.Empty(t, ShellWrapperPath())
+	} else {
+		assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())
+	}
+
+	// No control sicket run symlink on Windows.
+	if controlSocketRunSymlinkNamespaceFmt == "" {
+		assert.Empty(t, ShellWrapperPath())
+	} else {
+		assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace))
+	}
 }
 
 func TestInstallNoNamespace(t *testing.T) {

From 967aabf3803d321129704cefad6491030c32046a Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 12 Jun 2024 10:54:08 -0400
Subject: [PATCH 42/49] Fix use of hard coded service name.

---
 internal/pkg/agent/cmd/install.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index ffd008ee0cb..3d69591d1aa 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -232,7 +232,7 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 			err = install.StartService(topPath)
 			if err != nil {
 				progBar.Describe("Start Service failed, exiting...")
-				fmt.Fprintf(streams.Out, "Installation failed to start Elastic Agent service.\n")
+				fmt.Fprintf(streams.Out, "Installation failed to start '%s' service.\n", paths.ServiceName())
 				return fmt.Errorf("error starting service: %w", err)
 			}
 			progBar.Describe("Service Started")

From d5d3e3b4af5af0ba6ffd6f4a1af9ff4625434e31 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 12 Jun 2024 12:04:48 -0400
Subject: [PATCH 43/49] Fix merge errors in integration tests.

---
 testing/integration/switch_privileged_test.go   | 8 ++++----
 testing/integration/switch_unprivileged_test.go | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/testing/integration/switch_privileged_test.go b/testing/integration/switch_privileged_test.go
index 9a5acb73965..e11c3a1438c 100644
--- a/testing/integration/switch_privileged_test.go
+++ b/testing/integration/switch_privileged_test.go
@@ -55,7 +55,7 @@ func TestSwitchPrivilegedWithoutBasePath(t *testing.T) {
 	}
 
 	// Check that Agent was installed in default base path in unprivileged mode
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, true))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: false}))
 
 	// Switch to privileged mode
 	out, err = fixture.Exec(ctx, []string{"privileged", "-f"})
@@ -65,7 +65,7 @@ func TestSwitchPrivilegedWithoutBasePath(t *testing.T) {
 	}
 
 	// Check that Agent is running in default base path in privileged mode
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, false))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: true}))
 }
 
 func TestSwitchPrivilegedWithBasePath(t *testing.T) {
@@ -125,7 +125,7 @@ func TestSwitchPrivilegedWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path in unprivileged mode
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, true))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: false}))
 
 	// Switch to privileged mode
 	out, err = fixture.Exec(ctx, []string{"privileged", "-f"})
@@ -135,5 +135,5 @@ func TestSwitchPrivilegedWithBasePath(t *testing.T) {
 	}
 
 	// Check that Agent is running in the custom base path in privileged mode
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, false))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: true}))
 }
diff --git a/testing/integration/switch_unprivileged_test.go b/testing/integration/switch_unprivileged_test.go
index 9429a59613b..ea2fdcc060f 100644
--- a/testing/integration/switch_unprivileged_test.go
+++ b/testing/integration/switch_unprivileged_test.go
@@ -55,7 +55,7 @@ func TestSwitchUnprivilegedWithoutBasePath(t *testing.T) {
 	}
 
 	// Check that Agent was installed in default base path in privileged mode
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, false))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: true}))
 
 	// Switch to unprivileged mode
 	out, err = fixture.Exec(ctx, []string{"unprivileged", "-f"})
@@ -65,7 +65,7 @@ func TestSwitchUnprivilegedWithoutBasePath(t *testing.T) {
 	}
 
 	// Check that Agent is running in default base path in unprivileged mode
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, true))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: false}))
 }
 
 func TestSwitchUnprivilegedWithBasePath(t *testing.T) {
@@ -125,7 +125,7 @@ func TestSwitchUnprivilegedWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path in privileged mode
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, false))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: true}))
 
 	// Switch to unprivileged mode
 	out, err = fixture.Exec(ctx, []string{"unprivileged", "-f"})
@@ -135,5 +135,5 @@ func TestSwitchUnprivilegedWithBasePath(t *testing.T) {
 	}
 
 	// Check that Agent is running in the custom base path in unprivileged mode
-	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, true))
+	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: false}))
 }

From 5f2eab85b5f0ebf9430937c7f9226298589079b0 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Wed, 12 Jun 2024 13:59:10 -0400
Subject: [PATCH 44/49] Properly handle empty format string on Windows.

---
 .../application/paths/common_namespace.go      | 10 ++--------
 .../application/paths/common_namespace_test.go | 16 ++--------------
 .../agent/application/paths/paths_darwin.go    | 16 ++++++++++++++++
 .../pkg/agent/application/paths/paths_linux.go | 16 ++++++++++++++++
 .../agent/application/paths/paths_windows.go   | 18 ++++++++++++++----
 5 files changed, 50 insertions(+), 26 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index 5be28d89444..957a139b78a 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -114,12 +114,9 @@ func ShellWrapperPath() string {
 	namespace := InstallNamespace()
 	if namespace == "" {
 		return shellWrapperPath
-	} else if shellWrapperPathNamespaceFmt == "" {
-		// No shell wrapper path on Windows.
-		return ""
 	}
 
-	return fmt.Sprintf(shellWrapperPathNamespaceFmt, strings.ToLower(namespace))
+	return shellWrapperPathForNamespace(strings.ToLower(namespace))
 }
 
 // ControlSocketRunSymlink returns the shell wrapper path accounting for any namespace.
@@ -127,10 +124,7 @@ func ShellWrapperPath() string {
 func ControlSocketRunSymlink(namespace string) string {
 	if namespace == "" {
 		return controlSocketRunSymlink
-	} else if controlSocketRunSymlinkNamespaceFmt == "" {
-		// No control socket run symlink on Windows.
-		return ""
 	}
 
-	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
+	return controlSocketRunSymlinkForNamespace(namespace)
 }
diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index b907d0e69ac..1fd130deb70 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -24,20 +24,8 @@ func TestInstallNamespace(t *testing.T) {
 	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
 	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
 	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
-
-	// No shell wrapper path on Windows.
-	if shellWrapperPathNamespaceFmt == "" {
-		assert.Empty(t, ShellWrapperPath())
-	} else {
-		assert.Equal(t, fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace), ShellWrapperPath())
-	}
-
-	// No control sicket run symlink on Windows.
-	if controlSocketRunSymlinkNamespaceFmt == "" {
-		assert.Empty(t, ShellWrapperPath())
-	} else {
-		assert.Equal(t, fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace), ControlSocketRunSymlink(namespace))
-	}
+	assert.Equal(t, shellWrapperPathForNamespace(namespace), ShellWrapperPath())
+	assert.Equal(t, controlSocketRunSymlinkForNamespace(namespace), ControlSocketRunSymlink(namespace))
 }
 
 func TestInstallNoNamespace(t *testing.T) {
diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index 44ed4ba3fa1..01482f0ea26 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -6,6 +6,10 @@
 
 package paths
 
+import (
+	"fmt"
+)
+
 const (
 	// BinaryName is the name of the installed binary.
 	BinaryName = "elastic-agent"
@@ -34,6 +38,18 @@ exec %s/elastic-agent $@
 `
 )
 
+// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since shellWrapperPathNamespaceFmt is empty on Windows.
+func shellWrapperPathForNamespace(namespace string) string {
+	return fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace)
+}
+
+// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since controlSocketRunSymlinkNamespaceFmt is empty on Windows.
+func controlSocketRunSymlinkForNamespace(namespace string) string {
+	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
+}
+
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
 func ArePathsEqual(expected, actual string) bool {
 	return expected == actual
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index 5f46602f833..7095655b439 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -6,6 +6,10 @@
 
 package paths
 
+import (
+	"fmt"
+)
+
 const (
 	// BinaryName is the name of the installed binary.
 	BinaryName = "elastic-agent"
@@ -34,6 +38,18 @@ exec %s/elastic-agent $@
 	controlSocketRunSymlinkNamespaceFmt = "/run/elastic-agent-%s.sock"
 )
 
+// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since shellWrapperPathNamespaceFmt is empty on Windows.
+func shellWrapperPathForNamespace(namespace string) string {
+	return fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace)
+}
+
+// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since controlSocketRunSymlinkNamespaceFmt is empty on Windows.
+func controlSocketRunSymlinkForNamespace(namespace string) string {
+	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
+}
+
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
 func ArePathsEqual(expected, actual string) bool {
 	return expected == actual
diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index 7f93e2916a5..04cf1125d00 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -21,21 +21,31 @@ const (
 	DefaultBasePath = `C:\Program Files`
 
 	// controlSocketRunSymlink is not created on Windows.
-	controlSocketRunSymlink             = ""
-	controlSocketRunSymlinkNamespaceFmt = ""
+	controlSocketRunSymlink = ""
 
 	// serviceName is the service name when installed.
 	serviceName             = "Elastic Agent"
 	serviceNameNamespaceFmt = "Elastic Agent - %s"
 
 	// shellWrapperPath is the path to the installed shell wrapper.
-	shellWrapperPath             = ""
-	shellWrapperPathNamespaceFmt = ""
+	shellWrapperPath = ""
 
 	// ShellWrapper is the wrapper that is installed.
 	ShellWrapperFmt = "" // no wrapper on Windows
 )
 
+// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since shellWrapperPath is empty on Windows.
+func shellWrapperPathForNamespace(namespace string) string {
+	return ""
+}
+
+// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since controlSocketRunSymlink is empty on Windows.
+func controlSocketRunSymlinkForNamespace(namespace string) string {
+	return ""
+}
+
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
 func ArePathsEqual(expected, actual string) bool {
 	return strings.EqualFold(expected, actual)

From 3efe29daa77f4dfbd94e0407071ab0586e4a7466 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 13 Jun 2024 13:05:48 -0400
Subject: [PATCH 45/49] Add Address dropped in merge conflict resolution.

---
 internal/pkg/agent/configuration/grpc.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal/pkg/agent/configuration/grpc.go b/internal/pkg/agent/configuration/grpc.go
index 5938a591d77..f60b7d8de1f 100644
--- a/internal/pkg/agent/configuration/grpc.go
+++ b/internal/pkg/agent/configuration/grpc.go
@@ -34,6 +34,7 @@ func DefaultGRPCConfig() *GRPCConfig {
 	}
 
 	return &GRPCConfig{
+		Address:                 "localhost",
 		Port:                    defaultPort,
 		MaxMsgSize:              1024 * 1024 * 100, // grpc default 4MB is unsufficient for diagnostics
 		CheckinChunkingDisabled: false,             // on by default

From 414ae849ebd5d67b9f6e369753607083a6c013f9 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Thu, 13 Jun 2024 17:11:07 -0400
Subject: [PATCH 46/49] Fix ServiceName uses after merge.

---
 internal/pkg/agent/cmd/run_windows.go         | 2 +-
 internal/pkg/agent/install/install_windows.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/agent/cmd/run_windows.go b/internal/pkg/agent/cmd/run_windows.go
index 94505d87d4b..308de4f1a57 100644
--- a/internal/pkg/agent/cmd/run_windows.go
+++ b/internal/pkg/agent/cmd/run_windows.go
@@ -16,7 +16,7 @@ import (
 // the Application EventLog.  This is a best effort logger and no
 // errors are returned.
 func logExternal(msg string) {
-	eLog, err2 := eventlog.Open(paths.ServiceName)
+	eLog, err2 := eventlog.Open(paths.ServiceName())
 	if err2 != nil {
 		return
 	}
diff --git a/internal/pkg/agent/install/install_windows.go b/internal/pkg/agent/install/install_windows.go
index 23b0f507f56..cc53fcf670f 100644
--- a/internal/pkg/agent/install/install_windows.go
+++ b/internal/pkg/agent/install/install_windows.go
@@ -83,7 +83,7 @@ func withServiceOptions(username string, groupName string) ([]serviceOpt, error)
 // ReExec is not possible on Windows.
 func serviceConfigure(ownership utils.FileOwner) error {
 	// Modify registry to allow logging to eventlog as "Elastic Agent".
-	err := eventlog.InstallAsEventCreate(paths.ServiceName, eventlog.Info|eventlog.Warning|eventlog.Error)
+	err := eventlog.InstallAsEventCreate(paths.ServiceName(), eventlog.Info|eventlog.Warning|eventlog.Error)
 	if err != nil && !strings.Contains(err.Error(), "registry key already exists") {
 		return fmt.Errorf("unable to create registry key for logging: %w", err)
 	}

From f69ccfe7347ae871679836737729e723133a9e30 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 17 Jun 2024 13:53:27 -0400
Subject: [PATCH 47/49] Add --namespace installation option.

---
 internal/pkg/agent/cmd/install.go | 15 +++++++++++++--
 internal/pkg/agent/cmd/run.go     |  2 +-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index afb10fe1123..bbd44a230c7 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -26,6 +26,7 @@ const (
 	flagInstallBasePath     = "base-path"
 	flagInstallUnprivileged = "unprivileged"
 	flagInstallDevelopment  = "develop"
+	flagInstallNamespace    = "namespace"
 )
 
 func newInstallCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
@@ -50,7 +51,10 @@ would like the Agent to operate.
 	cmd.Flags().String(flagInstallBasePath, paths.DefaultBasePath, "The path where the Elastic Agent will be installed. It must be an absolute path.")
 	cmd.Flags().Bool(flagInstallUnprivileged, false, "Install in unprivileged mode, limiting the access of the Elastic Agent. (beta)")
 
-	cmd.Flags().Bool(flagInstallDevelopment, false, "Install an isolated Elastic Agent for development. Allows a non-development agent to be installed already. (experimental)")
+	cmd.Flags().Bool(flagInstallNamespace, false, "Install into an isolated namespace. Allows multiple Elastic Agents to be installed at once. (experimental)")
+	_ = cmd.Flags().MarkHidden(flagInstallNamespace) // For internal use only.
+
+	cmd.Flags().Bool(flagInstallDevelopment, false, "Install into a standardized development namespace, may enable development specific options. Allows multiple Elastic Agents to be installed at once. (experimental)")
 	_ = cmd.Flags().MarkHidden(flagInstallDevelopment) // For internal use only.
 
 	addEnrollFlags(cmd)
@@ -86,11 +90,18 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	isDevelopmentMode, _ := cmd.Flags().GetBool(flagInstallDevelopment)
 	if isDevelopmentMode {
-		fmt.Fprintln(streams.Out, "Development installation mode enabled; this is an experimental and currently unsupported feature.")
+		fmt.Fprintln(streams.Out, "Installing into development namespace; this is an experimental and currently unsupported feature.")
 		// For now, development mode only installs agent in a well known namespace to allow two agents on the same machine.
 		paths.SetInstallNamespace(paths.DevelopmentNamespace)
 	}
 
+	namespace, _ := cmd.Flags().GetString(flagInstallNamespace)
+	if namespace != "" {
+		fmt.Fprintf(streams.Out, "Installing into namespace '%s'; this is an experimental and currently unsupported feature.\n", namespace)
+		// Overrides the development namespace if namespace was specified separately.
+		paths.SetInstallNamespace(namespace)
+	}
+
 	topPath := paths.InstallPath(basePath)
 
 	status, reason := install.Status(topPath)
diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go
index a3d084c7714..bd855c71ad0 100644
--- a/internal/pkg/agent/cmd/run.go
+++ b/internal/pkg/agent/cmd/run.go
@@ -114,7 +114,7 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 	cmd.Flags().Duration("fleet-init-timeout", envTimeout(fleetInitTimeoutName), " Sets the initial timeout when starting up the fleet server under agent")
 	_ = cmd.Flags().MarkHidden("testing-mode")
 
-	cmd.Flags().Bool(flagRunDevelopment, false, "Run agent in development mode. Allows running when there is already and installed Elastic Agent. (experimental)")
+	cmd.Flags().Bool(flagRunDevelopment, false, "Run agent in development mode. Allows running when there is already an installed Elastic Agent. (experimental)")
 	_ = cmd.Flags().MarkHidden(flagRunDevelopment) // For internal use only.
 
 	return cmd

From d4efa5ffc033ac45938d35fc83632e6435518e19 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 17 Jun 2024 14:51:54 -0400
Subject: [PATCH 48/49] Use --namespace in integration tests.

---
 .../application/paths/common_namespace.go     |  2 +-
 .../paths/common_namespace_test.go            |  2 +-
 .../agent/application/paths/paths_darwin.go   | 16 ---------
 .../agent/application/paths/paths_linux.go    | 16 ---------
 .../pkg/agent/application/paths/paths_unix.go | 17 ++++++++++
 .../agent/application/paths/paths_windows.go  |  4 +--
 pkg/testing/fixture.go                        |  4 +--
 pkg/testing/fixture_install.go                | 30 ++++++++++------
 testing/installtest/checks.go                 |  2 +-
 testing/installtest/checks_unix.go            |  4 +--
 testing/integration/install_test.go           | 34 ++++++++++++-------
 11 files changed, 66 insertions(+), 65 deletions(-)

diff --git a/internal/pkg/agent/application/paths/common_namespace.go b/internal/pkg/agent/application/paths/common_namespace.go
index 957a139b78a..8b2651a30be 100644
--- a/internal/pkg/agent/application/paths/common_namespace.go
+++ b/internal/pkg/agent/application/paths/common_namespace.go
@@ -116,7 +116,7 @@ func ShellWrapperPath() string {
 		return shellWrapperPath
 	}
 
-	return shellWrapperPathForNamespace(strings.ToLower(namespace))
+	return ShellWrapperPathForNamespace(namespace)
 }
 
 // ControlSocketRunSymlink returns the shell wrapper path accounting for any namespace.
diff --git a/internal/pkg/agent/application/paths/common_namespace_test.go b/internal/pkg/agent/application/paths/common_namespace_test.go
index 1fd130deb70..6be3ffa72b8 100644
--- a/internal/pkg/agent/application/paths/common_namespace_test.go
+++ b/internal/pkg/agent/application/paths/common_namespace_test.go
@@ -24,7 +24,7 @@ func TestInstallNamespace(t *testing.T) {
 	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
 	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
 	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
-	assert.Equal(t, shellWrapperPathForNamespace(namespace), ShellWrapperPath())
+	assert.Equal(t, ShellWrapperPathForNamespace(namespace), ShellWrapperPath())
 	assert.Equal(t, controlSocketRunSymlinkForNamespace(namespace), ControlSocketRunSymlink(namespace))
 }
 
diff --git a/internal/pkg/agent/application/paths/paths_darwin.go b/internal/pkg/agent/application/paths/paths_darwin.go
index 01482f0ea26..44ed4ba3fa1 100644
--- a/internal/pkg/agent/application/paths/paths_darwin.go
+++ b/internal/pkg/agent/application/paths/paths_darwin.go
@@ -6,10 +6,6 @@
 
 package paths
 
-import (
-	"fmt"
-)
-
 const (
 	// BinaryName is the name of the installed binary.
 	BinaryName = "elastic-agent"
@@ -38,18 +34,6 @@ exec %s/elastic-agent $@
 `
 )
 
-// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
-// unconditionally since shellWrapperPathNamespaceFmt is empty on Windows.
-func shellWrapperPathForNamespace(namespace string) string {
-	return fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace)
-}
-
-// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
-// unconditionally since controlSocketRunSymlinkNamespaceFmt is empty on Windows.
-func controlSocketRunSymlinkForNamespace(namespace string) string {
-	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
-}
-
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
 func ArePathsEqual(expected, actual string) bool {
 	return expected == actual
diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go
index 7095655b439..5f46602f833 100644
--- a/internal/pkg/agent/application/paths/paths_linux.go
+++ b/internal/pkg/agent/application/paths/paths_linux.go
@@ -6,10 +6,6 @@
 
 package paths
 
-import (
-	"fmt"
-)
-
 const (
 	// BinaryName is the name of the installed binary.
 	BinaryName = "elastic-agent"
@@ -38,18 +34,6 @@ exec %s/elastic-agent $@
 	controlSocketRunSymlinkNamespaceFmt = "/run/elastic-agent-%s.sock"
 )
 
-// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
-// unconditionally since shellWrapperPathNamespaceFmt is empty on Windows.
-func shellWrapperPathForNamespace(namespace string) string {
-	return fmt.Sprintf(shellWrapperPathNamespaceFmt, namespace)
-}
-
-// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
-// unconditionally since controlSocketRunSymlinkNamespaceFmt is empty on Windows.
-func controlSocketRunSymlinkForNamespace(namespace string) string {
-	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
-}
-
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.
 func ArePathsEqual(expected, actual string) bool {
 	return expected == actual
diff --git a/internal/pkg/agent/application/paths/paths_unix.go b/internal/pkg/agent/application/paths/paths_unix.go
index 99d8cbe1ad5..164b7575b42 100644
--- a/internal/pkg/agent/application/paths/paths_unix.go
+++ b/internal/pkg/agent/application/paths/paths_unix.go
@@ -7,10 +7,27 @@
 package paths
 
 import (
+	"fmt"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
+const ()
+
+// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since shellWrapperPathNamespaceFmt is empty on Windows. The provided namespace is
+// always lowercased for consistency.
+func ShellWrapperPathForNamespace(namespace string) string {
+	return fmt.Sprintf(shellWrapperPathNamespaceFmt, strings.ToLower(namespace))
+}
+
+// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since controlSocketRunSymlinkNamespaceFmt is empty on Windows.
+func controlSocketRunSymlinkForNamespace(namespace string) string {
+	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
+}
+
 func initialControlSocketPath(topPath string) string {
 	return ControlSocketFromPath(runtime.GOOS, topPath)
 }
diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go
index 04cf1125d00..4734a34dd9f 100644
--- a/internal/pkg/agent/application/paths/paths_windows.go
+++ b/internal/pkg/agent/application/paths/paths_windows.go
@@ -34,9 +34,9 @@ const (
 	ShellWrapperFmt = "" // no wrapper on Windows
 )
 
-// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
+// ShellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
 // unconditionally since shellWrapperPath is empty on Windows.
-func shellWrapperPathForNamespace(namespace string) string {
+func ShellWrapperPathForNamespace(namespace string) string {
 	return ""
 }
 
diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go
index cbc98757f86..d2e5b2e40f5 100644
--- a/pkg/testing/fixture.go
+++ b/pkg/testing/fixture.go
@@ -832,8 +832,8 @@ func (f *Fixture) binaryPath() string {
 	workDir := f.workDir
 	if f.installed {
 		installDir := "Agent"
-		if f.installOpts != nil && f.installOpts.Develop {
-			installDir = paths.InstallDirNameForNamespace(paths.DevelopmentNamespace)
+		if f.installOpts != nil && f.installOpts.Namespace != "" {
+			installDir = paths.InstallDirNameForNamespace(f.installOpts.Namespace)
 		}
 
 		if f.installOpts != nil && f.installOpts.BasePath != "" {
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index 3a9cb21719c..fe756313a78 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -106,7 +106,8 @@ type InstallOpts struct {
 	NonInteractive bool   // --non-interactive
 	ProxyURL       string // --proxy-url
 	DelayEnroll    bool   // --delay-enroll
-	Develop        bool   // --develop, not supported for DEB and RPM.
+	Develop        bool   // --develop, not supported for DEB and RPM. Calling Install() sets Namespace to the development namespace so that checking only for a Namespace is sufficient.
+	Namespace      string // --namespace, not supported for DEB and RPM.
 
 	Privileged bool // inverse of --unprivileged (as false is the default)
 
@@ -114,7 +115,7 @@ type InstallOpts struct {
 	FleetBootstrapOpts
 }
 
-func (i InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
+func (i *InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
 	var args []string
 	if i.BasePath != "" {
 		args = append(args, "--base-path", i.BasePath)
@@ -137,8 +138,15 @@ func (i InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
 	if !i.Privileged {
 		args = append(args, "--unprivileged")
 	}
+	if i.Namespace != "" {
+		args = append(args, "--namespace="+i.Namespace)
+	}
 	if i.Develop {
 		args = append(args, "--develop")
+		if i.Namespace == "" {
+			// If --namespace was used it will override the development namespace.
+			i.Namespace = paths.DevelopmentNamespace
+		}
 	}
 
 	args = append(args, i.EnrollOpts.toCmdArgs()...)
@@ -157,8 +165,8 @@ func (i InstallOpts) toCmdArgs(operatingSystem string) ([]string, error) {
 func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ...process.CmdOption) ([]byte, error) {
 	f.t.Logf("[test %s] Inside fixture install function", f.t.Name())
 
-	// check for running agents before installing, but only if not using --develop whose point is allowing two agents at once.
-	if installOpts != nil && !installOpts.Develop {
+	// check for running agents before installing, but only if not installed into a namespace whose point is allowing two agents at once.
+	if installOpts != nil && installOpts.Namespace != "" {
 		assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at beginning of Install()")
 	}
 
@@ -205,9 +213,9 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 
 	installDir := "Agent"
 	socketRunSymlink := paths.ControlSocketRunSymlink("")
-	if installOpts.Develop {
-		installDir = paths.InstallDirNameForNamespace(paths.DevelopmentNamespace)
-		socketRunSymlink = paths.ControlSocketRunSymlink(paths.DevelopmentNamespace)
+	if installOpts.Namespace != "" {
+		installDir = paths.InstallDirNameForNamespace(installOpts.Namespace)
+		socketRunSymlink = paths.ControlSocketRunSymlink(installOpts.Namespace)
 	}
 
 	if installOpts.BasePath == "" {
@@ -273,12 +281,12 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 		processes := getElasticAgentProcesses(f.t)
 
 		// there can be a single agent left when using --develop mode
-		if f.installOpts != nil && f.installOpts.Develop {
-			assert.LessOrEqual(f.t, len(processes), 1, "More than one agent left running at the end of the test when --develop was used: %v", processes)
+		if f.installOpts != nil && f.installOpts.Namespace != "" {
+			assert.LessOrEqualf(f.t, len(processes), 1, "More than one agent left running at the end of the test when second agent in namespace %s was used: %v", f.installOpts.Namespace, processes)
 			// The agent left running has to be the non-development agent. The development agent should be uninstalled first as a convention.
 			if len(processes) > 0 {
-				assert.NotContains(f.t, processes[0].Cmdline, paths.InstallDirNameForNamespace(paths.DevelopmentNamespace),
-					"The agent installed with --develop was left running at the end of the test or was not uninstalled first: %v", processes)
+				assert.NotContainsf(f.t, processes[0].Cmdline, paths.InstallDirNameForNamespace(f.installOpts.Namespace),
+					"The agent installed into namespace %s was left running at the end of the test or was not uninstalled first: %v", f.installOpts.Namespace, processes)
 			}
 			return
 		}
diff --git a/testing/installtest/checks.go b/testing/installtest/checks.go
index b4704ee85ab..6160658b28a 100644
--- a/testing/installtest/checks.go
+++ b/testing/installtest/checks.go
@@ -39,7 +39,7 @@ func NamespaceTopPath(namespace string) string {
 
 type CheckOpts struct {
 	Privileged bool
-	Develop    bool
+	Namespace  string
 }
 
 func CheckSuccess(ctx context.Context, f *atesting.Fixture, topPath string, opts *CheckOpts) error {
diff --git a/testing/installtest/checks_unix.go b/testing/installtest/checks_unix.go
index e1503abd691..1cba1fd0925 100644
--- a/testing/installtest/checks_unix.go
+++ b/testing/installtest/checks_unix.go
@@ -65,8 +65,8 @@ func checkPlatform(ctx context.Context, _ *atesting.Fixture, topPath string, opt
 
 		// Executing `elastic-agent status` as the `elastic-agent-user` user should work.
 		shellWrapperName := "elastic-agent"
-		if opts.Develop {
-			shellWrapperName = "elastic-development-agent"
+		if opts.Namespace != "" {
+			shellWrapperName = paths.ShellWrapperPathForNamespace(opts.Namespace)
 		}
 
 		var output []byte
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
index ae7ca1080ec..f2159752943 100644
--- a/testing/integration/install_test.go
+++ b/testing/integration/install_test.go
@@ -64,8 +64,7 @@ func TestInstallWithoutBasePath(t *testing.T) {
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-	t.Run("check second agent installs with --develop",
-		testDevelopmentAgentCanInstall(ctx, fixture, installtest.NamespaceTopPath(paths.DevelopmentNamespace), opts))
+	t.Run("check second agent installs with --develop", testSecondAgentCanInstall(ctx, fixture, "", true, opts))
 
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
@@ -140,11 +139,10 @@ func TestInstallWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(basePath, "Elastic", "Agent")
-	devTopPath := filepath.Join(basePath, "Elastic", paths.InstallDirNameForNamespace(paths.DevelopmentNamespace))
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))
+	t.Run("check second agent installs with --namespace", testSecondAgentCanInstall(ctx, fixture, basePath, false, opts))
 
 	// Make sure uninstall from within the topPath fails on Windows
 	if runtime.GOOS == "windows" {
@@ -197,8 +195,7 @@ func TestInstallPrivilegedWithoutBasePath(t *testing.T) {
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, opts.BasePath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-	t.Run("check second agent installs with --develop",
-		testDevelopmentAgentCanInstall(ctx, fixture, installtest.NamespaceTopPath(paths.DevelopmentNamespace), opts))
+	t.Run("check second agent installs with --namespace", testSecondAgentCanInstall(ctx, fixture, "", false, opts))
 }
 
 func TestInstallPrivilegedWithBasePath(t *testing.T) {
@@ -244,14 +241,13 @@ func TestInstallPrivilegedWithBasePath(t *testing.T) {
 
 	// Check that Agent was installed in the custom base path
 	topPath := filepath.Join(randomBasePath, "Elastic", "Agent")
-	devTopPath := filepath.Join(randomBasePath, "Elastic", paths.InstallDirNameForNamespace(paths.DevelopmentNamespace))
 	require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{Privileged: opts.Privileged}))
 	t.Run("check agent package version", testAgentPackageVersion(ctx, fixture, true))
-	t.Run("check second agent installs with --develop", testDevelopmentAgentCanInstall(ctx, fixture, devTopPath, opts))
+	t.Run("check second agent installs with --develop", testSecondAgentCanInstall(ctx, fixture, randomBasePath, true, opts))
 }
 
-// Tests that a second agent for development purposes can be installed alongside the first one with the --develop option.
-func testDevelopmentAgentCanInstall(ctx context.Context, fixture *atesting.Fixture, topPath string, installOpts atesting.InstallOpts) func(*testing.T) {
+// Tests that a second agent can be installed in an isolated namespace, using either --develop or --namespace.
+func testSecondAgentCanInstall(ctx context.Context, fixture *atesting.Fixture, basePath string, develop bool, installOpts atesting.InstallOpts) func(*testing.T) {
 	return func(t *testing.T) {
 		// Get path to Elastic Agent executable
 		devFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
@@ -261,15 +257,27 @@ func testDevelopmentAgentCanInstall(ctx context.Context, fixture *atesting.Fixtu
 		err = devFixture.Prepare(ctx)
 		require.NoError(t, err)
 
-		installOpts.Develop = true
+		// If development mode was requested, the namespace will be automatically set to Development after Install().
+		// Otherwise, install into a test namespace.
+		installOpts.Develop = develop
+		if !installOpts.Develop {
+			installOpts.Namespace = "Testing"
+		}
+
 		devOut, err := devFixture.Install(ctx, &installOpts)
 		if err != nil {
-			t.Logf("install --develop output: %s", devOut)
+			t.Logf("install output: %s", devOut)
 			require.NoError(t, err)
 		}
+
+		topPath := installtest.NamespaceTopPath(installOpts.Namespace)
+		if basePath != "" {
+			topPath = filepath.Join(basePath, "Elastic", paths.InstallDirNameForNamespace(installOpts.Namespace))
+		}
+
 		require.NoError(t, installtest.CheckSuccess(ctx, fixture, topPath, &installtest.CheckOpts{
 			Privileged: installOpts.Privileged,
-			Develop:    installOpts.Develop,
+			Namespace:  installOpts.Namespace,
 		}))
 	}
 }

From 051d4a8bd095003511ed5346b2aadebebdefc958 Mon Sep 17 00:00:00 2001
From: Craig MacKenzie <craig.mackenzie@elastic.co>
Date: Mon, 17 Jun 2024 16:31:22 -0400
Subject: [PATCH 49/49] Get integration tests to pass.

---
 internal/pkg/agent/cmd/install.go | 2 +-
 pkg/testing/fixture_install.go    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/agent/cmd/install.go b/internal/pkg/agent/cmd/install.go
index bbd44a230c7..55f5f77067e 100644
--- a/internal/pkg/agent/cmd/install.go
+++ b/internal/pkg/agent/cmd/install.go
@@ -51,7 +51,7 @@ would like the Agent to operate.
 	cmd.Flags().String(flagInstallBasePath, paths.DefaultBasePath, "The path where the Elastic Agent will be installed. It must be an absolute path.")
 	cmd.Flags().Bool(flagInstallUnprivileged, false, "Install in unprivileged mode, limiting the access of the Elastic Agent. (beta)")
 
-	cmd.Flags().Bool(flagInstallNamespace, false, "Install into an isolated namespace. Allows multiple Elastic Agents to be installed at once. (experimental)")
+	cmd.Flags().String(flagInstallNamespace, "", "Install into an isolated namespace. Allows multiple Elastic Agents to be installed at once. (experimental)")
 	_ = cmd.Flags().MarkHidden(flagInstallNamespace) // For internal use only.
 
 	cmd.Flags().Bool(flagInstallDevelopment, false, "Install into a standardized development namespace, may enable development specific options. Allows multiple Elastic Agents to be installed at once. (experimental)")
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index fe756313a78..81aa7db3ac6 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -166,7 +166,7 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 	f.t.Logf("[test %s] Inside fixture install function", f.t.Name())
 
 	// check for running agents before installing, but only if not installed into a namespace whose point is allowing two agents at once.
-	if installOpts != nil && installOpts.Namespace != "" {
+	if installOpts != nil && !installOpts.Develop && installOpts.Namespace == "" {
 		assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at beginning of Install()")
 	}