Add RPM integration tests

- add sles runner
- add rhel runner
- add test that uses rhel runner
- add logging to ssh

pkg/testing/fixture.go

@@ -837,7 +837,7 @@ func (f *Fixture) binaryPath() string {
 			workDir = filepath.Join(paths.DefaultBasePath, "Elastic", "Agent")
 		}
 	}
-	if f.packageFormat == "deb" {
+	if f.packageFormat == "deb" || f.packageFormat == "rpm" {
 		workDir = "/usr/bin"
 	}
 	defaultBin := "elastic-agent"

pkg/testing/fixture_install.go

@@ -138,6 +138,8 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 		return f.installNoPkgManager(ctx, installOpts, opts)
 	case "deb":
 		return f.installDeb(ctx, installOpts, opts)
+	case "rpm":
+		return f.installRpm(ctx, installOpts, opts)
 	default:
 		return nil, fmt.Errorf("package format %s isn't supported yet", f.packageFormat)
 	}
@@ -435,6 +437,81 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts
 	return nil, nil
 }
 
+// installRpm installs the prepared Elastic Agent binary from the rpm
+// package and registers a t.Cleanup function to uninstall the agent if
+// it hasn't been uninstalled. It also takes care of collecting a
+// diagnostics when AGENT_COLLECT_DIAG=true or the test has failed.
+// It returns:
+//   - the combined output of Install command stdout and stderr
+//   - an error if any.
+func (f *Fixture) installRpm(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) {
+	f.t.Logf("[test %s] Inside fixture installRpm function", f.t.Name())
+	//Prepare so that the f.srcPackage string is populated
+	err := f.EnsurePrepared(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare: %w", err)
+	}
+
+	// sudo rpm -iv elastic-agent rpm
+	out, err := exec.CommandContext(ctx, "sudo", "rpm", "-i", "-v", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
+	if err != nil {
+		return out, fmt.Errorf("rpm install failed: %w output:%s", err, string(out))
+	}
+
+	f.t.Cleanup(func() {
+		f.t.Logf("[test %s] Inside fixture installRpm cleanup function", f.t.Name())
+		uninstallCtx, uninstallCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer uninstallCancel()
+		// stop elastic-agent, non fatal if error, might have been stopped before this.
+		f.t.Logf("running 'sudo systemctl stop elastic-agent'")
+		out, err := exec.CommandContext(uninstallCtx, "sudo", "systemctl", "stop", "elastic-agent").CombinedOutput()
+		if err != nil {
+			f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out))
+		}
+		// rpm -e elastic-agent rpm
+		f.t.Logf("running 'sudo rpm -e elastic-agent'")
+		out, err = exec.CommandContext(uninstallCtx, "sudo", "rpm", "-e", "elastic-agent").CombinedOutput()
+		if err != nil {
+			f.t.Logf("failed to 'sudo rpm -e elastic-agent': %s, output: %s", err, string(out))
+			f.t.FailNow()
+		}
+	})
+
+	// start elastic-agent
+	out, err = exec.CommandContext(ctx, "sudo", "systemctl", "start", "elastic-agent").CombinedOutput()
+	if err != nil {
+		return out, fmt.Errorf("systemctl start elastic-agent failed: %w", err)
+	}
+
+	// rpm install doesn't enroll, so need to do that
+	enrollArgs := []string{"elastic-agent", "enroll"}
+	if installOpts.Force {
+		enrollArgs = append(enrollArgs, "--force")
+	}
+	if installOpts.Insecure {
+		enrollArgs = append(enrollArgs, "--insecure")
+	}
+	if installOpts.ProxyURL != "" {
+		enrollArgs = append(enrollArgs, "--proxy-url="+installOpts.ProxyURL)
+	}
+	if installOpts.DelayEnroll {
+		enrollArgs = append(enrollArgs, "--delay-enroll")
+	}
+	if installOpts.EnrollOpts.URL != "" {
+		enrollArgs = append(enrollArgs, "--url", installOpts.EnrollOpts.URL)
+	}
+	if installOpts.EnrollOpts.EnrollmentToken != "" {
+		enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollOpts.EnrollmentToken)
+	}
+	// run sudo elastic-agent enroll
+	out, err = exec.CommandContext(ctx, "sudo", enrollArgs...).CombinedOutput()
+	if err != nil {
+		return out, fmt.Errorf("elastic-agent enroll failed: %w, output: %s args: %v", err, string(out), enrollArgs)
+	}
+
+	return nil, nil
+}
+
 type UninstallOpts struct {
 	Force          bool // --force
 	UninstallToken string
@@ -460,6 +537,8 @@ func (f *Fixture) Uninstall(ctx context.Context, uninstallOpts *UninstallOpts, o
 		return f.uninstallNoPkgManager(ctx, uninstallOpts, opts)
 	case "deb":
 		return f.uninstallDeb(ctx, uninstallOpts, opts)
+	case "rpm":
+		return f.uninstallRpm(ctx, uninstallOpts, opts)
 	default:
 		return nil, fmt.Errorf("uninstall of package format '%s' not supported yet", f.packageFormat)
 	}
@@ -478,6 +557,19 @@ func (f *Fixture) uninstallDeb(ctx context.Context, uninstallOpts *UninstallOpts
 	return out, nil
 }
 
+func (f *Fixture) uninstallRpm(ctx context.Context, uninstallOpts *UninstallOpts, opts []process.CmdOption) ([]byte, error) {
+	// stop elastic-agent, non fatal if error, might have been stopped before this.
+	out, err := exec.CommandContext(ctx, "sudo", "systemctl", "stop", "elastic-agent").CombinedOutput()
+	if err != nil {
+		f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out))
+	}
+	out, err = exec.CommandContext(ctx, "sudo", "rpm", "-e", "elastic-agent").CombinedOutput()
+	if err != nil {
+		return out, fmt.Errorf("error running 'sudo rpm -e elastic-agent': %w", err)
+	}
+	return out, nil
+}
+
 func (f *Fixture) uninstallNoPkgManager(ctx context.Context, uninstallOpts *UninstallOpts, opts []process.CmdOption) ([]byte, error) {
 	if !f.installed {
 		return nil, ErrNotInstalled

pkg/testing/ogc/supported.go

@@ -72,6 +72,32 @@ var ogcSupported = []LayoutOS{
 		Username:     "ubuntu",
 		RemotePath:   "/home/ubuntu/agent",
 	},
+	{
+		OS: define.OS{
+			Type:    define.Linux,
+			Arch:    define.AMD64,
+			Distro:  runner.Sles,
+			Version: "15",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-2", // 2 amd64 cpus
+		RunsOn:       "sles-15",
+		Username:     "sles",
+		RemotePath:   "/home/sles/agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Linux,
+			Arch:    define.AMD64,
+			Distro:  runner.Rhel,
+			Version: "8",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-2", // 2 amd64 cpus
+		RunsOn:       "rhel-8",
+		Username:     "rhel",
+		RemotePath:   "/home/rhel/agent",
+	},
 	{
 		OS: define.OS{
 			Type:    define.Windows,

pkg/testing/runner/debian.go

@@ -7,7 +7,6 @@ package runner
 import (
 	"context"
 	"fmt"
-	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -88,108 +87,7 @@ func (DebianRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Log
 
 // Copy places the required files on the host.
 func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error {
-	// copy the archive and extract it on the host
-	logger.Logf("Copying repo")
-	destRepoName := filepath.Base(repoArchive)
-	err := sshClient.Copy(repoArchive, destRepoName)
-	if err != nil {
-		return fmt.Errorf("failed to SCP repo archive %s: %w", repoArchive, err)
-	}
-
-	// remove build paths, on cases where the build path is different from agent.
-	for _, build := range builds {
-		for _, remoteBuildPath := range []string{build.Path, build.SHA512Path} {
-			relativeAgentDir := filepath.Join("agent", remoteBuildPath)
-			_, _, err := sshClient.Exec(ctx, "sudo", []string{"rm", "-rf", relativeAgentDir}, nil)
-			// doesn't need to be a fatal error.
-			if err != nil {
-				logger.Logf("error removing build dir %s: %w", relativeAgentDir, err)
-			}
-		}
-	}
-
-	// ensure that agent directory is removed (possible it already exists if instance already used)
-	stdout, stderr, err := sshClient.Exec(ctx,
-		"sudo", []string{"rm", "-rf", "agent"}, nil)
-	if err != nil {
-		return fmt.Errorf(
-			"failed to remove agent directory before unziping new one: %w. stdout: %q, stderr: %q",
-			err, stdout, stderr)
-	}
-
-	stdOut, errOut, err := sshClient.Exec(ctx, "unzip", []string{destRepoName, "-d", "agent"}, nil)
-	if err != nil {
-		return fmt.Errorf("failed to unzip %s to agent directory: %w (stdout: %s, stderr: %s)", destRepoName, err, stdOut, errOut)
-	}
-
-	// prepare for testing
-	logger.Logf("Running make mage and prepareOnRemote")
-	envs := `GOPATH="$HOME/go" PATH="$HOME/go/bin:$PATH"`
-	installMage := strings.NewReader(fmt.Sprintf(`cd agent && %s make mage && %s mage integration:prepareOnRemote`, envs, envs))
-	stdOut, errOut, err = sshClient.Exec(ctx, "bash", nil, installMage)
-	if err != nil {
-		return fmt.Errorf("failed to perform make mage and prepareOnRemote: %w (stdout: %s, stderr: %s)", err, stdOut, errOut)
-	}
-
-	// determine if the build needs to be replaced on the host
-	// if it already exists and the SHA512 are the same contents, then
-	// there is no reason to waste time uploading the build
-	for _, build := range builds {
-		copyBuild := true
-		localSHA512, err := os.ReadFile(build.SHA512Path)
-		if err != nil {
-			return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err)
-		}
-		hostSHA512Path := filepath.Base(build.SHA512Path)
-		hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path)
-		if err == nil {
-			if string(localSHA512) == string(hostSHA512) {
-				logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path))
-				copyBuild = false
-			}
-		}
-
-		if copyBuild {
-			// ensure the existing copies are removed first
-			toRemove := filepath.Base(build.Path)
-			stdOut, errOut, err = sshClient.Exec(ctx,
-				"sudo", []string{"rm", "-f", toRemove}, nil)
-			if err != nil {
-				return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)",
-					toRemove, err, stdOut, errOut)
-			}
-
-			toRemove = filepath.Base(build.SHA512Path)
-			stdOut, errOut, err = sshClient.Exec(ctx,
-				"sudo", []string{"rm", "-f", toRemove}, nil)
-			if err != nil {
-				return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)",
-					toRemove, err, stdOut, errOut)
-			}
-
-			logger.Logf("Copying agent build %s", filepath.Base(build.Path))
-		}
-
-		for _, buildPath := range []string{build.Path, build.SHA512Path} {
-			if copyBuild {
-				err = sshClient.Copy(buildPath, filepath.Base(buildPath))
-				if err != nil {
-					return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err)
-				}
-			}
-			insideAgentDir := filepath.Join("agent", buildPath)
-			stdOut, errOut, err = sshClient.Exec(ctx, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil)
-			if err != nil {
-				return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut)
-			}
-			stdOut, errOut, err = sshClient.Exec(ctx, "ln", []string{filepath.Base(buildPath), insideAgentDir}, nil)
-			if err != nil {
-				return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut)
-			}
-		}
-	}
-
-	return nil
+	return linuxCopy(ctx, sshClient, logger, repoArchive, builds)
 }
 
 // Run the test
@@ -242,39 +140,7 @@ func (DebianRunner) Run(ctx context.Context, verbose bool, sshClient SSHClient,
 
 // Diagnostics gathers any diagnostics from the host.
 func (DebianRunner) Diagnostics(ctx context.Context, sshClient SSHClient, logger Logger, destination string) error {
-	// take ownership, as sudo tests will create with root permissions (allow to fail in the case it doesn't exist)
-	diagnosticDir := "$HOME/agent/build/diagnostics"
-	_, _, _ = sshClient.Exec(ctx, "sudo", []string{"chown", "-R", "$USER:$USER", diagnosticDir}, nil)
-	stdOut, _, err := sshClient.Exec(ctx, "ls", []string{"-1", diagnosticDir}, nil)
-	if err != nil {
-		//nolint:nilerr // failed to list the directory, probably don't have any diagnostics (do nothing)
-		return nil
-	}
-	eachDiagnostic := strings.Split(string(stdOut), "\n")
-	for _, filename := range eachDiagnostic {
-		filename = strings.TrimSpace(filename)
-		if filename == "" {
-			continue
-		}
-
-		// don't use filepath.Join as we need this to work in Windows as well
-		// this is because if we use `filepath.Join` on a Windows host connected to a Linux host
-		// it will use a `\` and that will be incorrect for Linux
-		fp := fmt.Sprintf("%s/%s", diagnosticDir, filename)
-		// use filepath.Join on this path because it's a path on this specific host platform
-		dp := filepath.Join(destination, filename)
-		logger.Logf("Copying diagnostic %s", filename)
-		out, err := os.Create(dp)
-		if err != nil {
-			return fmt.Errorf("failed to create file %s: %w", dp, err)
-		}
-		err = sshClient.GetFileContentsOutput(ctx, fp, out)
-		_ = out.Close()
-		if err != nil {
-			return fmt.Errorf("failed to copy file from remote host to %s: %w", dp, err)
-		}
-	}
-	return nil
+	return linuxDiagnostics(ctx, sshClient, logger, destination)
 }
 
 func runTests(ctx context.Context, logger Logger, name string, prefix string, script string, sshClient SSHClient, tests []define.BatchPackageTests) ([]OSRunnerPackageResult, error) {