[8.9](backport #2941) Add Windows support to integration testing runner

.buildkite/scripts/steps/integration_tests.sh

@@ -4,7 +4,11 @@ set -euxo pipefail
 source .buildkite/scripts/common.sh
 
 # PACKAGE
+<<<<<<< HEAD
 DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package
+=======
+AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip mage package
+>>>>>>> 35dbbdea9b (Add Windows support to integration testing runner (#2941))
 
 # Run integration tests
 set +e

Makefile

@@ -15,8 +15,9 @@ ifndef MAGE_PRESENT
 	@echo Installing mage $(MAGE_VERSION).
 	@go install ${MAGE_IMPORT_PATH}@$(MAGE_VERSION)
 	@-mage -clean
+else
+	@echo Mage $(MAGE_VERSION) already installed.
 endif
-	@true
 
 
 ## help : Show this help.

internal/pkg/agent/cmd/watch.go

@@ -116,6 +116,9 @@ func watchCmd(log *logp.Logger, cfg *configuration.Configuration) error {
 	// cleanup older versions,
 	// in windows it might leave self untouched, this will get cleaned up
 	// later at the start, because for windows we leave marker untouched.
+	//
+	// Why is this being skipped on Windows? The comment above is not clear.
+	// issue: https://github.com/elastic/elastic-agent/issues/3027
 	removeMarker := !isWindows()
 	err = upgrade.Cleanup(log, marker.Hash, removeMarker, false)
 	if err != nil {

pkg/testing/fixture.go

@@ -608,8 +608,11 @@ func (f *Fixture) prepareComponents(workDir string, components ...UsableComponen
 		if err := copy.Copy(comp.BinaryPath, destBinary); err != nil {
 			return fmt.Errorf("failed to copy %s to %s: %w", comp.BinaryPath, destBinary, err)
 		}
-		if err := os.Chown(destBinary, os.Geteuid(), os.Getgid()); err != nil {
-			return fmt.Errorf("failed to chown %s: %w", destBinary, err)
+		if runtime.GOOS != "windows" {
+			// chown is not supported on Windows
+			if err := os.Chown(destBinary, os.Geteuid(), os.Getgid()); err != nil {
+				return fmt.Errorf("failed to chown %s: %w", destBinary, err)
+			}
 		}
 		if err := os.Chmod(destBinary, 0755); err != nil {
 			return fmt.Errorf("failed to chmod %s: %w", destBinary, err)

pkg/testing/fixture_install.go

@@ -11,6 +11,7 @@
 	"io/fs"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -145,23 +146,46 @@
 			}
 			require.NoErrorf(f.t, err, "uninstalling agent failed. Output: %q", out)
 		}
+<<<<<<< HEAD
+=======
+
+		// 5 minute timeout, to ensure that it at least doesn't get stuck.
+		// original context is not used as it could have a timeout on the context
+		// for the install and we don't want that context to prevent the uninstall
+		uninstallCtx, uninstallCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer uninstallCancel()
+		out, err := f.Uninstall(uninstallCtx, &UninstallOpts{Force: true, UninstallToken: f.uninstallToken})
+		f.setClient(nil)
+		if err != nil &&
+			(errors.Is(err, ErrNotInstalled) ||
+				strings.Contains(
+					err.Error(),
+					"elastic-agent: no such file or directory")) {
+			f.t.Logf("fixture.Install Cleanup: agent was already uninstalled, skipping uninstall")
+			// Agent fixture has already been uninstalled, perhaps by
+			// an explicit call to fixture.Uninstall, so nothing needs
+			// to be done here.
+			return
+		}
+		require.NoErrorf(f.t, err, "uninstalling agent failed. Output: %q", out)
+>>>>>>> 35dbbdea9b (Add Windows support to integration testing runner (#2941))
 	})
 
 	return out, nil
 }
 
 type UninstallOpts struct {
 	Force bool // --force
 }
 
 func (i UninstallOpts) toCmdArgs() []string {
 	var args []string
 	if i.Force {
 		args = append(args, "--force")
 	}
 
 	return args
 }
 
 // Uninstall uninstalls the installed Elastic Agent binary
 func (f *Fixture) Uninstall(ctx context.Context, uninstallOpts *UninstallOpts, opts ...process.CmdOption) ([]byte, error) {
@@ -177,6 +201,7 @@
 	if err != nil {
 		return out, fmt.Errorf("error running uninstall command: %w", err)
 	}
+	f.installed = false
 
 	// Check that Elastic Agent files are actually removed
 	basePath := f.installOpts.BasePath
@@ -216,11 +241,46 @@
 		f.t.Logf("failed to collect diagnostics; failed to create %s: %s", diagPath, err)
 		return
 	}
+<<<<<<< HEAD
 	outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", f.t.Name(), time.Now().Format(time.RFC3339)))
+=======
+
+	stamp := time.Now().Format(time.RFC3339)
+	if runtime.GOOS == "windows" {
+		// on Windows a filename cannot contain a ':' as this collides with disk labels (aka. C:\)
+		stamp = strings.ReplaceAll(stamp, ":", "-")
+	}
+
+	// Sub-test names are separated by "/" characters which are not valid filenames on Linux.
+	sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-")
+	outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", sanitizedTestName, stamp))
+>>>>>>> 35dbbdea9b (Add Windows support to integration testing runner (#2941))
 
 	output, err := f.Exec(ctx, []string{"diagnostics", "-f", outputPath})
 	if err != nil {
 		f.t.Logf("failed to collect diagnostics to %s (%s): %s", outputPath, err, output)
+<<<<<<< HEAD
+=======
+
+		// possible that the test was so fast that the Elastic Agent was just installed, the control protocol is
+		// not fully running yet. wait 15 seconds to try again, ensuring that best effort is performed in fetching
+		// diagnostics
+		if strings.Contains(string(output), "connection error") {
+			f.t.Logf("retrying in 15 seconds due to connection error; possible Elastic Agent was not fully started")
+			time.Sleep(15 * time.Second)
+			output, err = f.Exec(ctx, []string{"diagnostics", "-f", outputPath})
+			f.t.Logf("failed to collect diagnostics a second time at %s (%s): %s", outputPath, err, output)
+		}
+		if err != nil {
+			// If collecting diagnostics fails, zip up the entire installation directory with the hope that it will contain logs.
+			f.t.Logf("creating zip archive of the installation directory: %s", f.workDir)
+			zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory-%s.zip", sanitizedTestName, time.Now().Format(time.RFC3339)))
+			err = f.archiveInstallDirectory(f.workDir, zipPath)
+			if err != nil {
+				f.t.Logf("failed to zip install directory to %s: %s", zipPath, err)
+			}
+		}
+>>>>>>> 35dbbdea9b (Add Windows support to integration testing runner (#2941))
 	}
 }
 

pkg/testing/ogc/provisioner.go

@@ -284,7 +284,11 @@ func osBatchToOGC(cacheDir string, batch runner.OSBatch) Layout {
 		LayoutIntegrationTag,
 		batch.OS.Type,
 		batch.OS.Arch,
-		strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Distro, strings.Replace(batch.OS.Version, ".", "-", -1))),
+	}
+	if batch.OS.Type == define.Linux {
+		tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Distro, strings.Replace(batch.OS.Version, ".", "-", -1))))
+	} else {
+		tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Type, strings.Replace(batch.OS.Version, ".", "-", -1))))
 	}
 	if batch.Batch.Isolate {
 		tags = append(tags, "isolate")

pkg/testing/ogc/supported.go

@@ -9,6 +9,11 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/runner"
 )
 
+const (
+	// Google is for the Google Cloud Platform (GCP)
+	Google = "google"
+)
+
 // ogcSupported defines the set of supported OS's the OGC provisioner currently supports.
 //
 // In the case that a batch is not specific on the version and/or distro the first
@@ -22,7 +27,7 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "22.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "e2-standard-2", // 2 amd64 cpus
 		RunsOn:       "ubuntu-2204-lts",
 		Username:     "ubuntu",
@@ -35,7 +40,7 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "20.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "e2-standard-2", // 2 amd64 cpus
 		RunsOn:       "ubuntu-2004-lts",
 		Username:     "ubuntu",
@@ -48,7 +53,7 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "22.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "t2a-standard-2", // 2 arm64 cpus
 		RunsOn:       "ubuntu-2204-lts-arm64",
 		Username:     "ubuntu",
@@ -61,10 +66,82 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "20.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "t2a-standard-2", // 2 arm64 cpus
 		RunsOn:       "ubuntu-2004-lts-arm64",
 		Username:     "ubuntu",
 		RemotePath:   "/home/ubuntu/agent",
 	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2022",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2022",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2022-core",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2022-core",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2019",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2019",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2019-core",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2019-core",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2016",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2016",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2016-core",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2016-core",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
 }