Add Windows support to integration testing runner (#2941)

* Work on windows runner.

* More work on windows.

* More work for windows.

* Finish it up.

* Cleanup code from merge of main.

* More improvements.

* More cleaning.

* Add comment on why upgrade marker check is skipped on Windows.

.buildkite/scripts/steps/integration_tests.sh

@@ -14,7 +14,7 @@ else
  OVERRIDE_TEST_AGENT_VERSION=""
 fi
 # PACKAGE
-AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package
+AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip mage package
 
 # Run integration tests
 set +e

Makefile

@@ -15,8 +15,9 @@ ifndef MAGE_PRESENT
 	@echo Installing mage $(MAGE_VERSION).
 	@go install ${MAGE_IMPORT_PATH}@$(MAGE_VERSION)
 	@-mage -clean
+else
+	@echo Mage $(MAGE_VERSION) already installed.
 endif
-	@true
 
 ## help : Show this help.
 help: Makefile

internal/pkg/agent/cmd/watch.go

@@ -119,6 +119,9 @@ func watchCmd(log *logp.Logger, cfg *configuration.Configuration) error {
 	// cleanup older versions,
 	// in windows it might leave self untouched, this will get cleaned up
 	// later at the start, because for windows we leave marker untouched.
+	//
+	// Why is this being skipped on Windows? The comment above is not clear.
+	// issue: https://github.com/elastic/elastic-agent/issues/3027
 	removeMarker := !isWindows()
 	err = upgrade.Cleanup(log, marker.Hash, removeMarker, false)
 	if err != nil {

pkg/testing/fixture.go

@@ -630,8 +630,11 @@ func (f *Fixture) prepareComponents(workDir string, components ...UsableComponen
 		if err := copy.Copy(comp.BinaryPath, destBinary); err != nil {
 			return fmt.Errorf("failed to copy %s to %s: %w", comp.BinaryPath, destBinary, err)
 		}
-		if err := os.Chown(destBinary, os.Geteuid(), os.Getgid()); err != nil {
-			return fmt.Errorf("failed to chown %s: %w", destBinary, err)
+		if runtime.GOOS != "windows" {
+			// chown is not supported on Windows
+			if err := os.Chown(destBinary, os.Geteuid(), os.Getgid()); err != nil {
+				return fmt.Errorf("failed to chown %s: %w", destBinary, err)
+			}
 		}
 		if err := os.Chmod(destBinary, 0755); err != nil {
 			return fmt.Errorf("failed to chmod %s: %w", destBinary, err)

pkg/testing/fixture_install.go

@@ -13,6 +13,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -143,8 +144,12 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 				"keeping the agent installed will jeopardise other tests")
 		}
 
-		// don't use current `ctx` as it could be cancelled
-		out, err := f.Uninstall(context.Background(), &UninstallOpts{Force: true, UninstallToken: f.uninstallToken})
+		// 5 minute timeout, to ensure that it at least doesn't get stuck.
+		// original context is not used as it could have a timeout on the context
+		// for the install and we don't want that context to prevent the uninstall
+		uninstallCtx, uninstallCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer uninstallCancel()
+		out, err := f.Uninstall(uninstallCtx, &UninstallOpts{Force: true, UninstallToken: f.uninstallToken})
 		f.setClient(nil)
 		if err != nil &&
 			(errors.Is(err, ErrNotInstalled) ||
@@ -195,6 +200,7 @@ func (f *Fixture) Uninstall(ctx context.Context, uninstallOpts *UninstallOpts, o
 	if err != nil {
 		return out, fmt.Errorf("error running uninstall command: %w", err)
 	}
+	f.installed = false
 
 	// Check that Elastic Agent files are actually removed
 	basePath := f.installOpts.BasePath
@@ -235,20 +241,37 @@ func (f *Fixture) collectDiagnostics() {
 		return
 	}
 
+	stamp := time.Now().Format(time.RFC3339)
+	if runtime.GOOS == "windows" {
+		// on Windows a filename cannot contain a ':' as this collides with disk labels (aka. C:\)
+		stamp = strings.ReplaceAll(stamp, ":", "-")
+	}
+
 	// Sub-test names are separated by "/" characters which are not valid filenames on Linux.
 	sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-")
-	outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", sanitizedTestName, time.Now().Format(time.RFC3339)))
+	outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", sanitizedTestName, stamp))
 
 	output, err := f.Exec(ctx, []string{"diagnostics", "-f", outputPath})
 	if err != nil {
 		f.t.Logf("failed to collect diagnostics to %s (%s): %s", outputPath, err, output)
 
-		// If collecting diagnostics fails, zip up the entire installation directory with the hope that it will contain logs.
-		f.t.Logf("creating zip archive of the installation directory: %s", f.workDir)
-		zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory-%s.zip", sanitizedTestName, time.Now().Format(time.RFC3339)))
-		err = f.archiveInstallDirectory(f.workDir, zipPath)
+		// possible that the test was so fast that the Elastic Agent was just installed, the control protocol is
+		// not fully running yet. wait 15 seconds to try again, ensuring that best effort is performed in fetching
+		// diagnostics
+		if strings.Contains(string(output), "connection error") {
+			f.t.Logf("retrying in 15 seconds due to connection error; possible Elastic Agent was not fully started")
+			time.Sleep(15 * time.Second)
+			output, err = f.Exec(ctx, []string{"diagnostics", "-f", outputPath})
+			f.t.Logf("failed to collect diagnostics a second time at %s (%s): %s", outputPath, err, output)
+		}
 		if err != nil {
-			f.t.Logf("failed to zip install directory to %s: %s", zipPath, err)
+			// If collecting diagnostics fails, zip up the entire installation directory with the hope that it will contain logs.
+			f.t.Logf("creating zip archive of the installation directory: %s", f.workDir)
+			zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory-%s.zip", sanitizedTestName, time.Now().Format(time.RFC3339)))
+			err = f.archiveInstallDirectory(f.workDir, zipPath)
+			if err != nil {
+				f.t.Logf("failed to zip install directory to %s: %s", zipPath, err)
+			}
 		}
 	}
 }

pkg/testing/ogc/provisioner.go

@@ -284,7 +284,11 @@ func osBatchToOGC(cacheDir string, batch runner.OSBatch) Layout {
 		LayoutIntegrationTag,
 		batch.OS.Type,
 		batch.OS.Arch,
-		strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Distro, strings.Replace(batch.OS.Version, ".", "-", -1))),
+	}
+	if batch.OS.Type == define.Linux {
+		tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Distro, strings.Replace(batch.OS.Version, ".", "-", -1))))
+	} else {
+		tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Type, strings.Replace(batch.OS.Version, ".", "-", -1))))
 	}
 	if batch.Batch.Isolate {
 		tags = append(tags, "isolate")

pkg/testing/ogc/supported.go

@@ -9,6 +9,11 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/runner"
 )
 
+const (
+	// Google is for the Google Cloud Platform (GCP)
+	Google = "google"
+)
+
 // ogcSupported defines the set of supported OS's the OGC provisioner currently supports.
 //
 // In the case that a batch is not specific on the version and/or distro the first
@@ -22,7 +27,7 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "22.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "e2-standard-2", // 2 amd64 cpus
 		RunsOn:       "ubuntu-2204-lts",
 		Username:     "ubuntu",
@@ -35,7 +40,7 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "20.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "e2-standard-2", // 2 amd64 cpus
 		RunsOn:       "ubuntu-2004-lts",
 		Username:     "ubuntu",
@@ -48,7 +53,7 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "22.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "t2a-standard-2", // 2 arm64 cpus
 		RunsOn:       "ubuntu-2204-lts-arm64",
 		Username:     "ubuntu",
@@ -61,10 +66,82 @@ var ogcSupported = []LayoutOS{
 			Distro:  runner.Ubuntu,
 			Version: "20.04",
 		},
-		Provider:     runner.Google,
+		Provider:     Google,
 		InstanceSize: "t2a-standard-2", // 2 arm64 cpus
 		RunsOn:       "ubuntu-2004-lts-arm64",
 		Username:     "ubuntu",
 		RemotePath:   "/home/ubuntu/agent",
 	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2022",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2022",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2022-core",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2022-core",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2019",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2019",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2019-core",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2019-core",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2016",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2016",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
+	{
+		OS: define.OS{
+			Type:    define.Windows,
+			Arch:    define.AMD64,
+			Version: "2016-core",
+		},
+		Provider:     Google,
+		InstanceSize: "e2-standard-4", // 4 amd64 cpus
+		RunsOn:       "windows-2016-core",
+		Username:     "windows",
+		RemotePath:   "C:\\Users\\windows\\agent",
+	},
 }