Introduce filter by group and dry-run for integration tests (#5900) (#…

…6398)

* add filtering integration tests by group

* add dry-run integration test flag

* mage check for define.Require should skip TestMain

* Add define filter on sudo requirements

(cherry picked from commit 7978189)

Co-authored-by: Paolo Chilà <[email protected]>

pkg/testing/define/define.go

@@ -13,6 +13,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
@@ -143,6 +144,17 @@ func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Inf
 	if err := req.Validate(); err != nil {
 		panic(fmt.Sprintf("test %s has invalid requirements: %s", t.Name(), err))
 	}
+
+	filteredGroups := GroupsFilter.values
+	if len(filteredGroups) > 0 && !slices.Contains(filteredGroups, req.Group) {
+		t.Skipf("group %s not found in groups filter %s. Skipping", req.Group, filteredGroups)
+		return nil
+	}
+
+	if SudoFilter.value != nil && req.Sudo != *SudoFilter.value {
+		t.Skipf("sudo requirement %t not matching sudo filter %t. Skipping", req.Sudo, *SudoFilter.value)
+	}
+
 	if !req.Local && local {
 		t.Skip("running local only tests and this test doesn't support local")
 		return nil
@@ -173,6 +185,11 @@ func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Inf
 		t.Skipf("platform: %s, architecture: %s, version: %s, and distro: %s combination is not supported by test.  required: %v", runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform, req.OS)
 		return nil
 	}
+
+	if DryRun {
+		return dryRun(t, req)
+	}
+
 	namespace, err := getNamespace(t, local)
 	if err != nil {
 		panic(err)

pkg/testing/define/define_flags.go

@@ -0,0 +1,75 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package define
+
+import (
+	"flag"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+type optionalBoolFlag struct {
+	value *bool
+}
+
+func (o *optionalBoolFlag) String() string {
+	if o.value == nil {
+		return "nil"
+	}
+	return strconv.FormatBool(*o.value)
+}
+
+func (o *optionalBoolFlag) Set(s string) error {
+	bValue := s == "" || s == "true"
+	o.value = &bValue
+	return nil
+}
+
+type stringArrayFlag struct {
+	values []string
+}
+
+func (s *stringArrayFlag) String() string {
+	return fmt.Sprintf("%s", s.values)
+}
+
+func (s *stringArrayFlag) Set(stringValue string) error {
+	if stringValue == "" {
+		return nil
+	}
+	s.values = strings.Split(stringValue, ",")
+	return nil
+}
+
+var (
+	DryRun          bool
+	GroupsFilter    stringArrayFlag
+	PlatformsFilter stringArrayFlag
+	SudoFilter      optionalBoolFlag
+)
+
+func RegisterFlags(prefix string, set *flag.FlagSet) {
+	set.BoolVar(&DryRun, prefix+"dry-run", false, "Forces test in dry-run mode: skips the main test and puts a successful placeholder <TestName>/dry-run if the test would have run")
+	set.Var(&GroupsFilter, prefix+"groups", "test groups, comma-separated")
+	set.Var(&PlatformsFilter, prefix+"platforms", "test platforms, comma-separated")
+	set.Var(&SudoFilter, prefix+"sudo", "Filter tests by sudo requirements")
+}
+
+func dryRun(t *testing.T, req Requirements) *Info {
+	// always validate requirement is valid
+	if err := req.Validate(); err != nil {
+		t.Logf("test %s has invalid requirements: %s", t.Name(), err)
+		t.FailNow()
+		return nil
+	}
+	// skip the test as we are in dry run
+	t.Run("dry-run", func(t *testing.T) {
+		t.Log("Test dry-run successful")
+	})
+	t.Skip("Skipped because dry-run mode has been specified.")
+	return nil
+}

pkg/testing/define/parser.go

@@ -26,7 +26,7 @@ func ValidateDir(dir string) error {
 		for _, file := range pkg.Files {
 			for _, d := range file.Decls {
 				fn, ok := d.(*ast.FuncDecl)
-				if ok && strings.HasPrefix(fn.Name.Name, "Test") && fn.Recv == nil {
+				if ok && fn.Name.Name != "TestMain" && strings.HasPrefix(fn.Name.Name, "Test") && fn.Recv == nil {
 					if !validateRequireFromFunc(fn) {
 						return fmt.Errorf("test %s first statement must be a function call to define.Require", fn.Name.Name)
 					}

testing/integration/main_test.go

@@ -0,0 +1,32 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package integration
+
+import (
+	"flag"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+)
+
+var flagSet = flag.CommandLine
+
+func init() {
+	define.RegisterFlags("integration.", flagSet)
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	runExitCode := m.Run()
+
+	if define.DryRun {
+		// TODO add parsing of requirements and dump them
+		log.Print("Dry-run mode specified...")
+	}
+
+	os.Exit(runExitCode)
+}