diff --git a/pkg/testing/define/define.go b/pkg/testing/define/define.go index 3b245d3630a..cf8eec2af3f 100644 --- a/pkg/testing/define/define.go +++ b/pkg/testing/define/define.go @@ -143,6 +143,25 @@ func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Inf if err := req.Validate(); err != nil { panic(fmt.Sprintf("test %s has invalid requirements: %s", t.Name(), err)) } +<<<<<<< HEAD +======= + + filteredGroups := GroupsFilter.values + if len(filteredGroups) > 0 && !slices.Contains(filteredGroups, req.Group) { + t.Skipf("group %s not found in groups filter %s. Skipping", req.Group, filteredGroups) + return nil + } + + if SudoFilter.value != nil && req.Sudo != *SudoFilter.value { + t.Skipf("sudo requirement %t not matching sudo filter %t. Skipping", req.Sudo, *SudoFilter.value) + } + + // record autodiscover after filtering by group and sudo and before validating against the actual environment + if AutoDiscover { + discoverTest(t, req) + } + +>>>>>>> a7597b01d (Integration tests autodiscovery (#6199)) if !req.Local && local { t.Skip("running local only tests and this test doesn't support local") return nil diff --git a/pkg/testing/define/define_autodiscovery.go b/pkg/testing/define/define_autodiscovery.go new file mode 100644 index 00000000000..1b4fd558b84 --- /dev/null +++ b/pkg/testing/define/define_autodiscovery.go @@ -0,0 +1,298 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package define + +import ( + "fmt" + "sync" + + "gopkg.in/yaml.v3" +) + +// Default platforms. Overridable using InitAutodiscovery() +var defaultPlatforms = []TestPlatform{ + {OS: Windows, Arch: AMD64}, + + // Not supported by default + // {OS: Windows, Arch: ARM64}, + + // Current batching mechanism support this, not sure it's correct + {OS: Darwin, Arch: AMD64}, + {OS: Darwin, Arch: ARM64}, + {OS: Linux, Arch: AMD64}, + {OS: Linux, Arch: ARM64}, +} + +var defaultTestOS = TestOS{ + Name: "", + Version: "", +} + +var defaultTestPlatform = TestPlatform{ + OS: "", + Arch: "", +} + +// YAML/JSON output structs +type OutputRunner struct { + OSFamily string `json:"os_family" yaml:"os_family" ` + Arch string `json:"arch,omitempty"` + OS string `json:"os,omitempty"` + Version string `json:"version,omitempty"` + Groups []OutputGroup `json:"groups,omitempty"` +} + +type OutputGroup struct { + Name string + Tests []OutputTest +} + +type OutputTest struct { + Name string + Metadata TestMetadata +} + +// structs to aggregate test information +type TestPlatform struct { + OS string `json:"os" yaml:"os"` + Arch string `json:"arch" yaml:"arch"` +} + +type TestMetadata struct { + Local bool `json:"local" yaml:"local"` + Sudo bool `json:"sudo" yaml:"sudo"` +} + +type TestOS struct { + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` +} +type TestGroup struct { + Tests map[string]TestMetadata +} + +func NewTestGroup() TestGroup { + return TestGroup{ + Tests: map[string]TestMetadata{}, + } +} + +type TestByOS struct { + Groups map[string]TestGroup +} + +func NewTestByOS() TestByOS { + return TestByOS{ + Groups: map[string]TestGroup{}, + } +} + +type TestsByPlatform struct { + OperatingSystems map[TestOS]TestByOS `json:"os" yaml:"os"` +} + +func NewTestsByPlatform() TestsByPlatform { + return TestsByPlatform{OperatingSystems: map[TestOS]TestByOS{}} +} + +type DiscoveredTests struct { + Discovered map[TestPlatform]TestsByPlatform +} + +// test autodiscovery aggregator +var testAutodiscovery *DiscoveredTests +var testAutodiscoveryMx sync.Mutex + +type Named interface { + Name() string +} + +func InitAutodiscovery(initDefaultPlatforms []TestPlatform) { + testAutodiscoveryMx.Lock() + defer testAutodiscoveryMx.Unlock() + testAutodiscovery = &DiscoveredTests{ + Discovered: map[TestPlatform]TestsByPlatform{}, + } + + if initDefaultPlatforms != nil { + defaultPlatforms = initDefaultPlatforms + } +} + +func DumpAutodiscoveryYAML() ([]byte, error) { + testAutodiscoveryMx.Lock() + defer testAutodiscoveryMx.Unlock() + err := testAutodiscovery.normalizeDiscoveredTests() + if err != nil { + return nil, fmt.Errorf("normalizing discovered tests: %w", err) + } + + runners := mapToRunners(testAutodiscovery) + + return yaml.Marshal(runners) +} + +func mapToRunners(autodiscovery *DiscoveredTests) []OutputRunner { + + var mapped []OutputRunner + + for pltf, testsByOS := range autodiscovery.Discovered { + for testOS, testsByOS := range testsByOS.OperatingSystems { + or := OutputRunner{ + OSFamily: pltf.OS, + Arch: pltf.Arch, + OS: testOS.Name, + Version: testOS.Version, + Groups: make([]OutputGroup, 0, len(testsByOS.Groups)), + } + + for groupName, groupTests := range testsByOS.Groups { + or.Groups = append(or.Groups, mapGroup(groupName, groupTests)) + } + mapped = append(mapped, or) + } + } + + return mapped +} + +func mapGroup(name string, group TestGroup) OutputGroup { + og := OutputGroup{Name: name, Tests: make([]OutputTest, 0, len(group.Tests))} + for testName, testMetadata := range group.Tests { + og.Tests = append(og.Tests, OutputTest{ + Name: testName, + Metadata: testMetadata, + }) + } + + return og +} + +func discoverTest(test Named, reqs Requirements) { + testAutodiscoveryMx.Lock() + defer testAutodiscoveryMx.Unlock() + for _, p := range getPlatforms(reqs.OS) { + if testAutodiscovery == nil { + panic("testAutodiscovery is nil. Check that InitAutodiscovery() has been called properly") + } + mappedOSesForPlatform := ensureMapping(testAutodiscovery.Discovered, p, NewTestsByPlatform) + osForPlatform := getOSForPlatform(reqs.OS, p) + for _, o := range osForPlatform { + testsByOS := ensureMapping(mappedOSesForPlatform.OperatingSystems, o, NewTestByOS) + testGroup := ensureMapping(testsByOS.Groups, reqs.Group, NewTestGroup) + testGroup.Tests[test.Name()] = TestMetadata{ + Local: reqs.Local, + Sudo: reqs.Sudo, + } + } + } +} + +func ensureMapping[K comparable, V any](mappings map[K]V, k K, newValueCreateFunc func() V) V { + if existingValue, ok := mappings[k]; ok { + return existingValue + } + newValue := newValueCreateFunc() + mappings[k] = newValue + return newValue +} + +func getOSForPlatform(os []OS, p TestPlatform) []TestOS { + + var matchingOSes []TestOS + + for _, o := range os { + if o.Type == p.OS && o.Arch == p.Arch { + matchingOSes = append(matchingOSes, getTestOS(o)) + } + } + + if len(matchingOSes) > 0 { + return matchingOSes + } + + // no other OS has matched, return the default OS + return []TestOS{ + defaultTestOS, + } + +} + +func getTestOS(o OS) TestOS { + switch { + case o.Type == Linux: + return TestOS{ + Name: o.Distro, + Version: o.Version, + } + default: + return TestOS{ + Name: o.Type, + Version: o.Version, + } + } +} + +func getPlatforms(os []OS) []TestPlatform { + if len(os) == 0 { + return []TestPlatform{defaultTestPlatform} + } + + platforms := make([]TestPlatform, 0, len(os)) + for _, o := range os { + platforms = append(platforms, TestPlatform{ + OS: o.Type, + Arch: o.Arch, + }) + } + + return platforms +} + +// Normalization functions +func (dt *DiscoveredTests) normalizeDiscoveredTests() error { + + normalized := map[TestPlatform]TestsByPlatform{} + for pltf, oses := range dt.Discovered { + + if pltf.OS == "" && pltf.Arch != "" { + return fmt.Errorf("platform not supported: %v", pltf) + } + + if pltf.OS != "" && pltf.Arch != "" { + existingOSes := ensureMapping(normalized, pltf, NewTestsByPlatform) // normal case, append to normalized and go to the next platform + existingOSes.mergeOSes(oses) + continue + } + + // Arch and/or OS is not specified: fill in the supported archs for the OS type (potentially for all OSes) + for i, dp := range defaultPlatforms { + if pltf.OS == "" || pltf.OS == dp.OS { + existingOSes := ensureMapping(normalized, defaultPlatforms[i], NewTestsByPlatform) + existingOSes.mergeOSes(oses) + } + } + } + + dt.Discovered = normalized + + return nil +} + +func (tbp *TestsByPlatform) mergeOSes(from TestsByPlatform) { + for testOS, testsByOS := range from.OperatingSystems { + // iterate over all the OS definitions, ensuring that the entry exists in the destination map + existingTestsByOS := ensureMapping(tbp.OperatingSystems, testOS, NewTestByOS) + // iterate over source groups for this OS and merge + for grp, tests := range testsByOS.Groups { + // iterate over all the OS definitions, ensuring that the entry exists in the destination map + existingGroup := ensureMapping(existingTestsByOS.Groups, grp, NewTestGroup) + // add all the tests + for testName, testMeta := range tests.Tests { + existingGroup.Tests[testName] = testMeta + } + } + } +} diff --git a/pkg/testing/define/define_autodiscovery_test.go b/pkg/testing/define/define_autodiscovery_test.go new file mode 100644 index 00000000000..23bacde00a4 --- /dev/null +++ b/pkg/testing/define/define_autodiscovery_test.go @@ -0,0 +1,241 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package define + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +type NamedThing struct { + name string +} + +func (n *NamedThing) Name() string { + return n.name +} + +func Test_discoverTest(t *testing.T) { + type inputTest struct { + n Named + reqs Requirements + } + type args struct { + tests []inputTest + } + tests := []struct { + name string + args args + discoveredYAML string + }{ + { + name: "Default test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_all_default_platforms"}, + reqs: Requirements{ + Group: "foo", + }, + }, + }, + }, + discoveredYAML: ` + - os_family: linux + arch: arm64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: windows + arch: amd64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: darwin + arch: amd64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: darwin + arch: arm64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: linux + arch: amd64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false +`, + }, + + { + name: "Only windows test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_windows"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Windows}}, + }, + }, + }, + }, + }, + { + name: "Specific windows version test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_windows"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Windows, Version: "Windows Server 2019"}}, + }, + }, + }, + }, + }, + { + name: "Generic linux test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux}}, + }, + }, + }, + }, + }, + { + name: "Specific linux distro test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux, Distro: "Ubuntu"}}, + }, + }, + }, + }, + }, + { + name: "Specific linux distro and version test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux, Distro: "Ubuntu", Version: "24.04"}}, + }, + }, + }, + }, + }, + { + name: "Mix multiple tests with different groups", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux, Distro: "Ubuntu", Version: "24.04"}}, + }, + }, + { + n: &NamedThing{name: "Test_only_linux2"}, + reqs: Requirements{ + Group: "bar", + OS: []OS{{Type: Linux, Distro: "Ubuntu", Version: "24.04"}}, + }, + }, + { + n: &NamedThing{name: "Test_only_windows"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Windows, Version: "Windows Server 2019"}}, + }, + }, + { + n: &NamedThing{name: "Test_all_default_platforms"}, + reqs: Requirements{ + Group: "foo", + }, + }, + { + n: &NamedThing{name: "Test_all_default_platforms_sudo"}, + reqs: Requirements{ + Group: "bar", + Sudo: true, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + // reset map between testcases + InitAutodiscovery(nil) + t.Run(tt.name, func(t *testing.T) { + for _, ttarg := range tt.args.tests { + discoverTest(ttarg.n, ttarg.reqs) + } + actualTestYaml, err := DumpAutodiscoveryYAML() + t.Logf("Got autodiscovery YAML:\n%s\n", actualTestYaml) + assert.NoError(t, err) + if tt.discoveredYAML != "" { + expected := []OutputRunner{} + err = yaml.Unmarshal([]byte(tt.discoveredYAML), &expected) + require.NoError(t, err, "Error unmarshalling expected YAML") + actual := []OutputRunner{} + err = yaml.Unmarshal(actualTestYaml, &actual) + require.NoError(t, err, "Error unmarshalling actual YAML") + assert.ElementsMatch(t, expected, actual, "Generated runners do not match expected ones") + + } + }) + } +} diff --git a/pkg/testing/define/define_flags.go b/pkg/testing/define/define_flags.go new file mode 100644 index 00000000000..841acba8934 --- /dev/null +++ b/pkg/testing/define/define_flags.go @@ -0,0 +1,79 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package define + +import ( + "flag" + "fmt" + "strconv" + "strings" + "testing" +) + +type optionalBoolFlag struct { + value *bool +} + +func (o *optionalBoolFlag) String() string { + if o.value == nil { + return "nil" + } + return strconv.FormatBool(*o.value) +} + +func (o *optionalBoolFlag) Set(s string) error { + bValue := s == "" || s == "true" + o.value = &bValue + return nil +} + +type stringArrayFlag struct { + values []string +} + +func (s *stringArrayFlag) String() string { + return fmt.Sprintf("%s", s.values) +} + +func (s *stringArrayFlag) Set(stringValue string) error { + if stringValue == "" { + return nil + } + s.values = strings.Split(stringValue, ",") + return nil +} + +var ( + DryRun bool + GroupsFilter stringArrayFlag + PlatformsFilter stringArrayFlag + SudoFilter optionalBoolFlag + AutoDiscover bool + AutoDiscoveryOutput string +) + +func RegisterFlags(prefix string, set *flag.FlagSet) { + set.BoolVar(&DryRun, prefix+"dry-run", false, "Forces test in dry-run mode: skips the main test and puts a successful placeholder /dry-run if the test would have run") + set.Var(&GroupsFilter, prefix+"groups", "test groups, comma-separated") + set.Var(&PlatformsFilter, prefix+"platforms", "test platforms, comma-separated") + set.Var(&SudoFilter, prefix+"sudo", "Filter tests by sudo requirements") + set.BoolVar(&AutoDiscover, prefix+"autodiscover", false, "Auto discover tests (should be used together with -dry-run). Output will be a file that can be set with -autodiscoveryoutput") + set.StringVar(&AutoDiscoveryOutput, prefix+"autodiscoveryoutput", "discovered_tests.yaml", "Set the file location where the structured output for the discovered tests will be stored") +} + +func dryRun(t *testing.T, req Requirements) *Info { + // always validate requirement is valid + if err := req.Validate(); err != nil { + t.Logf("test %s has invalid requirements: %s", t.Name(), err) + t.FailNow() + return nil + } + // skip the test as we are in dry run + t.Run("dry-run", func(t *testing.T) { + t.Log("Test dry-run successful") + }) + t.Skip("Skipped because dry-run mode has been specified.") + return nil +} diff --git a/testing/integration/main_test.go b/testing/integration/main_test.go new file mode 100644 index 00000000000..8167b5c2e89 --- /dev/null +++ b/testing/integration/main_test.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package integration + +import ( + "flag" + "log" + "os" + "testing" + + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +var flagSet = flag.CommandLine + +func init() { + define.RegisterFlags("integration.", flagSet) +} + +func TestMain(m *testing.M) { + flag.Parse() + + if define.AutoDiscover { + define.InitAutodiscovery(nil) + } + + runExitCode := m.Run() + + if define.AutoDiscover { + + discoveredTests, err := define.DumpAutodiscoveryYAML() + if err != nil { + log.Println("Error dumping autodiscovery YAML:", err) + os.Exit(1) + } + + err = os.WriteFile(define.AutoDiscoveryOutput, discoveredTests, 0644) + if err != nil { + log.Printf("Error writing autodiscovery data in %q: %s", define.AutoDiscoveryOutput, err) + os.Exit(1) + } + + } + + os.Exit(runExitCode) +}