From 6b95ceaf0b6913aad4e0d8a1adf6af7da5c01edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Mon, 9 Dec 2024 15:19:13 +0100 Subject: [PATCH] Integration tests autodiscovery (#6199) * Define output model and yaml output * Add integration test autodiscovery flags * Change assertions to ElementsMatch in define autodiscovery tests --- pkg/testing/define/define.go | 5 + pkg/testing/define/define_autodiscovery.go | 298 ++++++++++++++++++ .../define/define_autodiscovery_test.go | 241 ++++++++++++++ pkg/testing/define/define_flags.go | 12 +- testing/integration/main_test.go | 22 +- 5 files changed, 571 insertions(+), 7 deletions(-) create mode 100644 pkg/testing/define/define_autodiscovery.go create mode 100644 pkg/testing/define/define_autodiscovery_test.go diff --git a/pkg/testing/define/define.go b/pkg/testing/define/define.go index 5bf988222ea..34262b08ae1 100644 --- a/pkg/testing/define/define.go +++ b/pkg/testing/define/define.go @@ -155,6 +155,11 @@ func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Inf t.Skipf("sudo requirement %t not matching sudo filter %t. Skipping", req.Sudo, *SudoFilter.value) } + // record autodiscover after filtering by group and sudo and before validating against the actual environment + if AutoDiscover { + discoverTest(t, req) + } + if !req.Local && local { t.Skip("running local only tests and this test doesn't support local") return nil diff --git a/pkg/testing/define/define_autodiscovery.go b/pkg/testing/define/define_autodiscovery.go new file mode 100644 index 00000000000..1b4fd558b84 --- /dev/null +++ b/pkg/testing/define/define_autodiscovery.go @@ -0,0 +1,298 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package define + +import ( + "fmt" + "sync" + + "gopkg.in/yaml.v3" +) + +// Default platforms. Overridable using InitAutodiscovery() +var defaultPlatforms = []TestPlatform{ + {OS: Windows, Arch: AMD64}, + + // Not supported by default + // {OS: Windows, Arch: ARM64}, + + // Current batching mechanism support this, not sure it's correct + {OS: Darwin, Arch: AMD64}, + {OS: Darwin, Arch: ARM64}, + {OS: Linux, Arch: AMD64}, + {OS: Linux, Arch: ARM64}, +} + +var defaultTestOS = TestOS{ + Name: "", + Version: "", +} + +var defaultTestPlatform = TestPlatform{ + OS: "", + Arch: "", +} + +// YAML/JSON output structs +type OutputRunner struct { + OSFamily string `json:"os_family" yaml:"os_family" ` + Arch string `json:"arch,omitempty"` + OS string `json:"os,omitempty"` + Version string `json:"version,omitempty"` + Groups []OutputGroup `json:"groups,omitempty"` +} + +type OutputGroup struct { + Name string + Tests []OutputTest +} + +type OutputTest struct { + Name string + Metadata TestMetadata +} + +// structs to aggregate test information +type TestPlatform struct { + OS string `json:"os" yaml:"os"` + Arch string `json:"arch" yaml:"arch"` +} + +type TestMetadata struct { + Local bool `json:"local" yaml:"local"` + Sudo bool `json:"sudo" yaml:"sudo"` +} + +type TestOS struct { + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` +} +type TestGroup struct { + Tests map[string]TestMetadata +} + +func NewTestGroup() TestGroup { + return TestGroup{ + Tests: map[string]TestMetadata{}, + } +} + +type TestByOS struct { + Groups map[string]TestGroup +} + +func NewTestByOS() TestByOS { + return TestByOS{ + Groups: map[string]TestGroup{}, + } +} + +type TestsByPlatform struct { + OperatingSystems map[TestOS]TestByOS `json:"os" yaml:"os"` +} + +func NewTestsByPlatform() TestsByPlatform { + return TestsByPlatform{OperatingSystems: map[TestOS]TestByOS{}} +} + +type DiscoveredTests struct { + Discovered map[TestPlatform]TestsByPlatform +} + +// test autodiscovery aggregator +var testAutodiscovery *DiscoveredTests +var testAutodiscoveryMx sync.Mutex + +type Named interface { + Name() string +} + +func InitAutodiscovery(initDefaultPlatforms []TestPlatform) { + testAutodiscoveryMx.Lock() + defer testAutodiscoveryMx.Unlock() + testAutodiscovery = &DiscoveredTests{ + Discovered: map[TestPlatform]TestsByPlatform{}, + } + + if initDefaultPlatforms != nil { + defaultPlatforms = initDefaultPlatforms + } +} + +func DumpAutodiscoveryYAML() ([]byte, error) { + testAutodiscoveryMx.Lock() + defer testAutodiscoveryMx.Unlock() + err := testAutodiscovery.normalizeDiscoveredTests() + if err != nil { + return nil, fmt.Errorf("normalizing discovered tests: %w", err) + } + + runners := mapToRunners(testAutodiscovery) + + return yaml.Marshal(runners) +} + +func mapToRunners(autodiscovery *DiscoveredTests) []OutputRunner { + + var mapped []OutputRunner + + for pltf, testsByOS := range autodiscovery.Discovered { + for testOS, testsByOS := range testsByOS.OperatingSystems { + or := OutputRunner{ + OSFamily: pltf.OS, + Arch: pltf.Arch, + OS: testOS.Name, + Version: testOS.Version, + Groups: make([]OutputGroup, 0, len(testsByOS.Groups)), + } + + for groupName, groupTests := range testsByOS.Groups { + or.Groups = append(or.Groups, mapGroup(groupName, groupTests)) + } + mapped = append(mapped, or) + } + } + + return mapped +} + +func mapGroup(name string, group TestGroup) OutputGroup { + og := OutputGroup{Name: name, Tests: make([]OutputTest, 0, len(group.Tests))} + for testName, testMetadata := range group.Tests { + og.Tests = append(og.Tests, OutputTest{ + Name: testName, + Metadata: testMetadata, + }) + } + + return og +} + +func discoverTest(test Named, reqs Requirements) { + testAutodiscoveryMx.Lock() + defer testAutodiscoveryMx.Unlock() + for _, p := range getPlatforms(reqs.OS) { + if testAutodiscovery == nil { + panic("testAutodiscovery is nil. Check that InitAutodiscovery() has been called properly") + } + mappedOSesForPlatform := ensureMapping(testAutodiscovery.Discovered, p, NewTestsByPlatform) + osForPlatform := getOSForPlatform(reqs.OS, p) + for _, o := range osForPlatform { + testsByOS := ensureMapping(mappedOSesForPlatform.OperatingSystems, o, NewTestByOS) + testGroup := ensureMapping(testsByOS.Groups, reqs.Group, NewTestGroup) + testGroup.Tests[test.Name()] = TestMetadata{ + Local: reqs.Local, + Sudo: reqs.Sudo, + } + } + } +} + +func ensureMapping[K comparable, V any](mappings map[K]V, k K, newValueCreateFunc func() V) V { + if existingValue, ok := mappings[k]; ok { + return existingValue + } + newValue := newValueCreateFunc() + mappings[k] = newValue + return newValue +} + +func getOSForPlatform(os []OS, p TestPlatform) []TestOS { + + var matchingOSes []TestOS + + for _, o := range os { + if o.Type == p.OS && o.Arch == p.Arch { + matchingOSes = append(matchingOSes, getTestOS(o)) + } + } + + if len(matchingOSes) > 0 { + return matchingOSes + } + + // no other OS has matched, return the default OS + return []TestOS{ + defaultTestOS, + } + +} + +func getTestOS(o OS) TestOS { + switch { + case o.Type == Linux: + return TestOS{ + Name: o.Distro, + Version: o.Version, + } + default: + return TestOS{ + Name: o.Type, + Version: o.Version, + } + } +} + +func getPlatforms(os []OS) []TestPlatform { + if len(os) == 0 { + return []TestPlatform{defaultTestPlatform} + } + + platforms := make([]TestPlatform, 0, len(os)) + for _, o := range os { + platforms = append(platforms, TestPlatform{ + OS: o.Type, + Arch: o.Arch, + }) + } + + return platforms +} + +// Normalization functions +func (dt *DiscoveredTests) normalizeDiscoveredTests() error { + + normalized := map[TestPlatform]TestsByPlatform{} + for pltf, oses := range dt.Discovered { + + if pltf.OS == "" && pltf.Arch != "" { + return fmt.Errorf("platform not supported: %v", pltf) + } + + if pltf.OS != "" && pltf.Arch != "" { + existingOSes := ensureMapping(normalized, pltf, NewTestsByPlatform) // normal case, append to normalized and go to the next platform + existingOSes.mergeOSes(oses) + continue + } + + // Arch and/or OS is not specified: fill in the supported archs for the OS type (potentially for all OSes) + for i, dp := range defaultPlatforms { + if pltf.OS == "" || pltf.OS == dp.OS { + existingOSes := ensureMapping(normalized, defaultPlatforms[i], NewTestsByPlatform) + existingOSes.mergeOSes(oses) + } + } + } + + dt.Discovered = normalized + + return nil +} + +func (tbp *TestsByPlatform) mergeOSes(from TestsByPlatform) { + for testOS, testsByOS := range from.OperatingSystems { + // iterate over all the OS definitions, ensuring that the entry exists in the destination map + existingTestsByOS := ensureMapping(tbp.OperatingSystems, testOS, NewTestByOS) + // iterate over source groups for this OS and merge + for grp, tests := range testsByOS.Groups { + // iterate over all the OS definitions, ensuring that the entry exists in the destination map + existingGroup := ensureMapping(existingTestsByOS.Groups, grp, NewTestGroup) + // add all the tests + for testName, testMeta := range tests.Tests { + existingGroup.Tests[testName] = testMeta + } + } + } +} diff --git a/pkg/testing/define/define_autodiscovery_test.go b/pkg/testing/define/define_autodiscovery_test.go new file mode 100644 index 00000000000..23bacde00a4 --- /dev/null +++ b/pkg/testing/define/define_autodiscovery_test.go @@ -0,0 +1,241 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package define + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +type NamedThing struct { + name string +} + +func (n *NamedThing) Name() string { + return n.name +} + +func Test_discoverTest(t *testing.T) { + type inputTest struct { + n Named + reqs Requirements + } + type args struct { + tests []inputTest + } + tests := []struct { + name string + args args + discoveredYAML string + }{ + { + name: "Default test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_all_default_platforms"}, + reqs: Requirements{ + Group: "foo", + }, + }, + }, + }, + discoveredYAML: ` + - os_family: linux + arch: arm64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: windows + arch: amd64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: darwin + arch: amd64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: darwin + arch: arm64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false + - os_family: linux + arch: amd64 + os: "" + version: "" + groups: + - name: foo + tests: + - name: Test_all_default_platforms + metadata: + local: false + sudo: false +`, + }, + + { + name: "Only windows test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_windows"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Windows}}, + }, + }, + }, + }, + }, + { + name: "Specific windows version test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_windows"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Windows, Version: "Windows Server 2019"}}, + }, + }, + }, + }, + }, + { + name: "Generic linux test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux}}, + }, + }, + }, + }, + }, + { + name: "Specific linux distro test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux, Distro: "Ubuntu"}}, + }, + }, + }, + }, + }, + { + name: "Specific linux distro and version test", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux, Distro: "Ubuntu", Version: "24.04"}}, + }, + }, + }, + }, + }, + { + name: "Mix multiple tests with different groups", + args: args{ + tests: []inputTest{ + { + n: &NamedThing{name: "Test_only_linux"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Linux, Distro: "Ubuntu", Version: "24.04"}}, + }, + }, + { + n: &NamedThing{name: "Test_only_linux2"}, + reqs: Requirements{ + Group: "bar", + OS: []OS{{Type: Linux, Distro: "Ubuntu", Version: "24.04"}}, + }, + }, + { + n: &NamedThing{name: "Test_only_windows"}, + reqs: Requirements{ + Group: "foo", + OS: []OS{{Type: Windows, Version: "Windows Server 2019"}}, + }, + }, + { + n: &NamedThing{name: "Test_all_default_platforms"}, + reqs: Requirements{ + Group: "foo", + }, + }, + { + n: &NamedThing{name: "Test_all_default_platforms_sudo"}, + reqs: Requirements{ + Group: "bar", + Sudo: true, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + // reset map between testcases + InitAutodiscovery(nil) + t.Run(tt.name, func(t *testing.T) { + for _, ttarg := range tt.args.tests { + discoverTest(ttarg.n, ttarg.reqs) + } + actualTestYaml, err := DumpAutodiscoveryYAML() + t.Logf("Got autodiscovery YAML:\n%s\n", actualTestYaml) + assert.NoError(t, err) + if tt.discoveredYAML != "" { + expected := []OutputRunner{} + err = yaml.Unmarshal([]byte(tt.discoveredYAML), &expected) + require.NoError(t, err, "Error unmarshalling expected YAML") + actual := []OutputRunner{} + err = yaml.Unmarshal(actualTestYaml, &actual) + require.NoError(t, err, "Error unmarshalling actual YAML") + assert.ElementsMatch(t, expected, actual, "Generated runners do not match expected ones") + + } + }) + } +} diff --git a/pkg/testing/define/define_flags.go b/pkg/testing/define/define_flags.go index 13eeb27aa75..841acba8934 100644 --- a/pkg/testing/define/define_flags.go +++ b/pkg/testing/define/define_flags.go @@ -46,10 +46,12 @@ func (s *stringArrayFlag) Set(stringValue string) error { } var ( - DryRun bool - GroupsFilter stringArrayFlag - PlatformsFilter stringArrayFlag - SudoFilter optionalBoolFlag + DryRun bool + GroupsFilter stringArrayFlag + PlatformsFilter stringArrayFlag + SudoFilter optionalBoolFlag + AutoDiscover bool + AutoDiscoveryOutput string ) func RegisterFlags(prefix string, set *flag.FlagSet) { @@ -57,6 +59,8 @@ func RegisterFlags(prefix string, set *flag.FlagSet) { set.Var(&GroupsFilter, prefix+"groups", "test groups, comma-separated") set.Var(&PlatformsFilter, prefix+"platforms", "test platforms, comma-separated") set.Var(&SudoFilter, prefix+"sudo", "Filter tests by sudo requirements") + set.BoolVar(&AutoDiscover, prefix+"autodiscover", false, "Auto discover tests (should be used together with -dry-run). Output will be a file that can be set with -autodiscoveryoutput") + set.StringVar(&AutoDiscoveryOutput, prefix+"autodiscoveryoutput", "discovered_tests.yaml", "Set the file location where the structured output for the discovered tests will be stored") } func dryRun(t *testing.T, req Requirements) *Info { diff --git a/testing/integration/main_test.go b/testing/integration/main_test.go index 6b803367773..8167b5c2e89 100644 --- a/testing/integration/main_test.go +++ b/testing/integration/main_test.go @@ -21,11 +21,27 @@ func init() { func TestMain(m *testing.M) { flag.Parse() + + if define.AutoDiscover { + define.InitAutodiscovery(nil) + } + runExitCode := m.Run() - if define.DryRun { - // TODO add parsing of requirements and dump them - log.Print("Dry-run mode specified...") + if define.AutoDiscover { + + discoveredTests, err := define.DumpAutodiscoveryYAML() + if err != nil { + log.Println("Error dumping autodiscovery YAML:", err) + os.Exit(1) + } + + err = os.WriteFile(define.AutoDiscoveryOutput, discoveredTests, 0644) + if err != nil { + log.Printf("Error writing autodiscovery data in %q: %s", define.AutoDiscoveryOutput, err) + os.Exit(1) + } + } os.Exit(runExitCode)