From 8a8abd046a1f549557d006031223919d71314f96 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 28 Nov 2023 23:30:00 -0500 Subject: [PATCH] Add ability to split integration tests into different groups (#3544) * Add ability to shard integration tests across muliple hosts. * Change to Group. * Add to dev guide. * Fix magefile. * Apply suggestions from code review Co-authored-by: Craig MacKenzie * Add define.Default constant. * Remove isolate. * Fix tests. * Update groups. * Adjust groups. * Adjust some groups, add debugging SSH key. * Only windows default group with all logs. * Remove windows specific. * Fix add_cloud_metadata allowed errors. * Another allowed error. * Yet another error. --------- Co-authored-by: Craig MacKenzie --- docs/test-framework-dev-guide.md | 21 +++ magefile.go | 15 ++ pkg/testing/define/batch.go | 20 +-- pkg/testing/define/batch_test.go | 158 +++--------------- pkg/testing/define/requirements.go | 19 ++- pkg/testing/define/testdata/sample_test.go | 94 +++++++---- pkg/testing/ogc/provisioner.go | 11 -- pkg/testing/runner/config.go | 4 + pkg/testing/runner/runner.go | 31 ++-- testing/integration/apm_propagation_test.go | 1 + testing/integration/beats_serverless_test.go | 1 + testing/integration/diagnostics_test.go | 2 + testing/integration/endpoint_security_test.go | 32 ++-- testing/integration/fake_test.go | 1 + testing/integration/fqdn_test.go | 1 + testing/integration/groups_test.go | 23 +++ testing/integration/install_test.go | 2 + .../integration/install_unprivileged_test.go | 2 + testing/integration/logs_ingestion_test.go | 6 + testing/integration/package_version_test.go | 1 + testing/integration/proxy_url_test.go | 5 + .../upgrade_broken_package_test.go | 1 + testing/integration/upgrade_downgrade_test.go | 1 + testing/integration/upgrade_fleet_test.go | 9 +- testing/integration/upgrade_gpg_test.go | 2 + testing/integration/upgrade_rollback_test.go | 2 + .../upgrade_standalone_inprogress.go | 3 + .../upgrade_standalone_retry_test.go | 1 + .../integration/upgrade_standalone_test.go | 1 + testing/integration/upgrade_uninstall_test.go | 1 + 30 files changed, 243 insertions(+), 228 deletions(-) create mode 100644 testing/integration/groups_test.go diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index 3e7ef9383b8..9e38e093103 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -67,6 +67,16 @@ between, and it can be very specific or not very specific. > **_NOTE:_** This only filters down the tests based on the platform. It will not execute a tests on a platform unless > the test defines as supporting it. +#### Selecting specific group + +By default, the runner will run all test groups. Each group runs on a dedicated machine instance. When working on groups of tests it's better to limit to a specific +group of tests instead of running all tests. This can be done by using the `TEST_GROUPS="default upgrade-standalone"` +environment variable. This variable can take multiple groups with a space between. + +- `TEST_GROUPS="default" mage integration:test` to execute only tests in the "default" group. +- `TEST_GROUPS="default upgrade-standalone" mage integration:test` to execute only tests in the "default" or +"upgrade-standalone" group. + #### Passing additional go test flags When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`. @@ -168,6 +178,17 @@ the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools` package for helper utilities. +### Test group + +Every `define.Require` must define a `Group` that it belongs too. Each group is executed on a separate instance with all tests with in the same group executed +on the same instance. Placing similar tests in the same group allows those tests to run on its own instance +as well as provides a way for a developer to select a specific group of tests with `TEST_GROUP="{group-name}"`. + +Grouping tests is another way of spreading out the testing load across multiple instances. The more groups that +are defined the more instances will be provisioned to complete all tests. A balance between a small good set of +groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can +out weight the benefits of creating another group. + ### Test namespaces Every test has access to its own unique namespace (a string value). This namespace can diff --git a/magefile.go b/magefile.go index fe320754b91..01b79947fcd 100644 --- a/magefile.go +++ b/magefile.go @@ -1844,6 +1844,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche DiagnosticsDir: diagDir, StateDir: ".integration-cache", Platforms: testPlatforms(), + Groups: testGroups(), Matrix: matrix, SingleTest: singleTest, VerboseMode: mg.Verbose(), @@ -1936,6 +1937,20 @@ func testPlatforms() []string { return platforms } +func testGroups() []string { + groupsStr := os.Getenv("TEST_GROUPS") + if groupsStr == "" { + return nil + } + var groups []string + for _, g := range strings.Split(groupsStr, " ") { + if g != "" { + groups = append(groups, g) + } + } + return groups +} + // Pre-requisite: user must have the gcloud CLI installed func authGCP(ctx context.Context) error { // We only need the service account token to exist. diff --git a/pkg/testing/define/batch.go b/pkg/testing/define/batch.go index 4efa9f5ad4c..24c71fca4a3 100644 --- a/pkg/testing/define/batch.go +++ b/pkg/testing/define/batch.go @@ -41,15 +41,16 @@ var defaultOS = []OS{ // Batch is a grouping of tests that all have the same requirements. type Batch struct { + // Group must be set on each test to define which group the tests belongs. + // Tests that are in the same group are executed on the same runner. + Group string `json:"group"` + // OS defines the operating systems this test batch needs. OS OS `json:"os"` // Stack defines the stack required for this batch. Stack *Stack `json:"stack,omitempty"` - // Isolate defines that this batch is isolated to a single test. - Isolate bool `json:"isolate"` - // Tests define the set of packages and tests that do not require sudo // privileges to be performed. Tests []BatchPackageTests `json:"tests"` @@ -177,15 +178,12 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch } for _, o := range set { var batch Batch - batchIdx := -1 - if !req.Isolate { - batchIdx = findBatchIdx(batches, o, req.Stack) - } + batchIdx := findBatchIdx(batches, req.Group, o, req.Stack) if batchIdx == -1 { // new batch required batch = Batch{ + Group: req.Group, OS: o, - Isolate: req.Isolate, Tests: nil, SudoTests: nil, } @@ -241,10 +239,10 @@ func appendPackageTest(tests []BatchPackageTests, pkg string, name string, stack return tests } -func findBatchIdx(batches []Batch, os OS, stack *Stack) int { +func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int { for i, b := range batches { - if b.Isolate { - // never add to an isolate batch + if b.Group != group { + // must be in the same group continue } if b.OS.Type != os.Type || b.OS.Arch != os.Arch { diff --git a/pkg/testing/define/batch_test.go b/pkg/testing/define/batch_test.go index 0f9afbf3691..a7e265b5f0e 100644 --- a/pkg/testing/define/batch_test.go +++ b/pkg/testing/define/batch_test.go @@ -93,6 +93,7 @@ func TestBatch(t *testing.T) { } expected := []Batch{ { + Group: Default, OS: OS{ Type: Darwin, Arch: AMD64, @@ -101,6 +102,7 @@ func TestBatch(t *testing.T) { SudoTests: darwinSudoTests, }, { + Group: Default, OS: OS{ Type: Darwin, Arch: ARM64, @@ -109,6 +111,7 @@ func TestBatch(t *testing.T) { SudoTests: darwinSudoTests, }, { + Group: Default, OS: OS{ Type: Linux, Arch: AMD64, @@ -117,6 +120,7 @@ func TestBatch(t *testing.T) { SudoTests: linuxSudoTests, }, { + Group: Default, OS: OS{ Type: Linux, Arch: ARM64, @@ -152,6 +156,7 @@ func TestBatch(t *testing.T) { SudoTests: linuxSudoTests, }, { + Group: Default, OS: OS{ Type: Windows, Arch: AMD64, @@ -160,170 +165,47 @@ func TestBatch(t *testing.T) { SudoTests: windowsSudoTests, }, { + Group: "one", OS: OS{ - Type: Darwin, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Darwin, - Arch: ARM64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Linux, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Linux, - Arch: ARM64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Windows, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Darwin, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestDarwinIsolate", - }, - }, - }, + Type: Linux, + Arch: ARM64, + Version: "20.04", + Distro: "ubuntu", }, - }, - { - OS: OS{ - Type: Darwin, - Arch: ARM64, + Stack: &Stack{ + Version: "8.8.0", }, - Isolate: true, Tests: []BatchPackageTests{ { Name: pkgName, Tests: []BatchPackageTest{ { - Name: "TestDarwinIsolate", + Name: "TestGroup_One_One", + Stack: true, }, - }, - }, - }, - }, - { - OS: OS{ - Type: Linux, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ { - Name: "TestLinuxIsolate", + Name: "TestGroup_One_Two", + Stack: true, }, }, }, }, }, { + Group: "two", OS: OS{ Type: Linux, Arch: ARM64, }, - Isolate: true, Tests: []BatchPackageTests{ { Name: pkgName, Tests: []BatchPackageTest{ { - Name: "TestLinuxIsolate", + Name: "TestGroup_Two_One", }, - }, - }, - }, - }, - { - OS: OS{ - Type: Windows, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ { - Name: "TestWindowsIsolate", + Name: "TestGroup_Two_Two", }, }, }, @@ -344,6 +226,7 @@ var testLinuxLocalTests = []BatchPackageTest{ var testLinuxLocalBatch = []Batch{ { + Group: Default, OS: OS{ Type: "linux", Arch: "amd64", @@ -356,6 +239,7 @@ var testLinuxLocalBatch = []Batch{ }, }, { + Group: Default, OS: OS{ Type: "linux", Arch: "arm64", diff --git a/pkg/testing/define/requirements.go b/pkg/testing/define/requirements.go index 2cfc4a35cb4..c62f874009f 100644 --- a/pkg/testing/define/requirements.go +++ b/pkg/testing/define/requirements.go @@ -11,6 +11,11 @@ import ( "github.com/elastic/elastic-agent/pkg/component" ) +const ( + // Default constant can be used as the default group for tests. + Default = "default" +) + const ( // Darwin is macOS platform Darwin = component.Darwin @@ -82,6 +87,13 @@ type Stack struct { // Requirements defines the testing requirements for the test to run. type Requirements struct { + // Group must be set on each test to define which group the tests belongs to. + // Tests that are in the same group are executed on the same runner. + // + // Useful when tests take a long time to complete and sharding them across multiple + // hosts can improve the total amount of time to complete all the tests. + Group string `json:"group"` + // OS defines the operating systems this test can run on. In the case // multiple are provided the test is ran multiple times one time on each // combination. @@ -97,10 +109,6 @@ type Requirements struct { // when a full test run is performed. Local bool `json:"local"` - // Isolate defines that this test must be isolated to its own dedicated VM and the test - // cannot be shared with other tests. - Isolate bool `json:"isolate"` - // Sudo defines that this test must run under superuser permissions. On Mac and Linux the // test gets executed under sudo and on Windows it gets run under Administrator. Sudo bool `json:"sudo"` @@ -108,6 +116,9 @@ type Requirements struct { // Validate returns an error if not valid. func (r Requirements) Validate() error { + if r.Group == "" { + return errors.New("group is required") + } for i, o := range r.OS { if err := o.Validate(); err != nil { return fmt.Errorf("invalid os %d: %w", i, err) diff --git a/pkg/testing/define/testdata/sample_test.go b/pkg/testing/define/testdata/sample_test.go index 51f9e158de8..01095aceaac 100644 --- a/pkg/testing/define/testdata/sample_test.go +++ b/pkg/testing/define/testdata/sample_test.go @@ -14,24 +14,21 @@ import ( func TestAnyLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, Local: true, }) } func TestAnySudo(t *testing.T) { define.Require(t, define.Requirements{ - Sudo: true, - }) -} - -func TestAnyIsolate(t *testing.T) { - define.Require(t, define.Requirements{ - Isolate: true, + Group: define.Default, + Sudo: true, }) } func TestDarwinLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Darwin, @@ -43,6 +40,7 @@ func TestDarwinLocal(t *testing.T) { func TestDarwinSudo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Darwin, @@ -52,19 +50,9 @@ func TestDarwinSudo(t *testing.T) { }) } -func TestDarwinIsolate(t *testing.T) { - define.Require(t, define.Requirements{ - OS: []define.OS{ - { - Type: define.Darwin, - }, - }, - Isolate: true, - }) -} - func TestLinuxLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Linux, @@ -76,6 +64,7 @@ func TestLinuxLocal(t *testing.T) { func TestLinuxSudo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Linux, @@ -85,52 +74,61 @@ func TestLinuxSudo(t *testing.T) { }) } -func TestLinuxIsolate(t *testing.T) { +func TestWindowsLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { - Type: define.Linux, + Type: define.Windows, }, }, - Isolate: true, + Local: true, }) } -func TestWindowsLocal(t *testing.T) { +func TestWindowsSudo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Windows, }, }, - Local: true, + Sudo: true, }) } -func TestWindowsSudo(t *testing.T) { +func TestSpecificCombinationOne(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { - Type: define.Windows, + Type: define.Linux, + Arch: define.ARM64, + Distro: "ubuntu", + Version: "20.04", }, }, - Sudo: true, }) } -func TestWindowsIsolate(t *testing.T) { +func TestSpecificCombinationTwo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { - Type: define.Windows, + Type: define.Linux, + Arch: define.ARM64, + Distro: "ubuntu", + Version: "20.04", }, }, - Isolate: true, }) } -func TestSpecificCombinationOne(t *testing.T) { +func TestSpecificCombinationWithCloud(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Linux, @@ -139,11 +137,15 @@ func TestSpecificCombinationOne(t *testing.T) { Version: "20.04", }, }, + Stack: &define.Stack{ + Version: "8.8.0", + }, }) } -func TestSpecificCombinationTwo(t *testing.T) { +func TestGroup_One_One(t *testing.T) { define.Require(t, define.Requirements{ + Group: "one", OS: []define.OS{ { Type: define.Linux, @@ -152,11 +154,15 @@ func TestSpecificCombinationTwo(t *testing.T) { Version: "20.04", }, }, + Stack: &define.Stack{ + Version: "8.8.0", + }, }) } -func TestSpecificCombinationWithCloud(t *testing.T) { +func TestGroup_One_Two(t *testing.T) { define.Require(t, define.Requirements{ + Group: "one", OS: []define.OS{ { Type: define.Linux, @@ -170,3 +176,27 @@ func TestSpecificCombinationWithCloud(t *testing.T) { }, }) } + +func TestGroup_Two_One(t *testing.T) { + define.Require(t, define.Requirements{ + Group: "two", + OS: []define.OS{ + { + Type: define.Linux, + Arch: define.ARM64, + }, + }, + }) +} + +func TestGroup_Two_Two(t *testing.T) { + define.Require(t, define.Requirements{ + Group: "two", + OS: []define.OS{ + { + Type: define.Linux, + Arch: define.ARM64, + }, + }, + }) +} diff --git a/pkg/testing/ogc/provisioner.go b/pkg/testing/ogc/provisioner.go index 54054d27bca..696fd90c974 100644 --- a/pkg/testing/ogc/provisioner.go +++ b/pkg/testing/ogc/provisioner.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "os" - "path" "path/filepath" "strings" "time" @@ -296,16 +295,6 @@ func osBatchToOGC(cacheDir string, batch runner.OSBatch) Layout { } else { tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Type, strings.Replace(batch.OS.Version, ".", "-", -1)))) } - if batch.Batch.Isolate { - tags = append(tags, "isolate") - var test define.BatchPackageTests - if len(batch.Batch.SudoTests) > 0 { - test = batch.Batch.SudoTests[0] - } else if len(batch.Batch.Tests) > 0 { - test = batch.Batch.Tests[0] - } - tags = append(tags, fmt.Sprintf("%s-%s", path.Base(test.Name), strings.ToLower(test.Tests[0].Name))) - } los, _ := findOSLayout(batch.OS.OS) return Layout{ Name: batch.ID, diff --git a/pkg/testing/runner/config.go b/pkg/testing/runner/config.go index 1448fa197dd..22494ed58e3 100644 --- a/pkg/testing/runner/config.go +++ b/pkg/testing/runner/config.go @@ -32,6 +32,10 @@ type Config struct { // this is used to copy the .tar.gz to the remote host BinaryName string + // Groups filters the tests to only run tests that are part of + // the groups defined in this list. + Groups []string + // Matrix enables matrix testing. This explodes each test to // run on all supported platforms the runner supports. Matrix bool diff --git a/pkg/testing/runner/runner.go b/pkg/testing/runner/runner.go index aab9b695a2b..f6a304c1de5 100644 --- a/pkg/testing/runner/runner.go +++ b/pkg/testing/runner/runner.go @@ -160,11 +160,13 @@ func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches var osBatches []OSBatch for _, b := range batches { - lbs, err := createBatches(b, platforms, cfg.Matrix) + lbs, err := createBatches(b, platforms, cfg.Groups, cfg.Matrix) if err != nil { return nil, err } - osBatches = append(osBatches, lbs...) + if lbs != nil { + osBatches = append(osBatches, lbs...) + } } if cfg.SingleTest != "" { osBatches, err = filterSingleTest(osBatches, cfg.SingleTest) @@ -888,8 +890,20 @@ func findBatchByID(id string, batches []OSBatch) (OSBatch, bool) { return OSBatch{}, false } -func createBatches(batch define.Batch, platforms []define.OS, matrix bool) ([]OSBatch, error) { +func batchInGroups(batch define.Batch, groups []string) bool { + for _, g := range groups { + if batch.Group == g { + return true + } + } + return false +} + +func createBatches(batch define.Batch, platforms []define.OS, groups []string, matrix bool) ([]OSBatch, error) { var batches []OSBatch + if len(groups) > 0 && !batchInGroups(batch, groups) { + return nil, nil + } specifics, err := getSupported(batch.OS, platforms) if errors.Is(err, ErrOSNotSupported) { var s SupportedOS @@ -1011,16 +1025,7 @@ func createBatchID(batch OSBatch) string { id += "-" + batch.OS.Distro } id += "-" + strings.Replace(batch.OS.Version, ".", "", -1) - if batch.Batch.Isolate { - if len(batch.Batch.Tests) > 0 { - // only ever has one test in an isolated batch - id += "-" + batch.Batch.Tests[0].Tests[0].Name - } - if len(batch.Batch.SudoTests) > 0 { - // only ever has one test in an isolated batch - id += "-" + batch.Batch.SudoTests[0].Tests[0].Name - } - } + id += "-" + strings.Replace(batch.Batch.Group, ".", "", -1) // The batchID needs to be at most 63 characters long otherwise // OGC will fail to instantiate the VM. diff --git a/testing/integration/apm_propagation_test.go b/testing/integration/apm_propagation_test.go index 62aa87b0d83..c1c573773c8 100644 --- a/testing/integration/apm_propagation_test.go +++ b/testing/integration/apm_propagation_test.go @@ -53,6 +53,7 @@ agent.monitoring: func TestAPMConfig(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Default, Stack: &define.Stack{}, }) f, err := define.NewFixture(t, define.Version()) diff --git a/testing/integration/beats_serverless_test.go b/testing/integration/beats_serverless_test.go index 57123e9142e..730d1d5075e 100644 --- a/testing/integration/beats_serverless_test.go +++ b/testing/integration/beats_serverless_test.go @@ -50,6 +50,7 @@ type BeatRunner struct { func TestBeatsServerless(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Default, OS: []define.OS{ {Type: define.Linux}, }, diff --git a/testing/integration/diagnostics_test.go b/testing/integration/diagnostics_test.go index 00d8d97aed0..c56b89c3e9a 100644 --- a/testing/integration/diagnostics_test.go +++ b/testing/integration/diagnostics_test.go @@ -88,6 +88,7 @@ type componentAndUnitNames struct { func TestDiagnosticsOptionalValues(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: false, }) @@ -113,6 +114,7 @@ func TestDiagnosticsOptionalValues(t *testing.T) { func TestDiagnosticsCommand(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: false, }) diff --git a/testing/integration/endpoint_security_test.go b/testing/integration/endpoint_security_test.go index e58afa72c45..de326d1b45a 100644 --- a/testing/integration/endpoint_security_test.go +++ b/testing/integration/endpoint_security_test.go @@ -74,10 +74,10 @@ var protectionTests = []struct { // test automatically. func TestInstallAndCLIUninstallWithEndpointSecurity(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation OS: []define.OS{ {Type: define.Linux}, }, @@ -100,10 +100,10 @@ func TestInstallAndCLIUninstallWithEndpointSecurity(t *testing.T) { // but at this point endpoint is already uninstalled. func TestInstallAndUnenrollWithEndpointSecurity(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation OS: []define.OS{ {Type: define.Linux}, }, @@ -128,10 +128,10 @@ func TestInstallAndUnenrollWithEndpointSecurity(t *testing.T) { func TestInstallWithEndpointSecurityAndRemoveEndpointIntegration(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation OS: []define.OS{ {Type: define.Linux}, }, @@ -491,10 +491,10 @@ func installElasticDefendPackage(t *testing.T, info *define.Info, policyID strin // path other than default func TestEndpointSecurityNonDefaultBasePath(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation }) ctx, cn := context.WithCancel(context.Background()) diff --git a/testing/integration/fake_test.go b/testing/integration/fake_test.go index cb685e4cb4a..2a17931f359 100644 --- a/testing/integration/fake_test.go +++ b/testing/integration/fake_test.go @@ -44,6 +44,7 @@ inputs: func TestFakeComponent(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: true, }) diff --git a/testing/integration/fqdn_test.go b/testing/integration/fqdn_test.go index 9bb1b34de89..4478de0c76b 100644 --- a/testing/integration/fqdn_test.go +++ b/testing/integration/fqdn_test.go @@ -32,6 +32,7 @@ import ( func TestFQDN(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Default, // placed in default only because its skipped OS: []define.OS{ {Type: define.Linux}, }, diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go new file mode 100644 index 00000000000..64d8cd7cf02 --- /dev/null +++ b/testing/integration/groups_test.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration + +package integration + +import "github.com/elastic/elastic-agent/pkg/testing/define" + +const ( + // Default group. + Default = define.Default + + // Fleet group of tests. Used for testing Elastic Agent with Fleet. + Fleet = "fleet" + + // FleetAirgapped group of tests. Used for testing Elastic Agent with Fleet and airgapped. + FleetAirgapped = "fleet-airgapped" + + // Upgrade group of tests. Used for testing upgrades. + Upgrade = "upgrade" +) diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go index e2e9d0206cf..40df66b1d15 100644 --- a/testing/integration/install_test.go +++ b/testing/integration/install_test.go @@ -24,6 +24,7 @@ import ( func TestInstallWithoutBasePath(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, // We require sudo for this test to run // `elastic-agent install`. Sudo: true, @@ -71,6 +72,7 @@ func TestInstallWithoutBasePath(t *testing.T) { func TestInstallWithBasePath(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, // We require sudo for this test to run // `elastic-agent install`. Sudo: true, diff --git a/testing/integration/install_unprivileged_test.go b/testing/integration/install_unprivileged_test.go index 6c260a42de0..a2700870df8 100644 --- a/testing/integration/install_unprivileged_test.go +++ b/testing/integration/install_unprivileged_test.go @@ -27,6 +27,7 @@ import ( func TestInstallUnprivilegedWithoutBasePath(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, // We require sudo for this test to run // `elastic-agent install` (even though it will // be installed as non-root). @@ -80,6 +81,7 @@ func TestInstallUnprivilegedWithoutBasePath(t *testing.T) { func TestInstallUnprivilegedWithBasePath(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, // We require sudo for this test to run // `elastic-agent install` (even though it will // be installed as non-root). diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index b0dcbc9b4c3..ff5c19f0405 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -38,6 +38,7 @@ import ( func TestLogIngestionFleetManaged(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Fleet, Stack: &define.Stack{}, Local: false, Sudo: true, @@ -133,6 +134,11 @@ func testMonitoringLogsAreShipped( "Failed to initialize artifact", "Failed to apply initial policy from on disk configuration", "elastic-agent-client error: rpc error: code = Canceled desc = context canceled", // can happen on restart + "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-id\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the openstack metadata to not work + "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/hostname\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/placement/availability-zone\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed with http status code 404", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed fetching EC2 Identity Document: operation error ec2imds: GetInstanceIdentityDocument, http response error StatusCode: 404, request to EC2 IMDS failed", // okay for the cloud metadata to not work }) }) t.Logf("error logs: Got %d documents", len(docs.Hits.Hits)) diff --git a/testing/integration/package_version_test.go b/testing/integration/package_version_test.go index 2b8ca6ebe90..b2d560526f5 100644 --- a/testing/integration/package_version_test.go +++ b/testing/integration/package_version_test.go @@ -22,6 +22,7 @@ import ( func TestPackageVersion(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: true, }) diff --git a/testing/integration/proxy_url_test.go b/testing/integration/proxy_url_test.go index e95469405be..c9ae99a248a 100644 --- a/testing/integration/proxy_url_test.go +++ b/testing/integration/proxy_url_test.go @@ -91,6 +91,7 @@ func TearDownTest(t *testing.T, p *ProxyURL) { func TestProxyURL_EnrollProxyAndNoProxyInThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -135,6 +136,7 @@ func TestProxyURL_EnrollProxyAndNoProxyInThePolicy(t *testing.T) { func TestProxyURL_EnrollProxyAndEmptyProxyInThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -180,6 +182,7 @@ func TestProxyURL_EnrollProxyAndEmptyProxyInThePolicy(t *testing.T) { func TestProxyURL_ProxyInThePolicyTakesPrecedence(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -239,6 +242,7 @@ func TestProxyURL_ProxyInThePolicyTakesPrecedence(t *testing.T) { func TestProxyURL_NoEnrollProxyAndProxyInThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -302,6 +306,7 @@ func TestProxyURL_NoEnrollProxyAndProxyInThePolicy(t *testing.T) { func TestProxyURL_RemoveProxyFromThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) diff --git a/testing/integration/upgrade_broken_package_test.go b/testing/integration/upgrade_broken_package_test.go index a9f090d132b..635480c0f36 100644 --- a/testing/integration/upgrade_broken_package_test.go +++ b/testing/integration/upgrade_broken_package_test.go @@ -25,6 +25,7 @@ import ( func TestUpgradeBrokenPackageVersion(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_downgrade_test.go b/testing/integration/upgrade_downgrade_test.go index ef4afd4ae41..8d8950c52d4 100644 --- a/testing/integration/upgrade_downgrade_test.go +++ b/testing/integration/upgrade_downgrade_test.go @@ -23,6 +23,7 @@ import ( func TestStandaloneDowngradeToSpecificSnapshotBuild(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go index 8d4f3b3c88d..c06d9f10843 100644 --- a/testing/integration/upgrade_fleet_test.go +++ b/testing/integration/upgrade_fleet_test.go @@ -41,6 +41,7 @@ import ( // versions as the standalone tests already perform those tests and would be redundant. func TestFleetManagedUpgrade(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Fleet, Stack: &define.Stack{}, Local: false, // requires Agent installation Sudo: true, // requires Agent installation @@ -91,12 +92,12 @@ func TestFleetManagedUpgrade(t *testing.T) { func TestFleetAirGappedUpgrade(t *testing.T) { stack := define.Require(t, define.Requirements{ + Group: FleetAirgapped, Stack: &define.Stack{}, // The test uses iptables to simulate the air-gaped environment. - OS: []define.OS{{Type: define.Linux}}, - Isolate: true, // Needed as the test blocks IPs using iptables. - Local: false, // Needed as the test requires Agent installation - Sudo: true, // Needed as the test uses iptables and installs the Agent + OS: []define.OS{{Type: define.Linux}}, + Local: false, // Needed as the test requires Agent installation + Sudo: true, // Needed as the test uses iptables and installs the Agent }) ctx, _ := testcontext.WithDeadline( diff --git a/testing/integration/upgrade_gpg_test.go b/testing/integration/upgrade_gpg_test.go index e2001dc1eca..96f7d8b50ea 100644 --- a/testing/integration/upgrade_gpg_test.go +++ b/testing/integration/upgrade_gpg_test.go @@ -22,6 +22,7 @@ import ( func TestStandaloneUpgradeWithGPGFallback(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) @@ -77,6 +78,7 @@ func TestStandaloneUpgradeWithGPGFallback(t *testing.T) { func TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_rollback_test.go b/testing/integration/upgrade_rollback_test.go index c91ac967df0..fcdfb3d492a 100644 --- a/testing/integration/upgrade_rollback_test.go +++ b/testing/integration/upgrade_rollback_test.go @@ -40,6 +40,7 @@ agent.upgrade.watcher: // that the Agent is rolled back to the previous version. func TestStandaloneUpgradeRollback(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) @@ -154,6 +155,7 @@ inputs: // rolled back to the previous version. func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_standalone_inprogress.go b/testing/integration/upgrade_standalone_inprogress.go index adc52cf0872..53b2c50fbed 100644 --- a/testing/integration/upgrade_standalone_inprogress.go +++ b/testing/integration/upgrade_standalone_inprogress.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build integration + package integration import ( @@ -24,6 +26,7 @@ import ( // the second upgrade. func TestStandaloneUpgradeFailsWhenUpgradeIsInProgress(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_standalone_retry_test.go b/testing/integration/upgrade_standalone_retry_test.go index 00e74b4cef0..4273452cc50 100644 --- a/testing/integration/upgrade_standalone_retry_test.go +++ b/testing/integration/upgrade_standalone_retry_test.go @@ -26,6 +26,7 @@ import ( func TestStandaloneUpgradeRetryDownload(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_standalone_test.go b/testing/integration/upgrade_standalone_test.go index f1d97bd12b5..d79a7f18da3 100644 --- a/testing/integration/upgrade_standalone_test.go +++ b/testing/integration/upgrade_standalone_test.go @@ -21,6 +21,7 @@ import ( func TestStandaloneUpgrade(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_uninstall_test.go b/testing/integration/upgrade_uninstall_test.go index a26a3bb6d1e..da3b1cae17d 100644 --- a/testing/integration/upgrade_uninstall_test.go +++ b/testing/integration/upgrade_uninstall_test.go @@ -24,6 +24,7 @@ import ( func TestStandaloneUpgradeUninstallKillWatcher(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation })