Skip to content

Commit

Permalink
Add ability to split integration tests into different groups (elastic…
Browse files Browse the repository at this point in the history 
…#3544)

* Add ability to shard integration tests across muliple hosts.

* Change to Group.

* Add to dev guide.

* Fix magefile.

* Apply suggestions from code review

Co-authored-by: Craig MacKenzie <[email protected]>

* Add define.Default constant.

* Remove isolate.

* Fix tests.

* Update groups.

* Adjust groups.

* Adjust some groups, add debugging SSH key.

* Only windows default group with all logs.

* Remove windows specific.

* Fix add_cloud_metadata allowed errors.

* Another allowed error.

* Yet another error.

---------

Co-authored-by: Craig MacKenzie <[email protected]>
  • Loading branch information
@blakerouse @cmacknz
blakerouse and cmacknz authored Nov 29, 2023
1 parent a03aa9c commit 8a8abd0
Show file tree
Hide file tree
Showing 30 changed files with 243 additions and 228 deletions.
21 changes: 21 additions & 0 deletions docs/test-framework-dev-guide.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,16 @@ between, and it can be very specific or not very specific.
> **_NOTE:_** This only filters down the tests based on the platform. It will not execute a tests on a platform unless
> the test defines as supporting it.
#### Selecting specific group

By default, the runner will run all test groups. Each group runs on a dedicated machine instance. When working on groups of tests it's better to limit to a specific
group of tests instead of running all tests. This can be done by using the `TEST_GROUPS="default upgrade-standalone"`
environment variable. This variable can take multiple groups with a space between.

- `TEST_GROUPS="default" mage integration:test` to execute only tests in the "default" group.
- `TEST_GROUPS="default upgrade-standalone" mage integration:test` to execute only tests in the "default" or
"upgrade-standalone" group.

#### Passing additional go test flags

When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`.
Expand Down Expand Up @@ -168,6 +178,17 @@ the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test
framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools`
package for helper utilities.

### Test group

Every `define.Require` must define a `Group` that it belongs too. Each group is executed on a separate instance with all tests with in the same group executed
on the same instance. Placing similar tests in the same group allows those tests to run on its own instance
as well as provides a way for a developer to select a specific group of tests with `TEST_GROUP="{group-name}"`.

Grouping tests is another way of spreading out the testing load across multiple instances. The more groups that
are defined the more instances will be provisioned to complete all tests. A balance between a small good set of
groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can
out weight the benefits of creating another group.

### Test namespaces

Every test has access to its own unique namespace (a string value). This namespace can
Expand Down
15 changes: 15 additions & 0 deletions magefile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1844,6 +1844,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
DiagnosticsDir: diagDir,
StateDir: ".integration-cache",
Platforms: testPlatforms(),
Groups: testGroups(),
Matrix: matrix,
SingleTest: singleTest,
VerboseMode: mg.Verbose(),
Expand Down Expand Up @@ -1936,6 +1937,20 @@ func testPlatforms() []string {
return platforms
}

func testGroups() []string {
groupsStr := os.Getenv("TEST_GROUPS")
if groupsStr == "" {
return nil
}
var groups []string
for _, g := range strings.Split(groupsStr, " ") {
if g != "" {
groups = append(groups, g)
}
}
return groups
}

// Pre-requisite: user must have the gcloud CLI installed
func authGCP(ctx context.Context) error {
// We only need the service account token to exist.
Expand Down
20 changes: 9 additions & 11 deletions pkg/testing/define/batch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,16 @@ var defaultOS = []OS{

// Batch is a grouping of tests that all have the same requirements.
type Batch struct {
// Group must be set on each test to define which group the tests belongs.
// Tests that are in the same group are executed on the same runner.
Group string `json:"group"`

// OS defines the operating systems this test batch needs.
OS OS `json:"os"`

// Stack defines the stack required for this batch.
Stack *Stack `json:"stack,omitempty"`

// Isolate defines that this batch is isolated to a single test.
Isolate bool `json:"isolate"`

// Tests define the set of packages and tests that do not require sudo
// privileges to be performed.
Tests []BatchPackageTests `json:"tests"`
Expand Down Expand Up @@ -177,15 +178,12 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch
}
for _, o := range set {
var batch Batch
batchIdx := -1
if !req.Isolate {
batchIdx = findBatchIdx(batches, o, req.Stack)
}
batchIdx := findBatchIdx(batches, req.Group, o, req.Stack)
if batchIdx == -1 {
// new batch required
batch = Batch{
Group: req.Group,
OS: o,
Isolate: req.Isolate,
Tests: nil,
SudoTests: nil,
}
Expand Down Expand Up @@ -241,10 +239,10 @@ func appendPackageTest(tests []BatchPackageTests, pkg string, name string, stack
return tests
}

func findBatchIdx(batches []Batch, os OS, stack *Stack) int {
func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int {
for i, b := range batches {
if b.Isolate {
// never add to an isolate batch
if b.Group != group {
// must be in the same group
continue
}
if b.OS.Type != os.Type || b.OS.Arch != os.Arch {
Expand Down
158 changes: 21 additions & 137 deletions pkg/testing/define/batch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ func TestBatch(t *testing.T) {
}
expected := []Batch{
{
Group: Default,
OS: OS{
Type: Darwin,
Arch: AMD64,
Expand All @@ -101,6 +102,7 @@ func TestBatch(t *testing.T) {
SudoTests: darwinSudoTests,
},
{
Group: Default,
OS: OS{
Type: Darwin,
Arch: ARM64,
Expand All @@ -109,6 +111,7 @@ func TestBatch(t *testing.T) {
SudoTests: darwinSudoTests,
},
{
Group: Default,
OS: OS{
Type: Linux,
Arch: AMD64,
Expand All @@ -117,6 +120,7 @@ func TestBatch(t *testing.T) {
SudoTests: linuxSudoTests,
},
{
Group: Default,
OS: OS{
Type: Linux,
Arch: ARM64,
Expand Down Expand Up @@ -152,6 +156,7 @@ func TestBatch(t *testing.T) {
SudoTests: linuxSudoTests,
},
{
Group: Default,
OS: OS{
Type: Windows,
Arch: AMD64,
Expand All @@ -160,170 +165,47 @@ func TestBatch(t *testing.T) {
SudoTests: windowsSudoTests,
},
{
Group: "one",
OS: OS{
Type: Darwin,
Arch: AMD64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestAnyIsolate",
},
},
},
},
},
{
OS: OS{
Type: Darwin,
Arch: ARM64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestAnyIsolate",
},
},
},
},
},
{
OS: OS{
Type: Linux,
Arch: AMD64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestAnyIsolate",
},
},
},
},
},
{
OS: OS{
Type: Linux,
Arch: ARM64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestAnyIsolate",
},
},
},
},
},
{
OS: OS{
Type: Windows,
Arch: AMD64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestAnyIsolate",
},
},
},
},
},
{
OS: OS{
Type: Darwin,
Arch: AMD64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestDarwinIsolate",
},
},
},
Type: Linux,
Arch: ARM64,
Version: "20.04",
Distro: "ubuntu",
},
},
{
OS: OS{
Type: Darwin,
Arch: ARM64,
Stack: &Stack{
Version: "8.8.0",
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestDarwinIsolate",
Name: "TestGroup_One_One",
Stack: true,
},
},
},
},
},
{
OS: OS{
Type: Linux,
Arch: AMD64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestLinuxIsolate",
Name: "TestGroup_One_Two",
Stack: true,
},
},
},
},
},
{
Group: "two",
OS: OS{
Type: Linux,
Arch: ARM64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestLinuxIsolate",
Name: "TestGroup_Two_One",
},
},
},
},
},
{
OS: OS{
Type: Windows,
Arch: AMD64,
},
Isolate: true,
Tests: []BatchPackageTests{
{
Name: pkgName,
Tests: []BatchPackageTest{
{
Name: "TestWindowsIsolate",
Name: "TestGroup_Two_Two",
},
},
},
Expand All @@ -344,6 +226,7 @@ var testLinuxLocalTests = []BatchPackageTest{

var testLinuxLocalBatch = []Batch{
{
Group: Default,
OS: OS{
Type: "linux",
Arch: "amd64",
Expand All @@ -356,6 +239,7 @@ var testLinuxLocalBatch = []Batch{
},
},
{
Group: Default,
OS: OS{
Type: "linux",
Arch: "arm64",
Expand Down
Loading

0 comments on commit 8a8abd0

Please sign in to comment.