From 1a449cdfbbb5018d338c7ce2c8422ecbcdb17ce2 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 12 Sep 2024 10:48:57 +0200 Subject: [PATCH 1/4] add integration test upgrading latest release to a build from the current PR (#5457) --- .../artifact/download/fs/verifier_test.go | 2 +- .../artifact/download/http/common_test.go | 2 +- pkg/testing/fixture.go | 77 +++++- pkg/testing/fixture_install.go | 28 +-- testing/integration/groups_test.go | 4 + testing/integration/package_version_test.go | 2 +- testing/integration/upgrade_fleet_test.go | 233 ++++++++++++++++++ testing/pgptest/pgp.go | 4 +- 8 files changed, 317 insertions(+), 35 deletions(-) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go index e4cd6eb167..f27cd899a8 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go @@ -266,7 +266,7 @@ func prepareTestCase(t *testing.T, a artifact.Artifact, version *agtversion.Pars err = os.WriteFile(filePathSHA, []byte(hashContent), 0644) require.NoErrorf(t, err, "could not write %q file", filePathSHA) - pub, sig := pgptest.Sing(t, bytes.NewReader(content)) + pub, sig := pgptest.Sign(t, bytes.NewReader(content)) err = os.WriteFile(filePathASC, sig, 0644) require.NoErrorf(t, err, "could not write %q file", filePathASC) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go b/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go index 1685e3aeef..0a3decbab8 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go @@ -71,7 +71,7 @@ func getElasticCoServer(t *testing.T) (*httptest.Server, []byte) { var resp []byte content := []byte("anything will do") hash := sha512.Sum512(content) - pub, sig := pgptest.Sing(t, bytes.NewReader(content)) + pub, sig := pgptest.Sign(t, bytes.NewReader(content)) handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { packageName := r.URL.Path[len(sourcePattern):] diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go index 0dbedc7f8b..b8619622ab 100644 --- a/pkg/testing/fixture.go +++ b/pkg/testing/fixture.go @@ -62,6 +62,11 @@ type Fixture struct { // Uninstall token value that is needed for the agent uninstall if it's tamper protected uninstallToken string + + // fileNamePrefix is a prefix to be used when saving files from this test. + // it's set by FileNamePrefix and once it's set, FileNamePrefix will return + // its value. + fileNamePrefix string } // FixtureOpt is an option for the fixture. @@ -1010,16 +1015,13 @@ func (f *Fixture) setClient(c client.Client) { func (f *Fixture) DumpProcesses(suffix string) { procs := getProcesses(f.t, `.*`) - dir, err := findProjectRoot(f.caller) + dir, err := f.DiagnosticsDir() if err != nil { - f.t.Logf("failed to dump process; failed to find project root: %s", err) + f.t.Logf("failed to dump process: %s", err) return } - // Sub-test names are separated by "/" characters which are not valid filenames on Linux. - sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-") - - filePath := filepath.Join(dir, "build", "diagnostics", fmt.Sprintf("TEST-%s-%s-%s-ProcessDump%s.json", sanitizedTestName, f.operatingSystem, f.architecture, suffix)) + filePath := filepath.Join(dir, fmt.Sprintf("%s-ProcessDump%s.json", f.FileNamePrefix(), suffix)) fileDir := path.Dir(filePath) if err := os.MkdirAll(fileDir, 0777); err != nil { f.t.Logf("failed to dump process; failed to create directory %s: %s", fileDir, err) @@ -1044,6 +1046,69 @@ func (f *Fixture) DumpProcesses(suffix string) { } } +// MoveToDiagnosticsDir moves file to 'build/diagnostics' which contents are +// available on CI if the test fails or on the agent's 'build/diagnostics' +// if the test is run locally. +// If the file name does nos start with Fixture.FileNamePrefix(), it'll be added +// to the filename when moving. +func (f *Fixture) MoveToDiagnosticsDir(file string) { + dir, err := f.DiagnosticsDir() + if err != nil { + f.t.Logf("failed to move file to diagnostcs directory: %s", err) + return + } + + filename := filepath.Base(file) + if !strings.HasPrefix(filename, f.FileNamePrefix()) { + filename = fmt.Sprintf("%s-%s", f.FileNamePrefix(), filename) + } + destFile := filepath.Join(dir, filename) + + f.t.Logf("moving %q to %q", file, destFile) + err = os.Rename(file, destFile) + if err != nil { + f.t.Logf("failed to move %q to %q: %v", file, destFile, err) + } +} + +// FileNamePrefix returns a sanitized and unique name to be used as prefix for +// files to be kept as resources for investigation when the test fails. +func (f *Fixture) FileNamePrefix() string { + if f.fileNamePrefix != "" { + return f.fileNamePrefix + } + + stamp := time.Now().Format(time.RFC3339) + // on Windows a filename cannot contain a ':' as this collides with disk + // labels (aka. C:\) + stamp = strings.ReplaceAll(stamp, ":", "-") + + // Subtest names are separated by "/" characters which are not valid + // filenames on Linux. + sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-") + prefix := fmt.Sprintf("%s-%s", sanitizedTestName, stamp) + + f.fileNamePrefix = prefix + return f.fileNamePrefix +} + +// DiagnosticsDir returned {projectRoot}/build/diagnostics path. Files on this path +// are saved if any test fails. Use it to save files for further investigation. +func (f *Fixture) DiagnosticsDir() (string, error) { + dir, err := findProjectRoot(f.caller) + if err != nil { + return "", fmt.Errorf("failed to find project root: %w", err) + } + + diagPath := filepath.Join(dir, "build", "diagnostics") + + if err := os.MkdirAll(diagPath, 0777); err != nil { + return "", fmt.Errorf("failed to create directory %s: %w", diagPath, err) + } + + return diagPath, nil +} + // validateComponents ensures that the provided UsableComponent's are valid. func validateComponents(components ...UsableComponent) error { for idx, comp := range components { diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go index f843ef9a3c..8bf833bbea 100644 --- a/pkg/testing/fixture_install.go +++ b/pkg/testing/fixture_install.go @@ -649,7 +649,7 @@ func (f *Fixture) collectDiagnostics() { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - diagPath, err := f.DiagDir() + diagPath, err := f.DiagnosticsDir() if err != nil { f.t.Logf("failed to collect diagnostics: %v", err) return @@ -661,15 +661,8 @@ func (f *Fixture) collectDiagnostics() { return } - stamp := time.Now().Format(time.RFC3339) - if runtime.GOOS == "windows" { - // on Windows a filename cannot contain a ':' as this collides with disk labels (aka. C:\) - stamp = strings.ReplaceAll(stamp, ":", "-") - } - - // Sub-test names are separated by "/" characters which are not valid filenames on Linux. - sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-") - outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", sanitizedTestName, stamp)) + prefix := f.FileNamePrefix() + outputPath := filepath.Join(diagPath, prefix+"-diagnostics.zip") output, err := f.Exec(ctx, []string{"diagnostics", "-f", outputPath}) if err != nil { @@ -689,8 +682,7 @@ func (f *Fixture) collectDiagnostics() { if err != nil { // If collecting diagnostics fails, zip up the entire installation directory with the hope that it will contain logs. f.t.Logf("creating zip archive of the installation directory: %s", f.workDir) - timestamp := strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "-") - zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory-%s.zip", sanitizedTestName, timestamp)) + zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory.zip", prefix)) err = f.archiveInstallDirectory(f.workDir, zipPath) if err != nil { f.t.Logf("failed to zip install directory to %s: %s", zipPath, err) @@ -699,18 +691,6 @@ func (f *Fixture) collectDiagnostics() { } } -// DiagDir returned {projectRoot}/build/diagnostics path. Files on this path -// are saved if any test fails. Use it to save files for further investigation. -func (f *Fixture) DiagDir() (string, error) { - dir, err := findProjectRoot(f.caller) - if err != nil { - return "", fmt.Errorf("failed to find project root: %w", err) - } - - diagPath := filepath.Join(dir, "build", "diagnostics") - return diagPath, nil -} - func (f *Fixture) archiveInstallDirectory(installPath string, outputPath string) error { file, err := os.Create(outputPath) if err != nil { diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go index 6186903824..0440f49f0f 100644 --- a/testing/integration/groups_test.go +++ b/testing/integration/groups_test.go @@ -25,6 +25,10 @@ const ( // privileged and airgapped. FleetAirgappedPrivileged = "fleet-airgapped-privileged" + // FleetUpgradeToPRBuild group of tests. Used for testing Elastic Agent + // upgrading to a build built from the PR being tested. + FleetUpgradeToPRBuild = "fleet-upgrade-to-pr-build" + // FQDN group of tests. Used for testing Elastic Agent with FQDN enabled. FQDN = "fqdn" diff --git a/testing/integration/package_version_test.go b/testing/integration/package_version_test.go index 64cbb5c88c..b3a7816bf3 100644 --- a/testing/integration/package_version_test.go +++ b/testing/integration/package_version_test.go @@ -207,7 +207,7 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) { } t.Logf("the test failed: trying to save the diagnostics used on the test") - diagDir, err := f.DiagDir() + diagDir, err := f.DiagnosticsDir() if err != nil { t.Logf("could not get diagnostics directory to save the diagnostics used on the test") return diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go index 7379658ee9..156baeffa1 100644 --- a/testing/integration/upgrade_fleet_test.go +++ b/testing/integration/upgrade_fleet_test.go @@ -8,8 +8,10 @@ package integration import ( "context" + "crypto/tls" "errors" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -17,6 +19,7 @@ import ( "os/exec" "path/filepath" "runtime" + "sort" "strings" "testing" "time" @@ -26,12 +29,14 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent-libs/testing/certutil" atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/tools/check" "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" "github.com/elastic/elastic-agent/pkg/testing/tools/testcontext" "github.com/elastic/elastic-agent/pkg/version" + "github.com/elastic/elastic-agent/testing/pgptest" "github.com/elastic/elastic-agent/testing/upgradetest" ) @@ -127,6 +132,147 @@ func TestFleetAirGappedUpgradePrivileged(t *testing.T) { testFleetAirGappedUpgrade(t, stack, false) } +func TestFleetUpgradeToPRBuild(t *testing.T) { + stack := define.Require(t, define.Requirements{ + Group: FleetUpgradeToPRBuild, + Stack: &define.Stack{}, + OS: []define.OS{{Type: define.Linux}}, // The test uses /etc/hosts. + Sudo: true, // The test uses /etc/hosts. + // The test requires: + // - bind to port 443 (HTTPS) + // - changes to /etc/hosts + // - changes to /etc/ssl/certs + // - agent installation + Local: false, + }) + + ctx := context.Background() + + // ========================= prepare from fixture ========================== + versions, err := upgradetest.GetUpgradableVersions() + require.NoError(t, err, "could not get upgradable versions") + + sortedVers := version.SortableParsedVersions(versions) + sort.Sort(sort.Reverse(sortedVers)) + + t.Logf("upgradable versions: %v", versions) + var latestRelease version.ParsedSemVer + for _, v := range versions { + if !v.IsSnapshot() { + latestRelease = *v + break + } + } + fromFixture, err := atesting.NewFixture(t, + latestRelease.String()) + require.NoError(t, err, "could not create fixture for latest release") + // make sure to download it before the test impersonates artifacts API + err = fromFixture.Prepare(ctx) + require.NoError(t, err, "could not prepare fromFixture") + + rootDir := t.TempDir() + rootPair, childPair, cert := prepareTLSCerts( + t, "artifacts.elastic.co", []net.IP{net.ParseIP("127.0.0.1")}) + + // ==================== prepare to fixture from PR build =================== + toFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err, "failed to get fixture with PR build") + + prBuildPkgPath, err := toFixture.SrcPackage(ctx) + require.NoError(t, err, "could not get path to PR build artifact") + + agentPkg, err := os.Open(prBuildPkgPath) + require.NoError(t, err, "could not open PR build artifact") + + // sign the build + pubKey, ascData := pgptest.Sign(t, agentPkg) + + // ========================== file server ================================== + downloadDir := filepath.Join(rootDir, "downloads", "beats", "elastic-agent") + err = os.MkdirAll(downloadDir, 0644) + require.NoError(t, err, "could not create download directory") + + server := startHTTPSFileServer(t, rootDir, cert) + defer server.Close() + + // add root CA to /etc/ssl/certs. It was the only option that worked + rootCAPath := filepath.Join("/etc/ssl/certs", "TestFleetUpgradeToPRBuild.pem") + err = os.WriteFile( + rootCAPath, + rootPair.Cert, 0440) + require.NoError(t, err, "could not write root CA to /etc/ssl/certs") + t.Cleanup(func() { + if err = os.Remove(rootCAPath); err != nil { + t.Log("cleanup: could not remove root CA") + } + }) + + // ====================== copy files to file server ====================== + // copy the agent package + _, filename := filepath.Split(prBuildPkgPath) + pkgDownloadPath := filepath.Join(downloadDir, filename) + copyFile(t, prBuildPkgPath, pkgDownloadPath) + copyFile(t, prBuildPkgPath+".sha512", pkgDownloadPath+".sha512") + + // copy the PGP key + gpgKeyElasticAgent := filepath.Join(rootDir, "GPG-KEY-elastic-agent") + err = os.WriteFile( + gpgKeyElasticAgent, pubKey, 0o644) + require.NoError(t, err, "could not write GPG-KEY-elastic-agent to disk") + + // copy the package signature + ascFile := filepath.Join(downloadDir, filename+".asc") + err = os.WriteFile( + ascFile, ascData, 0o600) + require.NoError(t, err, "could not write agent .asc file to disk") + + defer func() { + if !t.Failed() { + return + } + + prefix := fromFixture.FileNamePrefix() + "-" + + if err = os.WriteFile(filepath.Join(rootDir, prefix+"server.pem"), childPair.Cert, 0o777); err != nil { + t.Log("cleanup: could not save server cert for investigation") + } + if err = os.WriteFile(filepath.Join(rootDir, prefix+"server_key.pem"), childPair.Key, 0o777); err != nil { + t.Log("cleanup: could not save server cert key for investigation") + } + + if err = os.WriteFile(filepath.Join(rootDir, prefix+"server_key.pem"), rootPair.Key, 0o777); err != nil { + t.Log("cleanup: could not save rootCA key for investigation") + } + + toFixture.MoveToDiagnosticsDir(rootCAPath) + toFixture.MoveToDiagnosticsDir(pkgDownloadPath) + toFixture.MoveToDiagnosticsDir(pkgDownloadPath + ".sha512") + toFixture.MoveToDiagnosticsDir(gpgKeyElasticAgent) + toFixture.MoveToDiagnosticsDir(ascFile) + }() + + // ==== impersonate https://artifacts.elastic.co/GPG-KEY-elastic-agent ==== + impersonateHost(t, "artifacts.elastic.co", "127.0.0.1") + + // ==================== prepare agent's download source ==================== + downloadSource := kibana.DownloadSource{ + Name: "self-signed-" + uuid.Must(uuid.NewV4()).String(), + Host: server.URL + "/downloads/", + IsDefault: false, // other tests reuse the stack, let's not mess things up + } + + t.Logf("creating download source %q, using %q.", + downloadSource.Name, downloadSource.Host) + src, err := stack.KibanaClient.CreateDownloadSource(ctx, downloadSource) + require.NoError(t, err, "could not create download source") + policy := defaultPolicy() + policy.DownloadSourceID = src.Item.ID + t.Logf("policy %s using DownloadSourceID: %s", + policy.ID, policy.DownloadSourceID) + + testUpgradeFleetManagedElasticAgent(ctx, t, stack, fromFixture, toFixture, policy, false) +} + func testFleetAirGappedUpgrade(t *testing.T, stack *define.Info, unprivileged bool) { ctx, _ := testcontext.WithDeadline( t, context.Background(), time.Now().Add(10*time.Minute)) @@ -203,6 +349,7 @@ func testUpgradeFleetManagedElasticAgent( endFixture *atesting.Fixture, policy kibana.AgentPolicy, unprivileged bool) { + kibClient := info.KibanaClient startVersionInfo, err := startFixture.ExecVersion(ctx) @@ -426,3 +573,89 @@ func agentUpgradeDetailsString(a *kibana.AgentExisting) string { return fmt.Sprintf("%#v", *a.UpgradeDetails) } + +// startHTTPSFileServer prepares and returns a started HTTPS file server serving +// files from rootDir and using cert as its TLS certificate. +func startHTTPSFileServer(t *testing.T, rootDir string, cert tls.Certificate) *httptest.Server { + // it's useful for debugging + dl, err := os.ReadDir(rootDir) + require.NoError(t, err) + var files []string + for _, d := range dl { + files = append(files, d.Name()) + } + fmt.Printf("ArtifactsServer root dir %q, served files %q\n", + rootDir, files) + + fs := http.FileServer(http.Dir(rootDir)) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Logf("[fileserver] %s - %s", r.Method, r.URL.Path) + fs.ServeHTTP(w, r) + })) + + server.Listener, err = net.Listen("tcp", "127.0.0.1:443") + require.NoError(t, err, "could not create net listener for port 443") + + server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + server.StartTLS() + t.Logf("file server running on %s", server.URL) + + return server +} + +// prepareTLSCerts generates a CA and a child certificate for the given host and +// IPs. +func prepareTLSCerts(t *testing.T, host string, ips []net.IP) (certutil.Pair, certutil.Pair, tls.Certificate) { + rootKey, rootCACert, rootPair, err := certutil.NewRootCA() + require.NoError(t, err, "could not create root CA") + + _, childPair, err := certutil.GenerateChildCert( + host, + ips, + rootKey, + rootCACert) + require.NoError(t, err, "could not create child cert") + + cert, err := tls.X509KeyPair(childPair.Cert, childPair.Key) + require.NoError(t, err, "could not create tls.Certificates from child certificate") + + return rootPair, childPair, cert +} + +// impersonateHost impersonates 'host' by adding an entry to /etc/hosts mapping +// 'ip' to 'host'. +// It registers a function with t.Cleanup to restore /etc/hosts to its original +// state. +func impersonateHost(t *testing.T, host string, ip string) { + copyFile(t, "/etc/hosts", "/etc/hosts.old") + + entry := fmt.Sprintf("\n%s\t%s\n", ip, host) + f, err := os.OpenFile("/etc/hosts", os.O_WRONLY|os.O_APPEND, 0o644) + require.NoError(t, err, "could not open file for append") + + _, err = f.Write([]byte(entry)) + require.NoError(t, err, "could not write data to file") + require.NoError(t, f.Close(), "could not close file") + + t.Cleanup(func() { + err := os.Rename("/etc/hosts.old", "/etc/hosts") + require.NoError(t, err, "could not restore /etc/hosts") + }) +} + +func copyFile(t *testing.T, srcPath, dstPath string) { + t.Logf("copyFile: src %q, dst %q", srcPath, dstPath) + src, err := os.Open(srcPath) + require.NoError(t, err, "Failed to open source file") + defer src.Close() + + dst, err := os.Create(dstPath) + require.NoError(t, err, "Failed to create destination file") + defer dst.Close() + + _, err = io.Copy(dst, src) + require.NoError(t, err, "Failed to copy file") + + err = dst.Sync() + require.NoError(t, err, "Failed to sync dst file") +} diff --git a/testing/pgptest/pgp.go b/testing/pgptest/pgp.go index 05413a5def..775dd8cfb3 100644 --- a/testing/pgptest/pgp.go +++ b/testing/pgptest/pgp.go @@ -14,9 +14,9 @@ import ( "golang.org/x/crypto/openpgp/armor" //nolint:staticcheck // It still receives security fixes and it's just test code ) -// Sing signs data using RSA. It creates the key, sings data and returns the +// Sign signs data using RSA. It creates the key, sings data and returns the // ASCII armored public key and detached signature. -func Sing(t *testing.T, data io.Reader) ([]byte, []byte) { +func Sign(t *testing.T, data io.Reader) ([]byte, []byte) { pub := &bytes.Buffer{} asc := &bytes.Buffer{} From 976b6475ca0d9de55e3f5e37098b1bd0814e7bfc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 12 Sep 2024 12:37:59 +0200 Subject: [PATCH 2/4] mergify: avoid label automation always (#5521) Force merging as tests won't pass before a 9.x release is available. --- .mergify.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index a84dad1387..6394541da3 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -123,9 +123,9 @@ pull_request_rules: To fixup this pull request, you need to add the backport labels for the needed branches, such as: * `backport-./d./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit - - name: add backport-8.x label for main only + - name: add backport-8.x label for main only if no skipped or assigned already conditions: - - -label~=^backport-8.x + - -label~=^(backport-skip|backport-8.x)$ - base=main - -merged - -closed @@ -133,6 +133,7 @@ pull_request_rules: comment: message: | `backport-v8.x` has been added to help with the transition to the new branch `8.x`. + If you don't need it please use `backport-skip` label and remove the `backport-8.x` label. label: add: - backport-8.x From 80ad645335d66a620211220257d9e2873762d391 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 12 Sep 2024 12:38:20 +0200 Subject: [PATCH 3/4] mergify: replace queue action for queue_rules (#5517) Force merging as tests won't pass before a 9.x release is available. --- .mergify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index 6394541da3..9aec9904b3 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,5 +1,6 @@ queue_rules: - name: default + merge_method: squash conditions: - check-success=fleet-ci/pr-merge pull_request_rules: @@ -85,7 +86,6 @@ pull_request_rules: - "#approved-reviews-by>=1" actions: queue: - method: squash name: default - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed conditions: From d99b09b0769f6f34428321eedb00c0b4339c202b Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 12 Sep 2024 23:08:09 +0200 Subject: [PATCH 4/4] Improve the integration tests framework documentation (#5461) * Improve the integration tests framework documentation * Fix typo --- docs/test-framework-dev-guide.md | 136 ++++++++++++++++++------------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index 599c0f33af..74db5e7721 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -141,28 +141,70 @@ when running them manually, such as `ELASTICSEARCH_HOST`, `ELASTICSEARCH_USERNAM ### Debugging tests -#### Manually debugging tests on VMs -Many of the integration tests will install the Elastic-Agent and/or -require root to run, which makes it hard to just run them on our work -machines, the best way to circumvent that is to debug the tests -directly on a VM. `mage integration:DeployDebugTools` will show a menu -to select a VM and then install the common debugging tools: Delve, -Mage and Docker. It will also create the `~/elastic-agent` folder -containing the Git repository (required to package from within the VM) +#### Connecting to VMs +All VMs (including Windows) support connections via SSH, the framework +generates and stores the necessary SSH keys to access the VMs, the +easiest way to connect to them is using the SSH command returned by +`mage integration:SSH`. It will list the VMs and ask to select +one. + +On a Unix shell you can run `$(mage integration:SSH)`, the menu is +printed to stderr and the SSH command to stdout. After selecting the +VM you will have shell connected to it. + +#### Credentials for cloud stack/projects +All cloud deployments and projects can be listed with `mage +integration:listStacks`, they can be used to manually connect to +Kibana and Elasticsearch. + +If you need to manually run tests against any deployments, `mage +integration:GenerateEnvFile` will generate a file called `env.sh` that +exports environment variables for Unix compatible shells, you can load +them into your shell by running `source ./env.sh`. + +To easily deploy the credentials to any VM, just run `mage +integration:DeployEnvFile`. A menu will ask for the desired Stack and +VM. + +#### Manually running the tests (using `go test`) +If you want to run the tests manually, skipping the test runner, set the +`TEST_DEFINE_PREFIX` environment variable to any value and run your tests normally +with `go test`. E.g.: + +```shell +TEST_DEFINE_PREFIX=gambiarra go test -v -tags integration -run TestProxyURL ./testing/integration/ +``` + +You will need the environment variables containing the stack URL and +credentials for the tests to succeed. + +#### Installing debug/build tools +`mage integration:DeployDebugTools` will install a few tools necessary + to build the Elastic-Agent in the VM and debug tests: + - Docker + - Delve + - Mage + +When called, it will show a menu to select a VM and then install the +tools listed above. It will also create the `~/elastic-agent` folder +containing the Git repository (required o package from within the VM) and the last version of the code uploaded to the VM. This allows you to easily build/package the Elastic-Agent from within the VM as well as run any tests. -After deploying the debug tools, `mage integrationDeployEnvFile` will -create a `env.sh` and copy it to a selected VM, sourcing it will allow -you to any test against the Cloud Stack you selected. +In the VM there are two important folders: +- `agent`: that is created by the integration test framework and used + by `mage` to run the tests, it gets updated every time you run an + integration test from your machine. +- `elastic-agen`: that is a copy `agent` with Git information created + by `mage integration:DeployDebugTools`, the Git information there is + not a copy from your machine, but it will work if you need to + package the Elastic-Agent from the VM. Most of the time you won't + need it. -Example of how to run a test from within the VM: -``` -## Run a single test -SNAPSHOT=true TEST_PLATFORMS="linux/amd64" mage integration:single TestLogIngestionFleetManaged -## Let's suppose it has failed +**Step-by-Step commands** +```shell ## Install DebugTools mage -v integration:DeployDebugTools @@ -170,31 +212,47 @@ mage -v integration:DeployDebugTools mage -v integration:DeployEnvFile ## SSH into the VM -$(mage integration:SSHVM) +$(mage integration:SSH) ## From inside the VM, the test needs root sudo su source ./env.sh -cd elastic-agent +cd agent # That's the folder the mage automation uses to run the tests + +## Then run the test using `go test` + +TEST_DEFINE_PREFIX=gambiarra AGENT_VERSION="8.16.0-SNAPSHOT" go test -tags=integration -v ./testing/integration/ -run TestLogIngestionFleetManaged + +## Run the test using delve: ## Any flags passed to the test binary go after the '--', they also need to ## include the `test.` prefix if they're for `go test` TEST_DEFINE_PREFIX=gambiarra dlv test ./testing/integration/ --build-flags="-tags integration" -- -test.v -test.run TestLogIngestionFleetManaged ``` -**A Delve trick** +**A Delve trick:** If you didn't build the Elastic-Agent directly on the machine you're debugging, it is very likely the location of the source code is different, hence delve cannot show you the code it is running. To solve this, once on Delve shell, run: `` -config substitute-path /go/src/github.com/elastic/elastic-agent /home/ubuntu/elastic-agent` +config substitute-path /go/src/github.com/elastic/elastic-agent /home/ubuntu/agent `` where: - `/go/src/github.com/elastic/elastic-agent` is the path annotated in the binary you are debugging (the one Delve shows). -- `/home/ubuntu/elastic-agent` is where Delve should read the source +- `/home/ubuntu/agent` is where Delve should read the source code form. +#### Other useful mage targets: +- `integration:stacks` lists all stack deployments and connection + information in a human readable table. +- `integration:listInstances` lists all VMs and their connection + command in a human readable table. It also lists the URL for the + VM page on GCP, which is helpful to verify if the VM still exists + (OGC VMs are automatically deleted) +- `integration:printState` is a shortcut for running the two commands + above. + #### Auto diagnostics retrieval When an integration test fails the testing fixture will try its best to automatically collect the diagnostic information of the installed Elastic Agent. In the case that diagnostics is collected the test runner will @@ -223,42 +281,6 @@ until it reports a failure. - `TEST_RUN_UNTIL_FAILURE=true mage integration:single [testName]` -## Manually running the tests - -If you want to run the tests manually, skipping the test runner, set the -`TEST_DEFINE_PREFIX` environment variable to any value and run your tests normally -with `go test`. E.g.: - -```shell -TEST_DEFINE_PREFIX=gambiarra go test -v -tags integration -run TestProxyURL ./testing/integration/ -``` - -## Connecting to VMs and running tests -### Connecting to VMs -All VMs (including Windows) support connections via SSH, the framework -generates and stores the necessary SSH keys to access the VMs, the -easiest way to connect to them is using the SSH command returned by -`mage integration:SSHVM`. It will list the VMs and ask to select -one. - -On a Unix shell you can run `$(mage integration:SSHVM)`, the menu is -printed to stderr and the SSH command to stdout. After selecting the -VM you will have shell connected to it. - -### Credentials for cloud stack/projects -All cloud deployments and projects can be listed with `mage -integration:stacks`, they can be used to manually connect to -Kibana and Elasticsearch. - -If you need to manually run tests against any deployments, `mage -integration:GenerateEnvFile` will generate a file called `env.sh` that -exports environment variables for Unix compatible shells, you can load -them into your shell by running `source ./env.sh`. - -To easily deploy the credentials to any VM, just run `mage -integration:DeployEnvFile`. A menu will ask for the desired Stack and -VM. - ## Writing tests Write integration and E2E tests by adding them to the `testing/integration`