From 1715dd89170508edab30f316154d7659ebb22b0a Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 12 Sep 2024 10:48:57 +0200 Subject: [PATCH] add integration test upgrading latest release to a build from the current PR (#5457) (cherry picked from commit 1a449cdfbbb5018d338c7ce2c8422ecbcdb17ce2) --- .../artifact/download/fs/verifier_test.go | 2 +- .../artifact/download/http/common_test.go | 2 +- pkg/testing/fixture.go | 77 +++++- pkg/testing/fixture_install.go | 28 +-- testing/integration/groups_test.go | 4 + testing/integration/package_version_test.go | 2 +- testing/integration/upgrade_fleet_test.go | 233 ++++++++++++++++++ testing/pgptest/pgp.go | 4 +- 8 files changed, 317 insertions(+), 35 deletions(-) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go index 7496c0cb9b2..f1c25394b4c 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go @@ -266,7 +266,7 @@ func prepareTestCase(t *testing.T, a artifact.Artifact, version *agtversion.Pars err = os.WriteFile(filePathSHA, []byte(hashContent), 0644) require.NoErrorf(t, err, "could not write %q file", filePathSHA) - pub, sig := pgptest.Sing(t, bytes.NewReader(content)) + pub, sig := pgptest.Sign(t, bytes.NewReader(content)) err = os.WriteFile(filePathASC, sig, 0644) require.NoErrorf(t, err, "could not write %q file", filePathASC) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go b/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go index 3ec8603f047..b7cbfe64620 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go @@ -71,7 +71,7 @@ func getElasticCoServer(t *testing.T) (*httptest.Server, []byte) { var resp []byte content := []byte("anything will do") hash := sha512.Sum512(content) - pub, sig := pgptest.Sing(t, bytes.NewReader(content)) + pub, sig := pgptest.Sign(t, bytes.NewReader(content)) handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { packageName := r.URL.Path[len(sourcePattern):] diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go index 512181494c9..5e0f646e323 100644 --- a/pkg/testing/fixture.go +++ b/pkg/testing/fixture.go @@ -62,6 +62,11 @@ type Fixture struct { // Uninstall token value that is needed for the agent uninstall if it's tamper protected uninstallToken string + + // fileNamePrefix is a prefix to be used when saving files from this test. + // it's set by FileNamePrefix and once it's set, FileNamePrefix will return + // its value. + fileNamePrefix string } // FixtureOpt is an option for the fixture. @@ -1010,16 +1015,13 @@ func (f *Fixture) setClient(c client.Client) { func (f *Fixture) DumpProcesses(suffix string) { procs := getProcesses(f.t, `.*`) - dir, err := findProjectRoot(f.caller) + dir, err := f.DiagnosticsDir() if err != nil { - f.t.Logf("failed to dump process; failed to find project root: %s", err) + f.t.Logf("failed to dump process: %s", err) return } - // Sub-test names are separated by "/" characters which are not valid filenames on Linux. - sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-") - - filePath := filepath.Join(dir, "build", "diagnostics", fmt.Sprintf("TEST-%s-%s-%s-ProcessDump%s.json", sanitizedTestName, f.operatingSystem, f.architecture, suffix)) + filePath := filepath.Join(dir, fmt.Sprintf("%s-ProcessDump%s.json", f.FileNamePrefix(), suffix)) fileDir := path.Dir(filePath) if err := os.MkdirAll(fileDir, 0777); err != nil { f.t.Logf("failed to dump process; failed to create directory %s: %s", fileDir, err) @@ -1044,6 +1046,69 @@ func (f *Fixture) DumpProcesses(suffix string) { } } +// MoveToDiagnosticsDir moves file to 'build/diagnostics' which contents are +// available on CI if the test fails or on the agent's 'build/diagnostics' +// if the test is run locally. +// If the file name does nos start with Fixture.FileNamePrefix(), it'll be added +// to the filename when moving. +func (f *Fixture) MoveToDiagnosticsDir(file string) { + dir, err := f.DiagnosticsDir() + if err != nil { + f.t.Logf("failed to move file to diagnostcs directory: %s", err) + return + } + + filename := filepath.Base(file) + if !strings.HasPrefix(filename, f.FileNamePrefix()) { + filename = fmt.Sprintf("%s-%s", f.FileNamePrefix(), filename) + } + destFile := filepath.Join(dir, filename) + + f.t.Logf("moving %q to %q", file, destFile) + err = os.Rename(file, destFile) + if err != nil { + f.t.Logf("failed to move %q to %q: %v", file, destFile, err) + } +} + +// FileNamePrefix returns a sanitized and unique name to be used as prefix for +// files to be kept as resources for investigation when the test fails. +func (f *Fixture) FileNamePrefix() string { + if f.fileNamePrefix != "" { + return f.fileNamePrefix + } + + stamp := time.Now().Format(time.RFC3339) + // on Windows a filename cannot contain a ':' as this collides with disk + // labels (aka. C:\) + stamp = strings.ReplaceAll(stamp, ":", "-") + + // Subtest names are separated by "/" characters which are not valid + // filenames on Linux. + sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-") + prefix := fmt.Sprintf("%s-%s", sanitizedTestName, stamp) + + f.fileNamePrefix = prefix + return f.fileNamePrefix +} + +// DiagnosticsDir returned {projectRoot}/build/diagnostics path. Files on this path +// are saved if any test fails. Use it to save files for further investigation. +func (f *Fixture) DiagnosticsDir() (string, error) { + dir, err := findProjectRoot(f.caller) + if err != nil { + return "", fmt.Errorf("failed to find project root: %w", err) + } + + diagPath := filepath.Join(dir, "build", "diagnostics") + + if err := os.MkdirAll(diagPath, 0777); err != nil { + return "", fmt.Errorf("failed to create directory %s: %w", diagPath, err) + } + + return diagPath, nil +} + // validateComponents ensures that the provided UsableComponent's are valid. func validateComponents(components ...UsableComponent) error { for idx, comp := range components { diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go index 5430252150f..fb796c0e0c5 100644 --- a/pkg/testing/fixture_install.go +++ b/pkg/testing/fixture_install.go @@ -649,7 +649,7 @@ func (f *Fixture) collectDiagnostics() { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - diagPath, err := f.DiagDir() + diagPath, err := f.DiagnosticsDir() if err != nil { f.t.Logf("failed to collect diagnostics: %v", err) return @@ -661,15 +661,8 @@ func (f *Fixture) collectDiagnostics() { return } - stamp := time.Now().Format(time.RFC3339) - if runtime.GOOS == "windows" { - // on Windows a filename cannot contain a ':' as this collides with disk labels (aka. C:\) - stamp = strings.ReplaceAll(stamp, ":", "-") - } - - // Sub-test names are separated by "/" characters which are not valid filenames on Linux. - sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-") - outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", sanitizedTestName, stamp)) + prefix := f.FileNamePrefix() + outputPath := filepath.Join(diagPath, prefix+"-diagnostics.zip") output, err := f.Exec(ctx, []string{"diagnostics", "-f", outputPath}) if err != nil { @@ -689,8 +682,7 @@ func (f *Fixture) collectDiagnostics() { if err != nil { // If collecting diagnostics fails, zip up the entire installation directory with the hope that it will contain logs. f.t.Logf("creating zip archive of the installation directory: %s", f.workDir) - timestamp := strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "-") - zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory-%s.zip", sanitizedTestName, timestamp)) + zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory.zip", prefix)) err = f.archiveInstallDirectory(f.workDir, zipPath) if err != nil { f.t.Logf("failed to zip install directory to %s: %s", zipPath, err) @@ -699,18 +691,6 @@ func (f *Fixture) collectDiagnostics() { } } -// DiagDir returned {projectRoot}/build/diagnostics path. Files on this path -// are saved if any test fails. Use it to save files for further investigation. -func (f *Fixture) DiagDir() (string, error) { - dir, err := findProjectRoot(f.caller) - if err != nil { - return "", fmt.Errorf("failed to find project root: %w", err) - } - - diagPath := filepath.Join(dir, "build", "diagnostics") - return diagPath, nil -} - func (f *Fixture) archiveInstallDirectory(installPath string, outputPath string) error { file, err := os.Create(outputPath) if err != nil { diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go index b4bb4500dad..61c2978741a 100644 --- a/testing/integration/groups_test.go +++ b/testing/integration/groups_test.go @@ -25,6 +25,10 @@ const ( // privileged and airgapped. FleetAirgappedPrivileged = "fleet-airgapped-privileged" + // FleetUpgradeToPRBuild group of tests. Used for testing Elastic Agent + // upgrading to a build built from the PR being tested. + FleetUpgradeToPRBuild = "fleet-upgrade-to-pr-build" + // FQDN group of tests. Used for testing Elastic Agent with FQDN enabled. FQDN = "fqdn" diff --git a/testing/integration/package_version_test.go b/testing/integration/package_version_test.go index 1e84c96a45a..6b97a45d9e8 100644 --- a/testing/integration/package_version_test.go +++ b/testing/integration/package_version_test.go @@ -207,7 +207,7 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) { } t.Logf("the test failed: trying to save the diagnostics used on the test") - diagDir, err := f.DiagDir() + diagDir, err := f.DiagnosticsDir() if err != nil { t.Logf("could not get diagnostics directory to save the diagnostics used on the test") return diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go index 3c6d8fa759f..0c4026460a6 100644 --- a/testing/integration/upgrade_fleet_test.go +++ b/testing/integration/upgrade_fleet_test.go @@ -8,8 +8,10 @@ package integration import ( "context" + "crypto/tls" "errors" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -17,6 +19,7 @@ import ( "os/exec" "path/filepath" "runtime" + "sort" "strings" "testing" "time" @@ -26,12 +29,14 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent-libs/testing/certutil" atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/tools/check" "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" "github.com/elastic/elastic-agent/pkg/testing/tools/testcontext" "github.com/elastic/elastic-agent/pkg/version" + "github.com/elastic/elastic-agent/testing/pgptest" "github.com/elastic/elastic-agent/testing/upgradetest" ) @@ -127,6 +132,147 @@ func TestFleetAirGappedUpgradePrivileged(t *testing.T) { testFleetAirGappedUpgrade(t, stack, false) } +func TestFleetUpgradeToPRBuild(t *testing.T) { + stack := define.Require(t, define.Requirements{ + Group: FleetUpgradeToPRBuild, + Stack: &define.Stack{}, + OS: []define.OS{{Type: define.Linux}}, // The test uses /etc/hosts. + Sudo: true, // The test uses /etc/hosts. + // The test requires: + // - bind to port 443 (HTTPS) + // - changes to /etc/hosts + // - changes to /etc/ssl/certs + // - agent installation + Local: false, + }) + + ctx := context.Background() + + // ========================= prepare from fixture ========================== + versions, err := upgradetest.GetUpgradableVersions() + require.NoError(t, err, "could not get upgradable versions") + + sortedVers := version.SortableParsedVersions(versions) + sort.Sort(sort.Reverse(sortedVers)) + + t.Logf("upgradable versions: %v", versions) + var latestRelease version.ParsedSemVer + for _, v := range versions { + if !v.IsSnapshot() { + latestRelease = *v + break + } + } + fromFixture, err := atesting.NewFixture(t, + latestRelease.String()) + require.NoError(t, err, "could not create fixture for latest release") + // make sure to download it before the test impersonates artifacts API + err = fromFixture.Prepare(ctx) + require.NoError(t, err, "could not prepare fromFixture") + + rootDir := t.TempDir() + rootPair, childPair, cert := prepareTLSCerts( + t, "artifacts.elastic.co", []net.IP{net.ParseIP("127.0.0.1")}) + + // ==================== prepare to fixture from PR build =================== + toFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err, "failed to get fixture with PR build") + + prBuildPkgPath, err := toFixture.SrcPackage(ctx) + require.NoError(t, err, "could not get path to PR build artifact") + + agentPkg, err := os.Open(prBuildPkgPath) + require.NoError(t, err, "could not open PR build artifact") + + // sign the build + pubKey, ascData := pgptest.Sign(t, agentPkg) + + // ========================== file server ================================== + downloadDir := filepath.Join(rootDir, "downloads", "beats", "elastic-agent") + err = os.MkdirAll(downloadDir, 0644) + require.NoError(t, err, "could not create download directory") + + server := startHTTPSFileServer(t, rootDir, cert) + defer server.Close() + + // add root CA to /etc/ssl/certs. It was the only option that worked + rootCAPath := filepath.Join("/etc/ssl/certs", "TestFleetUpgradeToPRBuild.pem") + err = os.WriteFile( + rootCAPath, + rootPair.Cert, 0440) + require.NoError(t, err, "could not write root CA to /etc/ssl/certs") + t.Cleanup(func() { + if err = os.Remove(rootCAPath); err != nil { + t.Log("cleanup: could not remove root CA") + } + }) + + // ====================== copy files to file server ====================== + // copy the agent package + _, filename := filepath.Split(prBuildPkgPath) + pkgDownloadPath := filepath.Join(downloadDir, filename) + copyFile(t, prBuildPkgPath, pkgDownloadPath) + copyFile(t, prBuildPkgPath+".sha512", pkgDownloadPath+".sha512") + + // copy the PGP key + gpgKeyElasticAgent := filepath.Join(rootDir, "GPG-KEY-elastic-agent") + err = os.WriteFile( + gpgKeyElasticAgent, pubKey, 0o644) + require.NoError(t, err, "could not write GPG-KEY-elastic-agent to disk") + + // copy the package signature + ascFile := filepath.Join(downloadDir, filename+".asc") + err = os.WriteFile( + ascFile, ascData, 0o600) + require.NoError(t, err, "could not write agent .asc file to disk") + + defer func() { + if !t.Failed() { + return + } + + prefix := fromFixture.FileNamePrefix() + "-" + + if err = os.WriteFile(filepath.Join(rootDir, prefix+"server.pem"), childPair.Cert, 0o777); err != nil { + t.Log("cleanup: could not save server cert for investigation") + } + if err = os.WriteFile(filepath.Join(rootDir, prefix+"server_key.pem"), childPair.Key, 0o777); err != nil { + t.Log("cleanup: could not save server cert key for investigation") + } + + if err = os.WriteFile(filepath.Join(rootDir, prefix+"server_key.pem"), rootPair.Key, 0o777); err != nil { + t.Log("cleanup: could not save rootCA key for investigation") + } + + toFixture.MoveToDiagnosticsDir(rootCAPath) + toFixture.MoveToDiagnosticsDir(pkgDownloadPath) + toFixture.MoveToDiagnosticsDir(pkgDownloadPath + ".sha512") + toFixture.MoveToDiagnosticsDir(gpgKeyElasticAgent) + toFixture.MoveToDiagnosticsDir(ascFile) + }() + + // ==== impersonate https://artifacts.elastic.co/GPG-KEY-elastic-agent ==== + impersonateHost(t, "artifacts.elastic.co", "127.0.0.1") + + // ==================== prepare agent's download source ==================== + downloadSource := kibana.DownloadSource{ + Name: "self-signed-" + uuid.Must(uuid.NewV4()).String(), + Host: server.URL + "/downloads/", + IsDefault: false, // other tests reuse the stack, let's not mess things up + } + + t.Logf("creating download source %q, using %q.", + downloadSource.Name, downloadSource.Host) + src, err := stack.KibanaClient.CreateDownloadSource(ctx, downloadSource) + require.NoError(t, err, "could not create download source") + policy := defaultPolicy() + policy.DownloadSourceID = src.Item.ID + t.Logf("policy %s using DownloadSourceID: %s", + policy.ID, policy.DownloadSourceID) + + testUpgradeFleetManagedElasticAgent(ctx, t, stack, fromFixture, toFixture, policy, false) +} + func testFleetAirGappedUpgrade(t *testing.T, stack *define.Info, unprivileged bool) { ctx, _ := testcontext.WithDeadline( t, context.Background(), time.Now().Add(10*time.Minute)) @@ -203,6 +349,7 @@ func testUpgradeFleetManagedElasticAgent( endFixture *atesting.Fixture, policy kibana.AgentPolicy, unprivileged bool) { + kibClient := info.KibanaClient startVersionInfo, err := startFixture.ExecVersion(ctx) @@ -426,3 +573,89 @@ func agentUpgradeDetailsString(a *kibana.AgentExisting) string { return fmt.Sprintf("%#v", *a.UpgradeDetails) } + +// startHTTPSFileServer prepares and returns a started HTTPS file server serving +// files from rootDir and using cert as its TLS certificate. +func startHTTPSFileServer(t *testing.T, rootDir string, cert tls.Certificate) *httptest.Server { + // it's useful for debugging + dl, err := os.ReadDir(rootDir) + require.NoError(t, err) + var files []string + for _, d := range dl { + files = append(files, d.Name()) + } + fmt.Printf("ArtifactsServer root dir %q, served files %q\n", + rootDir, files) + + fs := http.FileServer(http.Dir(rootDir)) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Logf("[fileserver] %s - %s", r.Method, r.URL.Path) + fs.ServeHTTP(w, r) + })) + + server.Listener, err = net.Listen("tcp", "127.0.0.1:443") + require.NoError(t, err, "could not create net listener for port 443") + + server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + server.StartTLS() + t.Logf("file server running on %s", server.URL) + + return server +} + +// prepareTLSCerts generates a CA and a child certificate for the given host and +// IPs. +func prepareTLSCerts(t *testing.T, host string, ips []net.IP) (certutil.Pair, certutil.Pair, tls.Certificate) { + rootKey, rootCACert, rootPair, err := certutil.NewRootCA() + require.NoError(t, err, "could not create root CA") + + _, childPair, err := certutil.GenerateChildCert( + host, + ips, + rootKey, + rootCACert) + require.NoError(t, err, "could not create child cert") + + cert, err := tls.X509KeyPair(childPair.Cert, childPair.Key) + require.NoError(t, err, "could not create tls.Certificates from child certificate") + + return rootPair, childPair, cert +} + +// impersonateHost impersonates 'host' by adding an entry to /etc/hosts mapping +// 'ip' to 'host'. +// It registers a function with t.Cleanup to restore /etc/hosts to its original +// state. +func impersonateHost(t *testing.T, host string, ip string) { + copyFile(t, "/etc/hosts", "/etc/hosts.old") + + entry := fmt.Sprintf("\n%s\t%s\n", ip, host) + f, err := os.OpenFile("/etc/hosts", os.O_WRONLY|os.O_APPEND, 0o644) + require.NoError(t, err, "could not open file for append") + + _, err = f.Write([]byte(entry)) + require.NoError(t, err, "could not write data to file") + require.NoError(t, f.Close(), "could not close file") + + t.Cleanup(func() { + err := os.Rename("/etc/hosts.old", "/etc/hosts") + require.NoError(t, err, "could not restore /etc/hosts") + }) +} + +func copyFile(t *testing.T, srcPath, dstPath string) { + t.Logf("copyFile: src %q, dst %q", srcPath, dstPath) + src, err := os.Open(srcPath) + require.NoError(t, err, "Failed to open source file") + defer src.Close() + + dst, err := os.Create(dstPath) + require.NoError(t, err, "Failed to create destination file") + defer dst.Close() + + _, err = io.Copy(dst, src) + require.NoError(t, err, "Failed to copy file") + + err = dst.Sync() + require.NoError(t, err, "Failed to sync dst file") +} diff --git a/testing/pgptest/pgp.go b/testing/pgptest/pgp.go index c6c441536bf..a2a1aea6aee 100644 --- a/testing/pgptest/pgp.go +++ b/testing/pgptest/pgp.go @@ -14,9 +14,9 @@ import ( "golang.org/x/crypto/openpgp/armor" //nolint:staticcheck // It still receives security fixes and it's just test code ) -// Sing signs data using RSA. It creates the key, sings data and returns the +// Sign signs data using RSA. It creates the key, sings data and returns the // ASCII armored public key and detached signature. -func Sing(t *testing.T, data io.Reader) ([]byte, []byte) { +func Sign(t *testing.T, data io.Reader) ([]byte, []byte) { pub := &bytes.Buffer{} asc := &bytes.Buffer{}