From fdce58b66c7b7aaa7bd9de5cff8bdb0930c1b812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Tue, 5 Dec 2023 22:54:13 +0100 Subject: [PATCH] Pass the parsed version to all downloaders (#3824) * Pass the parsed version to all downloaders * add changelog * Add fs downloader unit tests * add unit tests for http downloader * add tests for snapshot downloader --------- Co-authored-by: Craig MacKenzie --- ...d-metadata-in-upgrade-version-strings.yaml | 32 ++ .../application/upgrade/artifact/artifact.go | 7 +- .../artifact/download/composed/downloader.go | 3 +- .../download/composed/downloader_test.go | 11 +- .../artifact/download/composed/verifier.go | 3 +- .../download/composed/verifier_test.go | 10 +- .../upgrade/artifact/download/downloader.go | 3 +- .../artifact/download/fs/downloader.go | 11 +- .../artifact/download/fs/downloader_test.go | 293 ++++++++++++++++++ .../upgrade/artifact/download/fs/verifier.go | 3 +- .../artifact/download/fs/verifier_test.go | 33 +- .../artifact/download/http/common_test.go | 3 +- .../artifact/download/http/downloader.go | 11 +- .../artifact/download/http/downloader_test.go | 188 +++++++++++ .../artifact/download/http/verifier.go | 5 +- .../artifact/download/http/verifier_test.go | 2 +- .../artifact/download/snapshot/downloader.go | 46 ++- .../download/snapshot/downloader_test.go | 224 ++++++++++++- .../artifact/download/snapshot/verifier.go | 23 +- .../upgrade/artifact/download/verifier.go | 3 +- .../application/upgrade/step_download.go | 4 +- .../application/upgrade/step_download_test.go | 2 +- 22 files changed, 847 insertions(+), 73 deletions(-) create mode 100644 changelog/fragments/1701445320-preserve-build-metadata-in-upgrade-version-strings.yaml create mode 100644 internal/pkg/agent/application/upgrade/artifact/download/fs/downloader_test.go diff --git a/changelog/fragments/1701445320-preserve-build-metadata-in-upgrade-version-strings.yaml b/changelog/fragments/1701445320-preserve-build-metadata-in-upgrade-version-strings.yaml new file mode 100644 index 00000000000..e8d9284e7a5 --- /dev/null +++ b/changelog/fragments/1701445320-preserve-build-metadata-in-upgrade-version-strings.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# Change summary; a 80ish characters long description of the change. +summary: Preserve build metadata in upgrade version strings + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; a word indicating the component this changeset affects. +component: agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/internal/pkg/agent/application/upgrade/artifact/artifact.go b/internal/pkg/agent/application/upgrade/artifact/artifact.go index c0e8c84a9d8..09b73785dbd 100644 --- a/internal/pkg/agent/application/upgrade/artifact/artifact.go +++ b/internal/pkg/agent/application/upgrade/artifact/artifact.go @@ -9,6 +9,7 @@ import ( "path/filepath" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) var packageArchMap = map[string]string{ @@ -31,18 +32,18 @@ type Artifact struct { } // GetArtifactName constructs a path to a downloaded artifact -func GetArtifactName(a Artifact, version, operatingSystem, arch string) (string, error) { +func GetArtifactName(a Artifact, version agtversion.ParsedSemVer, operatingSystem, arch string) (string, error) { key := fmt.Sprintf("%s-binary-%s", operatingSystem, arch) suffix, found := packageArchMap[key] if !found { return "", errors.New(fmt.Sprintf("'%s' is not a valid combination for a package", key), errors.TypeConfig) } - return fmt.Sprintf("%s-%s-%s", a.Cmd, version, suffix), nil + return fmt.Sprintf("%s-%s-%s", a.Cmd, version.String(), suffix), nil } // GetArtifactPath returns a full path of artifact for a program in specific version -func GetArtifactPath(a Artifact, version, operatingSystem, arch, targetDir string) (string, error) { +func GetArtifactPath(a Artifact, version agtversion.ParsedSemVer, operatingSystem, arch, targetDir string) (string, error) { artifactName, err := GetArtifactName(a, version, operatingSystem, arch) if err != nil { return "", err diff --git a/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader.go b/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader.go index b5de15fc9a8..476d5790b63 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader.go @@ -13,6 +13,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + "github.com/elastic/elastic-agent/pkg/version" ) // Downloader is a downloader with a predefined set of downloaders. @@ -35,7 +36,7 @@ func NewDownloader(downloaders ...download.Downloader) *Downloader { // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version string) (string, error) { +func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version *version.ParsedSemVer) (string, error) { var err error span, ctx := apm.StartSpan(ctx, "download", "app.internal") defer span.End() diff --git a/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader_test.go b/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader_test.go index c9820822d6f..26803adeb0b 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/composed/downloader_test.go @@ -9,8 +9,11 @@ import ( "errors" "testing" + "github.com/stretchr/testify/require" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" + agtversion "github.com/elastic/elastic-agent/pkg/version" "github.com/stretchr/testify/assert" ) @@ -23,7 +26,7 @@ type FailingDownloader struct { called bool } -func (d *FailingDownloader) Download(ctx context.Context, _ artifact.Artifact, _ string) (string, error) { +func (d *FailingDownloader) Download(context.Context, artifact.Artifact, *agtversion.ParsedSemVer) (string, error) { d.called = true return "", errors.New("failing") } @@ -34,7 +37,7 @@ type SuccDownloader struct { called bool } -func (d *SuccDownloader) Download(ctx context.Context, _ artifact.Artifact, _ string) (string, error) { +func (d *SuccDownloader) Download(context.Context, artifact.Artifact, *agtversion.ParsedSemVer) (string, error) { d.called = true return succ, nil } @@ -61,9 +64,11 @@ func TestComposed(t *testing.T) { }, } + parseVersion, err := agtversion.ParseVersion("1.2.3") + require.NoError(t, err) for _, tc := range testCases { d := NewDownloader(tc.downloaders[0], tc.downloaders[1]) - r, _ := d.Download(context.TODO(), artifact.Artifact{Name: "a"}, "b") + r, _ := d.Download(context.TODO(), artifact.Artifact{Name: "a"}, parseVersion) assert.Equal(t, tc.expectedResult, r == succ) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier.go b/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier.go index c0b9cce26e9..bfb305c8cf0 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) // Verifier is a verifier with a predefined set of verifiers. @@ -38,7 +39,7 @@ func NewVerifier(log *logger.Logger, verifiers ...download.Verifier) *Verifier { } // Verify checks the package from configured source. -func (v *Verifier) Verify(a artifact.Artifact, version string, skipDefaultPgp bool, pgpBytes ...string) error { +func (v *Verifier) Verify(a artifact.Artifact, version agtversion.ParsedSemVer, skipDefaultPgp bool, pgpBytes ...string) error { var err error for _, verifier := range v.vv { diff --git a/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier_test.go b/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier_test.go index 088c1a29b6d..d71129db785 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/composed/verifier_test.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" "github.com/stretchr/testify/assert" ) @@ -23,7 +24,7 @@ func (d *ErrorVerifier) Name() string { return "error" } -func (d *ErrorVerifier) Verify(a artifact.Artifact, version string, _ bool, _ ...string) error { +func (d *ErrorVerifier) Verify(artifact.Artifact, agtversion.ParsedSemVer, bool, ...string) error { d.called = true return errors.New("failing") } @@ -38,7 +39,7 @@ func (d *FailVerifier) Name() string { return "fail" } -func (d *FailVerifier) Verify(a artifact.Artifact, version string, _ bool, _ ...string) error { +func (d *FailVerifier) Verify(artifact.Artifact, agtversion.ParsedSemVer, bool, ...string) error { d.called = true return &download.InvalidSignatureError{File: "", Err: errors.New("invalid signature")} } @@ -53,7 +54,7 @@ func (d *SuccVerifier) Name() string { return "succ" } -func (d *SuccVerifier) Verify(a artifact.Artifact, version string, _ bool, _ ...string) error { +func (d *SuccVerifier) Verify(artifact.Artifact, agtversion.ParsedSemVer, bool, ...string) error { d.called = true return nil } @@ -86,9 +87,10 @@ func TestVerifier(t *testing.T) { }, } + testVersion := agtversion.NewParsedSemVer(1, 2, 3, "", "") for _, tc := range testCases { d := NewVerifier(log, tc.verifiers[0], tc.verifiers[1], tc.verifiers[2]) - err := d.Verify(artifact.Artifact{Name: "a", Cmd: "a", Artifact: "a/a"}, "b", false) + err := d.Verify(artifact.Artifact{Name: "a", Cmd: "a", Artifact: "a/a"}, *testVersion, false) assert.Equal(t, tc.expectedResult, err == nil) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/downloader.go b/internal/pkg/agent/application/upgrade/artifact/download/downloader.go index 19e102ab3c9..db32b7bfe97 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/downloader.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/downloader.go @@ -8,9 +8,10 @@ import ( "context" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" + "github.com/elastic/elastic-agent/pkg/version" ) // Downloader is an interface allowing download of an artifact type Downloader interface { - Download(ctx context.Context, a artifact.Artifact, version string) (string, error) + Download(ctx context.Context, a artifact.Artifact, version *version.ParsedSemVer) (string, error) } diff --git a/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader.go b/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader.go index 6de72f0143e..a95f04ba4c3 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) const ( @@ -38,7 +39,7 @@ func NewDownloader(config *artifact.Config) *Downloader { // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version string) (_ string, err error) { +func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version *agtversion.ParsedSemVer) (_ string, err error) { span, ctx := apm.StartSpan(ctx, "download", "app.internal") defer span.End() downloadedFiles := make([]string, 0, 2) @@ -52,20 +53,20 @@ func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version }() // download from source to dest - path, err := e.download(e.config.OS(), a, version, "") + path, err := e.download(e.config.OS(), a, *version, "") downloadedFiles = append(downloadedFiles, path) if err != nil { return "", err } - hashPath, err := e.download(e.config.OS(), a, version, ".sha512") + hashPath, err := e.download(e.config.OS(), a, *version, ".sha512") downloadedFiles = append(downloadedFiles, hashPath) return path, err } // DownloadAsc downloads the package .asc file from configured source. // It returns absolute path to the downloaded file and a no-nil error if any occurs. -func (e *Downloader) DownloadAsc(_ context.Context, a artifact.Artifact, version string) (string, error) { +func (e *Downloader) DownloadAsc(_ context.Context, a artifact.Artifact, version agtversion.ParsedSemVer) (string, error) { path, err := e.download(e.config.OS(), a, version, ".asc") if err != nil { os.Remove(path) @@ -78,7 +79,7 @@ func (e *Downloader) DownloadAsc(_ context.Context, a artifact.Artifact, version func (e *Downloader) download( operatingSystem string, a artifact.Artifact, - version, + version agtversion.ParsedSemVer, extension string) (string, error) { filename, err := artifact.GetArtifactName(a, version, operatingSystem, e.config.Arch()) if err != nil { diff --git a/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader_test.go b/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader_test.go new file mode 100644 index 00000000000..0010ad33d5a --- /dev/null +++ b/internal/pkg/agent/application/upgrade/artifact/download/fs/downloader_test.go @@ -0,0 +1,293 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package fs + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" + agtversion "github.com/elastic/elastic-agent/pkg/version" +) + +type file struct { + Name string + Body []byte +} + +func TestDownloader_Download(t *testing.T) { + type fields struct { + config *artifact.Config + } + type args struct { + a artifact.Artifact + version *agtversion.ParsedSemVer + } + tests := []struct { + name string + files []file + fields fields + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path released version", + files: []file{ + { + "elastic-agent-1.2.3-linux-x86_64.tar.gz", + []byte("This is a fake linux elastic agent archive"), + }, + { + "elastic-agent-1.2.3-linux-x86_64.tar.gz.sha512", + []byte("somesha512 elastic-agent-1.2.3-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "", "")}, + want: "elastic-agent-1.2.3-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "no hash released version", + files: []file{ + { + "elastic-agent-1.2.3-linux-x86_64.tar.gz", + []byte("This is a fake linux elastic agent archive"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "", "")}, + want: "elastic-agent-1.2.3-linux-x86_64.tar.gz", + wantErr: assert.Error, + }, + { + name: "happy path snapshot version", + files: []file{ + { + "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz", + []byte("This is a fake linux elastic agent archive"), + }, + { + "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512", + []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "")}, + want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "happy path released version with build metadata", + files: []file{ + { + "elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz", + []byte("This is a fake linux elastic agent archive"), + }, + { + "elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz.sha512", + []byte("somesha512 elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "", "build19700101")}, + want: "elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "happy path snapshot version with build metadata", + files: []file{ + { + "elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz", + []byte("This is a fake linux elastic agent archive"), + }, + { + "elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz.sha512", + []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "build19700101")}, + want: "elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + dropPath := t.TempDir() + targetDirPath := t.TempDir() + + createFiles(t, dropPath, tt.files) + + config := tt.fields.config + config.DropPath = dropPath + config.TargetDirectory = targetDirPath + + e := &Downloader{ + dropPath: dropPath, + config: config, + } + got, err := e.Download(context.TODO(), tt.args.a, tt.args.version) + if !tt.wantErr(t, err, fmt.Sprintf("Download(%v, %v)", tt.args.a, tt.args.version)) { + return + } + assert.Equalf(t, filepath.Join(targetDirPath, tt.want), got, "Download(%v, %v)", tt.args.a, tt.args.version) + }) + } +} + +func createFiles(t *testing.T, dstPath string, files []file) { + for _, f := range files { + dstFile := filepath.Join(dstPath, f.Name) + err := os.WriteFile(dstFile, f.Body, 0o666) + require.NoErrorf(t, err, "error preparing file %s: %v", dstFile, err) + } +} + +func TestDownloader_DownloadAsc(t *testing.T) { + type fields struct { + config *artifact.Config + } + type args struct { + a artifact.Artifact + version agtversion.ParsedSemVer + } + tests := []struct { + name string + files []file + fields fields + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path released version", + files: []file{ + { + "elastic-agent-1.2.3-linux-x86_64.tar.gz.asc", + []byte("fake signature for elastic-agent package"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: *agtversion.NewParsedSemVer(1, 2, 3, "", "")}, + want: "elastic-agent-1.2.3-linux-x86_64.tar.gz.asc", + wantErr: assert.NoError, + }, + { + name: "happy path snapshot version", + files: []file{ + { + "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.asc", + []byte("fake signature for elastic-agent package"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: *agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "")}, + want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.asc", + wantErr: assert.NoError, + }, + { + name: "happy path released version with build metadata", + files: []file{ + { + "elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz.asc", + []byte("fake signature for elastic-agent package"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: *agtversion.NewParsedSemVer(1, 2, 3, "", "build19700101")}, + want: "elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz.asc", + wantErr: assert.NoError, + }, + { + name: "happy path snapshot version with build metadata", + files: []file{ + { + "elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz.asc", + []byte("fake signature for elastic-agent package"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: *agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "build19700101")}, + want: "elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz.asc", + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dropPath := t.TempDir() + targetDirPath := t.TempDir() + + createFiles(t, dropPath, tt.files) + + config := tt.fields.config + config.DropPath = dropPath + config.TargetDirectory = targetDirPath + + e := &Downloader{ + dropPath: dropPath, + config: config, + } + got, err := e.DownloadAsc(context.TODO(), tt.args.a, tt.args.version) + if !tt.wantErr(t, err, fmt.Sprintf("DownloadAsc(%v, %v)", tt.args.a, tt.args.version)) { + return + } + assert.Equalf(t, filepath.Join(targetDirPath, tt.want), got, "DownloadAsc(%v, %v)", tt.args.a, tt.args.version) + }) + } +} diff --git a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier.go b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier.go index 8c7861e1c75..6576143198f 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) const ( @@ -64,7 +65,7 @@ func NewVerifier(log *logger.Logger, config *artifact.Config, pgp []byte) (*Veri // Verify checks downloaded package on preconfigured // location against a key stored on elastic.co website. -func (v *Verifier) Verify(a artifact.Artifact, version string, skipDefaultPgp bool, pgpBytes ...string) error { +func (v *Verifier) Verify(a artifact.Artifact, version agtversion.ParsedSemVer, skipDefaultPgp bool, pgpBytes ...string) error { filename, err := artifact.GetArtifactName(a, version, v.config.OS(), v.config.Arch()) if err != nil { return fmt.Errorf("could not get artifact name: %w", err) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go index 4bd605142f3..280a4c374b3 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/fs/verifier_test.go @@ -22,15 +22,14 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/internal/pkg/release" "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" "github.com/elastic/elastic-agent/testing/pgptest" ) -const ( - version = "7.5.1" -) +var testVersion = agtversion.NewParsedSemVer(7, 5, 1, "", "") var ( - beatSpec = artifact.Artifact{ + agentSpec = artifact.Artifact{ Name: "Elastic Agent", Cmd: "elastic-agent", Artifact: "beat/elastic-agent"} @@ -48,7 +47,7 @@ func TestFetchVerify(t *testing.T) { ctx := context.Background() a := artifact.Artifact{ Name: "elastic-agent", Cmd: "elastic-agent", Artifact: "beats/elastic-agent"} - version := "8.0.0" + version := agtversion.NewParsedSemVer(8, 0, 0, "", "") filename := "elastic-agent-8.0.0-darwin-x86_64.tar.gz" targetFilePath := filepath.Join(targetPath, filename) @@ -80,7 +79,7 @@ func TestFetchVerify(t *testing.T) { // first download verify should fail: // download skipped, as invalid package is prepared upfront // verify fails and cleans download - err = verifier.Verify(a, version, false) + err = verifier.Verify(a, *version, false) var checksumErr *download.ChecksumMismatchError require.ErrorAs(t, err, &checksumErr) @@ -109,7 +108,7 @@ func TestFetchVerify(t *testing.T) { _, err = os.Stat(ascTargetFilePath) require.NoError(t, err) - err = verifier.Verify(a, version, false) + err = verifier.Verify(a, *version, false) require.NoError(t, err) // Bad GPG public key. @@ -126,7 +125,7 @@ func TestFetchVerify(t *testing.T) { // Missing .asc file. { - err = verifier.Verify(a, version, false) + err = verifier.Verify(a, *version, false) require.Error(t, err) // Don't delete these files when GPG validation failure. @@ -139,7 +138,7 @@ func TestFetchVerify(t *testing.T) { err = os.WriteFile(targetFilePath+".asc", []byte("bad sig"), 0o600) require.NoError(t, err) - err = verifier.Verify(a, version, false) + err = verifier.Verify(a, *version, false) var invalidSigErr *download.InvalidSignatureError assert.ErrorAs(t, err, &invalidSigErr) @@ -217,12 +216,12 @@ func TestVerify(t *testing.T) { }, } - pgpKey := prepareTestCase(t, beatSpec, version, config) + pgpKey := prepareTestCase(t, agentSpec, testVersion, config) testClient := NewDownloader(config) - artifactPath, err := testClient.Download(context.Background(), beatSpec, version) + artifactPath, err := testClient.Download(context.Background(), agentSpec, testVersion) require.NoError(t, err, "fs.Downloader could not download artifacts") - _, err = testClient.DownloadAsc(context.Background(), beatSpec, version) + _, err = testClient.DownloadAsc(context.Background(), agentSpec, *testVersion) require.NoError(t, err, "fs.Downloader could not download artifacts .asc file") _, err = os.Stat(artifactPath) @@ -231,7 +230,7 @@ func TestVerify(t *testing.T) { testVerifier, err := NewVerifier(log, config, pgpKey) require.NoError(t, err) - err = testVerifier.Verify(beatSpec, version, false, tc.RemotePGPUris...) + err = testVerifier.Verify(agentSpec, *testVersion, false, tc.RemotePGPUris...) require.NoError(t, err) // log message informing remote PGP was skipped @@ -245,13 +244,9 @@ func TestVerify(t *testing.T) { // its corresponding checksum (.sha512) and signature (.asc) files. // It creates the necessary key to sing the artifact and returns the public key // to verify the signature. -func prepareTestCase( - t *testing.T, - a artifact.Artifact, - version string, - cfg *artifact.Config) []byte { +func prepareTestCase(t *testing.T, a artifact.Artifact, version *agtversion.ParsedSemVer, cfg *artifact.Config) []byte { - filename, err := artifact.GetArtifactName(a, version, cfg.OperatingSystem, cfg.Architecture) + filename, err := artifact.GetArtifactName(a, *version, cfg.OperatingSystem, cfg.Architecture) require.NoErrorf(t, err, "could not get artifact name") err = os.MkdirAll(cfg.DropPath, 0777) diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go b/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go index cfc899420c2..9094723eedb 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go @@ -19,16 +19,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" + agtversion "github.com/elastic/elastic-agent/pkg/version" "github.com/elastic/elastic-agent/testing/pgptest" ) const ( - version = "7.5.1" sourcePattern = "/downloads/beats/filebeat/" source = "http://artifacts.elastic.co/downloads/" ) var ( + version = agtversion.NewParsedSemVer(7, 5, 1, "", "") beatSpec = artifact.Artifact{ Name: "filebeat", Cmd: "filebeat", diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/downloader.go b/internal/pkg/agent/application/upgrade/artifact/download/http/downloader.go index 50fc6849f21..ffd28a3fc16 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/downloader.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/downloader.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) const ( @@ -93,7 +94,7 @@ func (e *Downloader) Reload(c *artifact.Config) error { // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version string) (_ string, err error) { +func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version *agtversion.ParsedSemVer) (_ string, err error) { remoteArtifact := a.Artifact downloadedFiles := make([]string, 0, 2) defer func() { @@ -107,13 +108,13 @@ func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version }() // download from source to dest - path, err := e.download(ctx, remoteArtifact, e.config.OS(), a, version) + path, err := e.download(ctx, remoteArtifact, e.config.OS(), a, *version) downloadedFiles = append(downloadedFiles, path) if err != nil { return "", err } - hashPath, err := e.downloadHash(ctx, remoteArtifact, e.config.OS(), a, version) + hashPath, err := e.downloadHash(ctx, remoteArtifact, e.config.OS(), a, *version) downloadedFiles = append(downloadedFiles, hashPath) return path, err } @@ -135,7 +136,7 @@ func (e *Downloader) composeURI(artifactName, packageName string) (string, error return uri.String(), nil } -func (e *Downloader) download(ctx context.Context, remoteArtifact string, operatingSystem string, a artifact.Artifact, version string) (string, error) { +func (e *Downloader) download(ctx context.Context, remoteArtifact string, operatingSystem string, a artifact.Artifact, version agtversion.ParsedSemVer) (string, error) { filename, err := artifact.GetArtifactName(a, version, operatingSystem, e.config.Arch()) if err != nil { return "", errors.New(err, "generating package name failed") @@ -149,7 +150,7 @@ func (e *Downloader) download(ctx context.Context, remoteArtifact string, operat return e.downloadFile(ctx, remoteArtifact, filename, fullPath) } -func (e *Downloader) downloadHash(ctx context.Context, remoteArtifact string, operatingSystem string, a artifact.Artifact, version string) (string, error) { +func (e *Downloader) downloadHash(ctx context.Context, remoteArtifact string, operatingSystem string, a artifact.Artifact, version agtversion.ParsedSemVer) (string, error) { filename, err := artifact.GetArtifactName(a, version, operatingSystem, e.config.Arch()) if err != nil { return "", errors.New(err, "generating package name failed") diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/downloader_test.go b/internal/pkg/agent/application/upgrade/artifact/download/http/downloader_test.go index 94e3ce856e2..d8c6e2a9304 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/downloader_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/downloader_test.go @@ -5,13 +5,16 @@ package http import ( + "bytes" "context" "fmt" + "io" "io/ioutil" "net" "net/http" "net/http/httptest" "os" + "path/filepath" "regexp" "strconv" "testing" @@ -23,6 +26,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" "github.com/docker/go-units" "github.com/stretchr/testify/assert" @@ -341,3 +345,187 @@ func printLogs(t *testing.T, logs []observer.LoggedEntry) { t.Logf("[%s] %s", entry.Level, entry.Message) } } + +var agentSpec = artifact.Artifact{ + Name: "Elastic Agent", + Cmd: "elastic-agent", + Artifact: "beat/elastic-agent", +} + +type downloadHttpResponse struct { + statusCode int + headers http.Header + Body []byte +} + +func TestDownloadVersion(t *testing.T) { + + type fields struct { + config *artifact.Config + } + type args struct { + a artifact.Artifact + version *agtversion.ParsedSemVer + } + tests := []struct { + name string + files map[string]downloadHttpResponse + fields fields + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path released version", + files: map[string]downloadHttpResponse{ + "/beat/elastic-agent/elastic-agent-1.2.3-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + "/beat/elastic-agent/elastic-agent-1.2.3-linux-x86_64.tar.gz.sha512": { + statusCode: http.StatusOK, + Body: []byte("somesha512 elastic-agent-1.2.3-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "", "")}, + want: "elastic-agent-1.2.3-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "no hash released version", + files: map[string]downloadHttpResponse{ + "/beat/elastic-agent/elastic-agent-1.2.3-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "", "")}, + want: "elastic-agent-1.2.3-linux-x86_64.tar.gz", + wantErr: assert.Error, + }, + { + name: "happy path snapshot version", + files: map[string]downloadHttpResponse{ + "/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + "/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": { + statusCode: http.StatusOK, + Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "")}, + want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "happy path released version with build metadata", + files: map[string]downloadHttpResponse{ + "/beat/elastic-agent/elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + "/beat/elastic-agent/elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz.sha512": { + statusCode: http.StatusOK, + Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "", "build19700101")}, + want: "elastic-agent-1.2.3+build19700101-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "happy path snapshot version with build metadata", + files: map[string]downloadHttpResponse{ + "/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + "/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz.sha512": { + statusCode: http.StatusOK, + Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "build19700101")}, + want: "elastic-agent-1.2.3-SNAPSHOT+build19700101-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + targetDirPath := t.TempDir() + + handleDownload := func(rw http.ResponseWriter, req *http.Request) { + path := req.URL.Path + + resp, ok := tt.files[path] + if !ok { + rw.WriteHeader(http.StatusNotFound) + return + } + + for k, values := range resp.headers { + for _, v := range values { + rw.Header().Set(k, v) + } + } + + rw.WriteHeader(resp.statusCode) + _, err := io.Copy(rw, bytes.NewReader(resp.Body)) + assert.NoError(t, err, "error writing response content") + } + server := httptest.NewServer(http.HandlerFunc(handleDownload)) + defer server.Close() + + elasticClient := server.Client() + log, _ := logger.NewTesting("downloader") + upgradeDetails := details.NewDetails(tt.args.version.String(), details.StateRequested, "") + config := tt.fields.config + config.SourceURI = server.URL + config.TargetDirectory = targetDirPath + downloader := NewDownloaderWithClient(log, config, *elasticClient, upgradeDetails) + + got, err := downloader.Download(context.TODO(), tt.args.a, tt.args.version) + + if !tt.wantErr(t, err, fmt.Sprintf("Download(%v, %v)", tt.args.a, tt.args.version)) { + return + } + + assert.Equalf(t, filepath.Join(targetDirPath, tt.want), got, "Download(%v, %v)", tt.args.a, tt.args.version) + }) + } + +} diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/verifier.go b/internal/pkg/agent/application/upgrade/artifact/download/http/verifier.go index 50aa64fab1e..5197f931285 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/verifier.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/verifier.go @@ -16,6 +16,7 @@ import ( "time" "github.com/elastic/elastic-agent-libs/transport/httpcommon" + agtversion "github.com/elastic/elastic-agent/pkg/version" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" @@ -87,7 +88,7 @@ func (v *Verifier) Reload(c *artifact.Config) error { // Verify checks downloaded package on preconfigured // location against a key stored on elastic.co website. -func (v *Verifier) Verify(a artifact.Artifact, version string, skipDefaultPgp bool, pgpBytes ...string) error { +func (v *Verifier) Verify(a artifact.Artifact, version agtversion.ParsedSemVer, skipDefaultPgp bool, pgpBytes ...string) error { artifactPath, err := artifact.GetArtifactPath(a, version, v.config.OS(), v.config.Arch(), v.config.TargetDirectory) if err != nil { return errors.New(err, "retrieving package path") @@ -115,7 +116,7 @@ func (v *Verifier) Verify(a artifact.Artifact, version string, skipDefaultPgp bo return nil } -func (v *Verifier) verifyAsc(a artifact.Artifact, version string, skipDefaultKey bool, pgpSources ...string) error { +func (v *Verifier) verifyAsc(a artifact.Artifact, version agtversion.ParsedSemVer, skipDefaultKey bool, pgpSources ...string) error { filename, err := artifact.GetArtifactName(a, version, v.config.OS(), v.config.Arch()) if err != nil { return errors.New(err, "retrieving package name") diff --git a/internal/pkg/agent/application/upgrade/artifact/download/http/verifier_test.go b/internal/pkg/agent/application/upgrade/artifact/download/http/verifier_test.go index 66c8bd715e0..3d5c74a9a8a 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/http/verifier_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/http/verifier_test.go @@ -64,7 +64,7 @@ func TestVerify(t *testing.T) { t.Fatal(err) } - err = testVerifier.Verify(beatSpec, version, false) + err = testVerifier.Verify(beatSpec, *version, false) require.NoError(t, err) }) } diff --git a/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go b/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go index ecf2497851c..5c417531304 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go @@ -8,7 +8,9 @@ import ( "context" "encoding/json" "fmt" + gohttp "net/http" "strings" + "time" "github.com/elastic/elastic-agent-libs/transport/httpcommon" @@ -26,6 +28,7 @@ const snapshotURIFormat = "https://snapshots.elastic.co/%s-%s/downloads/" type Downloader struct { downloader download.Downloader versionOverride *agtversion.ParsedSemVer + client *gohttp.Client } // NewDownloader creates a downloader which first checks local directory @@ -34,19 +37,30 @@ type Downloader struct { // artifact.Config struct is part of agent configuration and a version // override makes no sense there func NewDownloader(log *logger.Logger, config *artifact.Config, versionOverride *agtversion.ParsedSemVer, upgradeDetails *details.Details) (download.Downloader, error) { - cfg, err := snapshotConfig(config, versionOverride) + client, err := config.HTTPTransportSettings.Client( + httpcommon.WithAPMHTTPInstrumentation(), + httpcommon.WithKeepaliveSettings{Disable: false, IdleConnTimeout: 30 * time.Second}, + ) if err != nil { - return nil, fmt.Errorf("error creating snapshot config: %w", err) + return nil, err } - httpDownloader, err := http.NewDownloader(log, cfg, upgradeDetails) + return NewDownloaderWithClient(log, config, versionOverride, client, upgradeDetails) +} + +func NewDownloaderWithClient(log *logger.Logger, config *artifact.Config, versionOverride *agtversion.ParsedSemVer, client *gohttp.Client, upgradeDetails *details.Details) (download.Downloader, error) { + // TODO: decide an appropriate timeout for this + cfg, err := snapshotConfig(context.TODO(), client, config, versionOverride) if err != nil { - return nil, fmt.Errorf("failed to create snapshot downloader: %w", err) + return nil, fmt.Errorf("error creating snapshot config: %w", err) } + httpDownloader := http.NewDownloaderWithClient(log, cfg, *client, upgradeDetails) + return &Downloader{ downloader: httpDownloader, versionOverride: versionOverride, + client: client, }, nil } @@ -56,7 +70,8 @@ func (e *Downloader) Reload(c *artifact.Config) error { return nil } - cfg, err := snapshotConfig(c, e.versionOverride) + // TODO: decide an appropriate timeout for this + cfg, err := snapshotConfig(context.TODO(), e.client, c, e.versionOverride) if err != nil { return fmt.Errorf("snapshot.downloader: failed to generate snapshot config: %w", err) } @@ -66,12 +81,14 @@ func (e *Downloader) Reload(c *artifact.Config) error { // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version string) (string, error) { - return e.downloader.Download(ctx, a, version) +func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version *agtversion.ParsedSemVer) (string, error) { + // remove build metadata to match filename of the package for the specific snapshot build + strippedVersion := agtversion.NewParsedSemVer(version.Major(), version.Minor(), version.Patch(), version.Prerelease(), "") + return e.downloader.Download(ctx, a, strippedVersion) } -func snapshotConfig(config *artifact.Config, versionOverride *agtversion.ParsedSemVer) (*artifact.Config, error) { - snapshotURI, err := snapshotURI(versionOverride, config) +func snapshotConfig(ctx context.Context, client *gohttp.Client, config *artifact.Config, versionOverride *agtversion.ParsedSemVer) (*artifact.Config, error) { + snapshotURI, err := snapshotURI(ctx, client, versionOverride, config) if err != nil { return nil, fmt.Errorf("failed to detect remote snapshot repo, proceeding with configured: %w", err) } @@ -88,7 +105,7 @@ func snapshotConfig(config *artifact.Config, versionOverride *agtversion.ParsedS }, nil } -func snapshotURI(versionOverride *agtversion.ParsedSemVer, config *artifact.Config) (string, error) { +func snapshotURI(ctx context.Context, client *gohttp.Client, versionOverride *agtversion.ParsedSemVer, config *artifact.Config) (string, error) { // Respect a non-default source URI even if the version is a snapshot. if config.SourceURI != artifact.DefaultSourceURI { return config.SourceURI, nil @@ -107,14 +124,13 @@ func snapshotURI(versionOverride *agtversion.ParsedSemVer, config *artifact.Conf version = versionOverride.CoreVersion() } - // we go through the artifact API to find the location of the latest snapshot build for the specified version - client, err := config.HTTPTransportSettings.Client(httpcommon.WithAPMHTTPInstrumentation()) + artifactsURI := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", version) + request, err := gohttp.NewRequestWithContext(ctx, gohttp.MethodGet, artifactsURI, nil) if err != nil { - return "", err + return "", fmt.Errorf("creating request to artifact api: %w", err) } - artifactsURI := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", version) - resp, err := client.Get(artifactsURI) + resp, err := client.Do(request) if err != nil { return "", err } diff --git a/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go b/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go index 18ed58b0d65..d7dc8d433e1 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go @@ -5,23 +5,241 @@ package snapshot import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" - "github.com/elastic/elastic-agent/pkg/version" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" + "github.com/elastic/elastic-agent/pkg/core/logger" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) func TestNonDefaultSourceURI(t *testing.T) { - version, err := version.ParseVersion("8.12.0-SNAPSHOT") + version, err := agtversion.ParseVersion("8.12.0-SNAPSHOT") require.NoError(t, err) config := artifact.Config{ SourceURI: "localhost:1234", } - sourceURI, err := snapshotURI(version, &config) + sourceURI, err := snapshotURI(context.TODO(), http.DefaultClient, version, &config) require.NoError(t, err) require.Equal(t, config.SourceURI, sourceURI) } + +const artifactAPIElasticAgentSearchResponse = ` +{ + "packages": { + "elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz": { + "url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz", + "sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz.sha512", + "asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz.asc", + "type": "tar", + "architecture": "aarch64", + "os": [ + "darwin" + ] + }, + "elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip": { + "url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip", + "sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip.sha512", + "asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip.asc", + "type": "zip", + "architecture": "x86_64", + "os": [ + "windows" + ] + }, + "elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz": { + "url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz", + "sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz.sha512", + "asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz.asc", + "type": "tar", + "architecture": "arm64", + "os": [ + "linux" + ] + }, + "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": { + "url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz", + "sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512", + "asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.asc", + "type": "tar", + "architecture": "x86_64", + "os": [ + "linux" + ] + }, + "elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz": { + "url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz", + "sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz.sha512", + "asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz.asc", + "type": "tar", + "architecture": "arm64", + "os": [ + "linux" + ] + }, + "elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz": { + "url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz", + "sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz.sha512", + "asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz.asc", + "type": "tar", + "architecture": "x86_64", + "os": [ + "darwin" + ] + } + }, + "manifests": { + "last-update-time": "Tue, 05 Dec 2023 15:47:06 UTC", + "seconds-since-last-update": 201 + } +} +` + +var agentSpec = artifact.Artifact{ + Name: "Elastic Agent", + Cmd: "elastic-agent", + Artifact: "beat/elastic-agent", +} + +type downloadHttpResponse struct { + statusCode int + headers http.Header + Body []byte +} + +func TestDownloadVersion(t *testing.T) { + + type fields struct { + config *artifact.Config + } + type args struct { + a artifact.Artifact + version *agtversion.ParsedSemVer + } + tests := []struct { + name string + files map[string]downloadHttpResponse + fields fields + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path snapshot version", + files: map[string]downloadHttpResponse{ + "/1.2.3-33e8d7e1/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + "/1.2.3-33e8d7e1/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": { + statusCode: http.StatusOK, + Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"), + }, + "/v1/search/1.2.3-SNAPSHOT/elastic-agent": { + statusCode: http.StatusOK, + headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: []byte(artifactAPIElasticAgentSearchResponse), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "")}, + want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + { + name: "happy path snapshot version with build metadata", + files: map[string]downloadHttpResponse{ + "/1.2.3-buildid/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": { + statusCode: http.StatusOK, + Body: []byte("This is a fake linux elastic agent archive"), + }, + "/1.2.3-buildid/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": { + statusCode: http.StatusOK, + Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"), + }, + }, + fields: fields{ + config: &artifact.Config{ + OperatingSystem: "linux", + Architecture: "64", + }, + }, + args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "buildid")}, + want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz", + wantErr: assert.NoError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + targetDirPath := t.TempDir() + + handleDownload := func(rw http.ResponseWriter, req *http.Request) { + path := req.URL.Path + + resp, ok := tt.files[path] + if !ok { + rw.WriteHeader(http.StatusNotFound) + return + } + + for k, values := range resp.headers { + for _, v := range values { + rw.Header().Set(k, v) + } + } + + rw.WriteHeader(resp.statusCode) + _, err := io.Copy(rw, bytes.NewReader(resp.Body)) + assert.NoError(t, err, "error writing out response body") + } + server := httptest.NewTLSServer(http.HandlerFunc(handleDownload)) + defer server.Close() + + log, _ := logger.NewTesting("downloader") + upgradeDetails := details.NewDetails(tt.args.version.String(), details.StateRequested, "") + + config := tt.fields.config + config.TargetDirectory = targetDirPath + config.SourceURI = "https://artifacts.elastic.co/downloads/" + + client := server.Client() + transport := client.Transport.(*http.Transport) + + transport.TLSClientConfig.InsecureSkipVerify = true + transport.DialContext = func(_ context.Context, network, s string) (net.Conn, error) { + _ = s + return net.Dial(network, server.Listener.Addr().String()) + } + downloader, err := NewDownloaderWithClient(log, config, tt.args.version, client, upgradeDetails) + require.NoError(t, err) + got, err := downloader.Download(context.TODO(), tt.args.a, tt.args.version) + + if !tt.wantErr(t, err, fmt.Sprintf("Download(%v, %v)", tt.args.a, tt.args.version)) { + return + } + + assert.Equalf(t, filepath.Join(targetDirPath, tt.want), got, "Download(%v, %v)", tt.args.a, tt.args.version) + }) + } + +} diff --git a/internal/pkg/agent/application/upgrade/artifact/download/snapshot/verifier.go b/internal/pkg/agent/application/upgrade/artifact/download/snapshot/verifier.go index 060c5e9fa10..69ed5dfe7f2 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/snapshot/verifier.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/snapshot/verifier.go @@ -5,6 +5,10 @@ package snapshot import ( + "context" + gohttp "net/http" + + "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download/http" @@ -16,6 +20,7 @@ import ( type Verifier struct { verifier download.Verifier versionOverride *agtversion.ParsedSemVer + client *gohttp.Client } func (v *Verifier) Name() string { @@ -25,7 +30,14 @@ func (v *Verifier) Name() string { // NewVerifier creates a downloader which first checks local directory // and then fallbacks to remote if configured. func NewVerifier(log *logger.Logger, config *artifact.Config, pgp []byte, versionOverride *agtversion.ParsedSemVer) (download.Verifier, error) { - cfg, err := snapshotConfig(config, versionOverride) + + client, err := config.HTTPTransportSettings.Client(httpcommon.WithAPMHTTPInstrumentation()) + if err != nil { + return nil, err + } + + // TODO: decide an appropriate timeout for this + cfg, err := snapshotConfig(context.TODO(), client, config, versionOverride) if err != nil { return nil, err } @@ -37,12 +49,14 @@ func NewVerifier(log *logger.Logger, config *artifact.Config, pgp []byte, versio return &Verifier{ verifier: v, versionOverride: versionOverride, + client: client, }, nil } // Verify checks the package from configured source. -func (v *Verifier) Verify(a artifact.Artifact, version string, skipDefaultPgp bool, pgpBytes ...string) error { - return v.verifier.Verify(a, version, skipDefaultPgp, pgpBytes...) +func (v *Verifier) Verify(a artifact.Artifact, version agtversion.ParsedSemVer, skipDefaultPgp bool, pgpBytes ...string) error { + strippedVersion := agtversion.NewParsedSemVer(version.Major(), version.Minor(), version.Patch(), version.Prerelease(), "") + return v.verifier.Verify(a, *strippedVersion, skipDefaultPgp, pgpBytes...) } func (v *Verifier) Reload(c *artifact.Config) error { @@ -51,7 +65,8 @@ func (v *Verifier) Reload(c *artifact.Config) error { return nil } - cfg, err := snapshotConfig(c, v.versionOverride) + // TODO: decide an appropriate timeout for this + cfg, err := snapshotConfig(context.TODO(), v.client, c, v.versionOverride) if err != nil { return errors.New(err, "snapshot.downloader: failed to generate snapshot config") } diff --git a/internal/pkg/agent/application/upgrade/artifact/download/verifier.go b/internal/pkg/agent/application/upgrade/artifact/download/verifier.go index 79fc2348711..e466c0119ea 100644 --- a/internal/pkg/agent/application/upgrade/artifact/download/verifier.go +++ b/internal/pkg/agent/application/upgrade/artifact/download/verifier.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) const ( @@ -83,7 +84,7 @@ type Verifier interface { // If the checksum does no match Verify returns a *download.ChecksumMismatchError. // If the PGP signature check fails then Verify returns a // *download.InvalidSignatureError. - Verify(a artifact.Artifact, version string, skipDefaultPgp bool, pgpBytes ...string) error + Verify(a artifact.Artifact, version agtversion.ParsedSemVer, skipDefaultPgp bool, pgpBytes ...string) error } // VerifySHA512HashWithCleanup calls VerifySHA512Hash and, in case of a diff --git a/internal/pkg/agent/application/upgrade/step_download.go b/internal/pkg/agent/application/upgrade/step_download.go index 07785d0fe1b..579ec656f55 100644 --- a/internal/pkg/agent/application/upgrade/step_download.go +++ b/internal/pkg/agent/application/upgrade/step_download.go @@ -121,7 +121,7 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri } } - if err := verifier.Verify(agentArtifact, parsedVersion.VersionWithPrerelease(), skipDefaultPgp, pgpBytes...); err != nil { + if err := verifier.Verify(agentArtifact, *parsedVersion, skipDefaultPgp, pgpBytes...); err != nil { return "", errors.New(err, "failed verification of agent binary") } return path, nil @@ -219,7 +219,7 @@ func (u *Upgrader) downloadOnce( // All download artifacts expect a name that includes .[-SNAPSHOT] so we have to // make sure not to include build metadata we might have in the parsed version (for snapshots we already // used that to configure the URL we download the files from) - path, err := downloader.Download(ctx, agentArtifact, version.VersionWithPrerelease()) + path, err := downloader.Download(ctx, agentArtifact, version) if err != nil { return "", fmt.Errorf("unable to download package: %w", err) } diff --git a/internal/pkg/agent/application/upgrade/step_download_test.go b/internal/pkg/agent/application/upgrade/step_download_test.go index df554beccc1..af485aaca77 100644 --- a/internal/pkg/agent/application/upgrade/step_download_test.go +++ b/internal/pkg/agent/application/upgrade/step_download_test.go @@ -28,7 +28,7 @@ type mockDownloader struct { downloadErr error } -func (md *mockDownloader) Download(ctx context.Context, agentArtifact artifact.Artifact, version string) (string, error) { +func (md *mockDownloader) Download(ctx context.Context, a artifact.Artifact, version *agtversion.ParsedSemVer) (string, error) { return md.downloadPath, md.downloadErr }