Skip to content

Commit

Permalink
Don't use artifact API for snapshot downloads (#4693)
Browse files Browse the repository at this point in the history 
* The Elastic Agent upgrade to a snapshot is now using the more reliable
snapshot API
* The artifact fetcher in testing is now using the same snapshot API

The artifact API is now used only in a single integration test-case `TestStandaloneDowngradeToSpecificSnapshotBuild`.
  • Loading branch information
@rdner
rdner authored May 9, 2024
1 parent 13a4157 commit f25124a
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 260 deletions.
5 changes: 5 additions & 0 deletions changelog/fragments/1715158488-use-snapshot-api.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
kind: enhancement
summary: Use more stable snapshot API for upgrades to snapshot versions
component: "elastic-agent"
pr: https://github.com/elastic/elastic-agent/pull/4693
issue: https://github.com/elastic/elastic-agent/issues/4458
68 changes: 29 additions & 39 deletions internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,60 +124,50 @@ func snapshotURI(ctx context.Context, client *gohttp.Client, versionOverride *ag
version = versionOverride.CoreVersion()
}

artifactsURI := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", version)
request, err := gohttp.NewRequestWithContext(ctx, gohttp.MethodGet, artifactsURI, nil)
// otherwise, if we don't know the exact build and we're trying to find the latest snapshot build
buildID, err := findLatestSnapshot(ctx, client, version)
if err != nil {
return "", fmt.Errorf("creating request to artifact api: %w", err)
return "", fmt.Errorf("failed to find snapshot information for version %q: %w", version, err)
}

resp, err := client.Do(request)
return fmt.Sprintf(snapshotURIFormat, version, buildID), nil
}

func findLatestSnapshot(ctx context.Context, client *gohttp.Client, version string) (buildID string, err error) {
latestSnapshotURI := fmt.Sprintf("https://snapshots.elastic.co/latest/%s-SNAPSHOT.json", version)
request, err := gohttp.NewRequestWithContext(ctx, gohttp.MethodGet, latestSnapshotURI, nil)
if err != nil {
return "", err
return "", fmt.Errorf("failed to create request to the snapshot API: %w", err)
}
defer resp.Body.Close()

body := struct {
Packages map[string]interface{} `json:"packages"`
}{}

dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&body); err != nil {
resp, err := client.Do(request)
if err != nil {
return "", err
}
defer resp.Body.Close()

if len(body.Packages) == 0 {
return "", fmt.Errorf("no packages found in snapshot repo")
}
switch resp.StatusCode {
case gohttp.StatusNotFound:
return "", fmt.Errorf("snapshot for version %q not found", version)

for k, pkg := range body.Packages {
pkgMap, ok := pkg.(map[string]interface{})
if !ok {
return "", fmt.Errorf("content of '%s' is not a map", k)
case gohttp.StatusOK:
var info struct {
BuildID string `json:"build_id"`
}

uriVal, found := pkgMap["url"]
if !found {
return "", fmt.Errorf("item '%s' does not contain url", k)
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&info); err != nil {
return "", err
}

uri, ok := uriVal.(string)
if !ok {
return "", fmt.Errorf("uri is not a string")
parts := strings.Split(info.BuildID, "-")
if len(parts) != 2 {
return "", fmt.Errorf("wrong format for a build ID: %s", info.BuildID)
}

// Because we're iterating over a map from the API response,
// the order is random and some elements there do not contain the
// `/beats/elastic-agent/` substring, so we need to go through the
// whole map before returning an error.
//
// One of the elements that might be there and do not contain this
// substring is the `elastic-agent-shipper`, whose URL is something like:
// https://snapshots.elastic.co/8.7.0-d050210c/downloads/elastic-agent-shipper/elastic-agent-shipper-8.7.0-SNAPSHOT-linux-x86_64.tar.gz
index := strings.Index(uri, "/beats/elastic-agent/")
if index != -1 {
return uri[:index], nil
}
}
return parts[1], nil

return "", fmt.Errorf("uri not detected")
default:
return "", fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, latestSnapshotURI)
}
}
138 changes: 23 additions & 115 deletions internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"

Expand All @@ -37,91 +38,30 @@ func TestNonDefaultSourceURI(t *testing.T) {

}

const artifactAPIElasticAgentSearchResponse = `
{
"packages": {
"elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz": {
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz",
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz.sha512",
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz.asc",
"type": "tar",
"architecture": "aarch64",
"os": [
"darwin"
]
},
"elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip": {
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip",
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip.sha512",
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip.asc",
"type": "zip",
"architecture": "x86_64",
"os": [
"windows"
]
},
"elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz": {
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz",
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz.sha512",
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz.asc",
"type": "tar",
"architecture": "arm64",
"os": [
"linux"
]
},
"elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": {
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz",
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512",
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.asc",
"type": "tar",
"architecture": "x86_64",
"os": [
"linux"
]
},
"elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz": {
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz",
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz.sha512",
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz.asc",
"type": "tar",
"architecture": "arm64",
"os": [
"linux"
]
},
"elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz": {
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz",
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz.sha512",
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz.asc",
"type": "tar",
"architecture": "x86_64",
"os": [
"darwin"
]
}
},
"manifests": {
"last-update-time": "Tue, 05 Dec 2023 15:47:06 UTC",
"seconds-since-last-update": 201
}
}
`

var agentSpec = artifact.Artifact{
Name: "Elastic Agent",
Cmd: "elastic-agent",
Artifact: "beat/elastic-agent",
}

type downloadHttpResponse struct {
statusCode int
headers http.Header
Body []byte
func readFile(t *testing.T, name string) []byte {
bytes, err := os.ReadFile(name)
require.NoError(t, err)

return bytes
}

func TestDownloadVersion(t *testing.T) {

files := map[string][]byte{
// links for the latest snapshot
"/latest/8.14.0-SNAPSHOT.json": readFile(t, "./testdata/latest-snapshot.json"),
"/8.14.0-6d69ee76/downloads/beat/elastic-agent/elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz": {},
"/8.14.0-6d69ee76/downloads/beat/elastic-agent/elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz.sha512": {},

// links for a specific build
"/8.13.3-76ce1a63/downloads/beat/elastic-agent/elastic-agent-8.13.3-SNAPSHOT-linux-x86_64.tar.gz": {},
"/8.13.3-76ce1a63/downloads/beat/elastic-agent/elastic-agent-8.13.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": {},
}
type fields struct {
config *artifact.Config
}
Expand All @@ -131,59 +71,33 @@ func TestDownloadVersion(t *testing.T) {
}
tests := []struct {
name string
files map[string]downloadHttpResponse
fields fields
args args
want string
wantErr assert.ErrorAssertionFunc
}{
{
name: "happy path snapshot version",
files: map[string]downloadHttpResponse{
"/1.2.3-33e8d7e1/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": {
statusCode: http.StatusOK,
Body: []byte("This is a fake linux elastic agent archive"),
},
"/1.2.3-33e8d7e1/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": {
statusCode: http.StatusOK,
Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"),
},
"/v1/search/1.2.3-SNAPSHOT/elastic-agent": {
statusCode: http.StatusOK,
headers: map[string][]string{"Content-Type": {"application/json"}},
Body: []byte(artifactAPIElasticAgentSearchResponse),
},
},
fields: fields{
config: &artifact.Config{
OperatingSystem: "linux",
Architecture: "64",
},
},
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "")},
want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz",
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(8, 14, 0, "SNAPSHOT", "")},
want: "elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz",
wantErr: assert.NoError,
},
{
name: "happy path snapshot version with build metadata",
files: map[string]downloadHttpResponse{
"/1.2.3-buildid/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": {
statusCode: http.StatusOK,
Body: []byte("This is a fake linux elastic agent archive"),
},
"/1.2.3-buildid/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": {
statusCode: http.StatusOK,
Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"),
},
},
fields: fields{
config: &artifact.Config{
OperatingSystem: "linux",
Architecture: "64",
},
},
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "buildid")},
want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz",
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(8, 13, 3, "SNAPSHOT", "76ce1a63")},
want: "elastic-agent-8.13.3-SNAPSHOT-linux-x86_64.tar.gz",
wantErr: assert.NoError,
},
}
Expand All @@ -195,21 +109,15 @@ func TestDownloadVersion(t *testing.T) {

handleDownload := func(rw http.ResponseWriter, req *http.Request) {
path := req.URL.Path
t.Logf("incoming request for %s", path)

resp, ok := tt.files[path]
file, ok := files[path]
if !ok {
rw.WriteHeader(http.StatusNotFound)
return
}

for k, values := range resp.headers {
for _, v := range values {
rw.Header().Set(k, v)
}
}

rw.WriteHeader(resp.statusCode)
_, err := io.Copy(rw, bytes.NewReader(resp.Body))
_, err := io.Copy(rw, bytes.NewReader(file))
assert.NoError(t, err, "error writing out response body")
}
server := httptest.NewTLSServer(http.HandlerFunc(handleDownload))
Expand Down
6 changes: 6 additions & 0 deletions ...al/pkg/agent/application/upgrade/artifact/download/snapshot/testdata/latest-snapshot.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"version": "8.14.0-SNAPSHOT",
"build_id": "8.14.0-6d69ee76",
"manifest_url": "https://snapshots.elastic.co/8.14.0-6d69ee76/manifest-8.14.0-SNAPSHOT.json",
"summary_url": "https://snapshots.elastic.co/8.14.0-6d69ee76/summary-8.14.0-SNAPSHOT.html"
}
Loading

0 comments on commit f25124a

Please sign in to comment.