Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove use of artifacts-api.elastic.co and its client #5050

Merged
merged 3 commits into from
Jul 5, 2024
Merged

Remove use of artifacts-api.elastic.co and its client #5050

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
65d3d96
Remove use of artifacts-api.elastic.co and its client
rdner Jul 3, 2024
b8336a4
Restore still used structures
rdner Jul 3, 2024
ad78b97
Fix tests
rdner Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions dev-tools/mage/checksums.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"github.com/otiai10/copy"

"github.com/elastic/elastic-agent/dev-tools/mage/manifest"
"github.com/elastic/elastic-agent/pkg/testing/tools"
)

const ComponentSpecFileSuffix = ".spec.yml"
Expand Down Expand Up @@ -95,7 +94,7 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string
}

// This is a helper function for flattenDependencies that's used when building from a manifest
func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *tools.Build) map[string]string {
func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string {
checksums := make(map[string]string)
if manifestResponse == nil {
return checksums
Expand Down Expand Up @@ -210,7 +209,7 @@ func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, ver
// for projects in an Independent Agent Release to have different versions since the opted-in
// ones will be one patch version higher than the opted-out/previously released projects.
// This function tries to find the versions from the package name
func getComponentVersion(componentName string, requiredPackage string, componentProject tools.Project) string {
func getComponentVersion(componentName string, requiredPackage string, componentProject manifest.Project) string {
var componentVersion string
var foundIt bool
// Iterate over all the packages in the component project
Expand Down
84 changes: 64 additions & 20 deletions dev-tools/mage/manifest/manifest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,54 @@ import (
"github.com/magefile/mage/mg"
"golang.org/x/sync/errgroup"

"github.com/elastic/elastic-agent/pkg/testing/tools"
"github.com/elastic/elastic-agent/pkg/version"
)

type Build struct {
Projects map[string]Project `json:"projects"`
StartTime string `json:"start_time"`
ReleaseBranch string `json:"release_branch"`
Prefix string `json:"prefix"`
EndTime string `json:"end_time"`
ManifestVersion string `json:"manifest_version"`
Version string `json:"version"`
Branch string `json:"branch"`
BuildID string `json:"build_id"`
BuildDurationSeconds int `json:"build_duration_seconds"`
}
type Project struct {
Branch string `json:"branch"`
CommitHash string `json:"commit_hash"`
CommitURL string `json:"commit_url"`
ExternalArtifactsManifestURL string `json:"external_artifacts_manifest_url"`
BuildDurationSeconds int `json:"build_duration_seconds"`
Packages map[string]Package `json:"packages"`
Dependencies []Dependency `json:"dependencies"`
}

type Package struct {
URL string `json:"url"`
ShaURL string `json:"sha_url"`
AscURL string `json:"asc_url"`
Type string `json:"type"`
Architecture string `json:"architecture"`
Os []string `json:"os"`
Classifier string `json:"classifier"`
Attributes struct {
IncludeInRepo string `json:"include_in_repo"`
ArtifactNoKpi string `json:"artifactNoKpi"`
Internal string `json:"internal"`
ArtifactID string `json:"artifact_id"`
Oss string `json:"oss"`
Group string `json:"group"`
} `json:"attributes"`
}

type Dependency struct {
Prefix string `json:"prefix"`
BuildUri string `json:"build_uri"`
}

// A backoff schedule for when and how often to retry failed HTTP
// requests. The first element is the time to wait after the
// first failure, the second the time to wait after the second
Expand Down Expand Up @@ -50,24 +94,24 @@ var PlatformPackages = map[string]string{
// ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager.
// The project names are those used in the "projects" list in the unified release manifest.
// See the sample manifests in the testdata directory.
var ExpectedBinaries = map[string]Project{
"agentbeat": Project{Name: "beats", Platforms: AllPlatforms},
"apm-server": Project{Name: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}},
"cloudbeat": Project{Name: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"cloud-defend": Project{Name: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"endpoint-security": Project{Name: "endpoint-dev", Platforms: AllPlatforms},
"fleet-server": Project{Name: "fleet-server", Platforms: AllPlatforms},
"pf-elastic-collector": Project{Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"pf-elastic-symbolizer": Project{Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"pf-host-agent": Project{Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
var ExpectedBinaries = map[string]BinarySpec{
"agentbeat": {Name: "beats", Platforms: AllPlatforms},
"apm-server": {Name: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}},
"cloudbeat": {Name: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"cloud-defend": {Name: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"endpoint-security": {Name: "endpoint-dev", Platforms: AllPlatforms},
"fleet-server": {Name: "fleet-server", Platforms: AllPlatforms},
"pf-elastic-collector": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"pf-elastic-symbolizer": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
"pf-host-agent": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
}

type Project struct {
type BinarySpec struct {
Name string
Platforms []Platform
}

func (proj Project) SupportsPlatform(platform string) bool {
func (proj BinarySpec) SupportsPlatform(platform string) bool {
for _, p := range proj.Platforms {
if p.Platform() == platform {
return true
Expand Down Expand Up @@ -95,10 +139,10 @@ func (p Platform) Platform() string {
var AllPlatforms = []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}, {"darwin", "aarch64"}}

// DownloadManifest is going to download the given manifest file and return the ManifestResponse
func DownloadManifest(ctx context.Context, manifest string) (tools.Build, error) {
func DownloadManifest(ctx context.Context, manifest string) (Build, error) {
manifestUrl, urlError := url.Parse(manifest)
if urlError != nil {
return tools.Build{}, errorInvalidManifestURL
return Build{}, errorInvalidManifestURL
}
var valid = false
for _, manifestHost := range AllowedManifestHosts {
Expand All @@ -108,13 +152,13 @@ func DownloadManifest(ctx context.Context, manifest string) (tools.Build, error)
}
if !valid {
log.Printf("Not allowed %s, valid ones are %+v", manifestUrl.Host, AllowedManifestHosts)
return tools.Build{}, errorNotAllowedManifestURL
return Build{}, errorNotAllowedManifestURL
}
sanitizedUrl := fmt.Sprintf("https://%s%s", manifestUrl.Host, manifestUrl.Path)
f := func() (tools.Build, error) { return downloadManifestData(ctx, sanitizedUrl) }
f := func() (Build, error) { return downloadManifestData(ctx, sanitizedUrl) }
manifestResponse, err := doWithRetries(f)
if err != nil {
return tools.Build{}, fmt.Errorf("downloading manifest: %w", err)
return Build{}, fmt.Errorf("downloading manifest: %w", err)
}
if mg.Verbose() {
log.Printf(">>>> Downloaded manifest %s", manifest)
Expand Down Expand Up @@ -185,8 +229,8 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string
return nil
}

func resolveManifestPackage(project tools.Project, binary string, platformPkg string, version string) ([]string, error) {
var val tools.Package
func resolveManifestPackage(project Project, binary string, platformPkg string, version string) ([]string, error) {
var val Package
var ok bool

// Try the normal/easy case first
Expand Down
6 changes: 2 additions & 4 deletions dev-tools/mage/manifest/manifest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ import (

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/elastic/elastic-agent/pkg/testing/tools"
)

var (
Expand All @@ -28,8 +26,8 @@ var (
manifest8_14_0_build202406201002 string
)

func getManifestJsonData(t *testing.T, contents string) tools.Build {
var response tools.Build
func getManifestJsonData(t *testing.T, contents string) Build {
var response Build

err := json.NewDecoder(strings.NewReader(contents)).Decode(&response)
assert.NoError(t, err)
Expand Down
6 changes: 2 additions & 4 deletions dev-tools/mage/manifest/manifestspecs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ import (
"time"

"github.com/magefile/mage/mg"

"github.com/elastic/elastic-agent/pkg/testing/tools"
)

func doWithRetries[T any](f func() (T, error)) (T, error) {
Expand Down Expand Up @@ -75,8 +73,8 @@ func downloadFile(ctx context.Context, url string, filepath string) (string, err
return outFile.Name(), nil
}

func downloadManifestData(ctx context.Context, url string) (tools.Build, error) {
var response tools.Build
func downloadManifestData(ctx context.Context, url string) (Build, error) {
var response Build
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return response, fmt.Errorf("failed to create manifest request: %w", err)
Expand Down
19 changes: 9 additions & 10 deletions magefile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ import (
"github.com/elastic/elastic-agent/pkg/testing/multipass"
"github.com/elastic/elastic-agent/pkg/testing/ogc"
"github.com/elastic/elastic-agent/pkg/testing/runner"
"github.com/elastic/elastic-agent/pkg/testing/tools"
"github.com/elastic/elastic-agent/pkg/testing/tools/git"
pv "github.com/elastic/elastic-agent/pkg/testing/tools/product_versions"
"github.com/elastic/elastic-agent/pkg/testing/tools/snapshots"
Expand Down Expand Up @@ -483,7 +482,7 @@ func Package(ctx context.Context) error {
}

var err error
var manifestResponse *tools.Build
var manifestResponse *manifest.Build
if devtools.PackagingFromManifest {
manifestResponse, _, err = downloadManifestAndSetVersion(ctx, devtools.ManifestURL)
if err != nil {
Expand Down Expand Up @@ -962,7 +961,7 @@ func runAgent(ctx context.Context, env map[string]string) error {
return sh.Run("docker", dockerCmdArgs...)
}

func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *tools.Build, agentPackaging, agentBinaryTarget mg.Fn) error {
func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *manifest.Build, agentPackaging, agentBinaryTarget mg.Fn) error {
fmt.Println("--- Package Elastic-Agent")

requiredPackages := []string{}
Expand Down Expand Up @@ -1135,7 +1134,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi

// flattenDependencies will extract all the required packages collected in archivePath and dropPath in flatPath and
// regenerate checksums
func flattenDependencies(requiredPackages []string, packageVersion, archivePath, dropPath, flatPath string, manifestResponse *tools.Build) {
func flattenDependencies(requiredPackages []string, packageVersion, archivePath, dropPath, flatPath string, manifestResponse *manifest.Build) {

for _, rp := range requiredPackages {
targetPath := filepath.Join(archivePath, rp)
Expand Down Expand Up @@ -1266,7 +1265,7 @@ func PackageUsingDRA(ctx context.Context) error {
return packageAgent(ctx, platforms, parsedVersion.VersionWithPrerelease(), manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(useDRAAgentBinaryForPackage, devtools.ManifestURL))
}

func downloadManifestAndSetVersion(ctx context.Context, url string) (*tools.Build, *version.ParsedSemVer, error) {
func downloadManifestAndSetVersion(ctx context.Context, url string) (*manifest.Build, *version.ParsedSemVer, error) {
resp, err := manifest.DownloadManifest(ctx, url)
if err != nil {
return nil, nil, fmt.Errorf("downloading manifest: %w", err)
Expand Down Expand Up @@ -1348,12 +1347,12 @@ func mapManifestPlatformToAgentPlatform(manifestPltf string) (string, bool) {
return mappedPltf, found
}

func filterPackagesByPlatform(pkgs map[string]tools.Package) map[string]tools.Package {
func filterPackagesByPlatform(pkgs map[string]manifest.Package) map[string]manifest.Package {
if mg.Verbose() {
log.Printf("unfiltered packages: %v", pkgs)
}
platforms := devtools.Platforms.Names()
filteredPackages := map[string]tools.Package{}
filteredPackages := map[string]manifest.Package{}
for pkgName, pkgDesc := range pkgs {
if mg.Verbose() {
log.Printf("checking if %s:%v should be included", pkgName, pkgDesc)
Expand All @@ -1375,7 +1374,7 @@ func filterPackagesByPlatform(pkgs map[string]tools.Package) map[string]tools.Pa
return filteredPackages
}

func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir string, projects ...string) (map[string]tools.Package, error) {
func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir string, projects ...string) (map[string]manifest.Package, error) {
build, err := manifest.DownloadManifest(ctx, manifestUrl)
if err != nil {
return nil, fmt.Errorf("downloading manifest from %q: %w", manifestUrl, err)
Expand All @@ -1390,7 +1389,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s

// sync access to the downloadedArtifacts map
mx := new(sync.Mutex)
downloadedArtifacts := map[string]tools.Package{}
downloadedArtifacts := map[string]manifest.Package{}
errGrp, errCtx := errgroup.WithContext(ctx)

for _, projectName := range projects {
Expand All @@ -1408,7 +1407,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s
log.Printf("packages to download: %v", filteredPackages)
}
for pkgName, pkgDesc := range filteredPackages {
downloadFunc := func(pkgName string, pkgDesc tools.Package) func() error {
downloadFunc := func(pkgName string, pkgDesc manifest.Package) func() error {
return func() error {
artifactDownloadPath := filepath.Join(draDownloadDir, pkgName)
err := manifest.DownloadPackage(errCtx, pkgDesc.URL, artifactDownloadPath)
Expand Down
Loading