Skip to content

Commit

Permalink
Restore used manifest structures
Browse files Browse the repository at this point in the history
  • Loading branch information
@rdner
rdner committed Jul 3, 2024
1 parent 4130700 commit 899434d
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 17 deletions.
58 changes: 51 additions & 7 deletions dev-tools/mage/manifest/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,54 @@ import (
"github.com/magefile/mage/mg"
"golang.org/x/sync/errgroup"

"github.com/elastic/elastic-agent/pkg/testing/tools"
"github.com/elastic/elastic-agent/pkg/version"
)

type Build struct {
Projects map[string]Project `json:"projects"`
StartTime string `json:"start_time"`
ReleaseBranch string `json:"release_branch"`
Prefix string `json:"prefix"`
EndTime string `json:"end_time"`
ManifestVersion string `json:"manifest_version"`
Version string `json:"version"`
Branch string `json:"branch"`
BuildID string `json:"build_id"`
BuildDurationSeconds int `json:"build_duration_seconds"`
}
type Project struct {
Branch string `json:"branch"`
CommitHash string `json:"commit_hash"`
CommitURL string `json:"commit_url"`
ExternalArtifactsManifestURL string `json:"external_artifacts_manifest_url"`
BuildDurationSeconds int `json:"build_duration_seconds"`
Packages map[string]Package `json:"packages"`
Dependencies []Dependency `json:"dependencies"`
}

type Package struct {
URL string `json:"url"`
ShaURL string `json:"sha_url"`
AscURL string `json:"asc_url"`
Type string `json:"type"`
Architecture string `json:"architecture"`
Os []string `json:"os"`
Classifier string `json:"classifier"`
Attributes struct {
IncludeInRepo string `json:"include_in_repo"`
ArtifactNoKpi string `json:"artifactNoKpi"`
Internal string `json:"internal"`
ArtifactID string `json:"artifact_id"`
Oss string `json:"oss"`
Group string `json:"group"`
} `json:"attributes"`
}

type Dependency struct {
Prefix string `json:"prefix"`
BuildUri string `json:"build_uri"`
}

// A backoff schedule for when and how often to retry failed HTTP
// requests. The first element is the time to wait after the
// first failure, the second the time to wait after the second
Expand All @@ -39,10 +83,10 @@ var errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not all
var AllowedManifestHosts = []string{"snapshots.elastic.co", "staging.elastic.co"}

// DownloadManifest is going to download the given manifest file and return the ManifestResponse
func DownloadManifest(manifest string) (tools.Build, error) {
func DownloadManifest(manifest string) (Build, error) {
manifestUrl, urlError := url.Parse(manifest)
if urlError != nil {
return tools.Build{}, errorInvalidManifestURL
return Build{}, errorInvalidManifestURL
}
var valid = false
for _, manifestHost := range AllowedManifestHosts {
Expand All @@ -52,13 +96,13 @@ func DownloadManifest(manifest string) (tools.Build, error) {
}
if !valid {
log.Printf("Not allowed %s, valid ones are %+v", manifestUrl.Host, AllowedManifestHosts)
return tools.Build{}, errorNotAllowedManifestURL
return Build{}, errorNotAllowedManifestURL
}
sanitizedUrl := fmt.Sprintf("https://%s%s", manifestUrl.Host, manifestUrl.Path)
f := func() (tools.Build, error) { return downloadManifestData(sanitizedUrl) }
f := func() (Build, error) { return downloadManifestData(sanitizedUrl) }
manifestResponse, err := doWithRetries(f)
if err != nil {
return tools.Build{}, fmt.Errorf("downloading manifest: %w", err)
return Build{}, fmt.Errorf("downloading manifest: %w", err)
}
if mg.Verbose() {
log.Printf(">>>> Downloaded manifest %s", manifest)
Expand All @@ -67,7 +111,7 @@ func DownloadManifest(manifest string) (tools.Build, error) {
return manifestResponse, nil
}

func resolveManifestPackage(project tools.Project, pkg string, reqPackage string, version string) []string {
func resolveManifestPackage(project Project, pkg string, reqPackage string, version string) []string {
packageName := fmt.Sprintf("%s-%s-%s", pkg, version, reqPackage)
val, ok := project.Packages[packageName]
if !ok {
Expand Down
6 changes: 2 additions & 4 deletions dev-tools/mage/manifest/manifestspecs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ import (
"time"

"github.com/magefile/mage/mg"

"github.com/elastic/elastic-agent/pkg/testing/tools"
)

func doWithRetries[T any](f func() (T, error)) (T, error) {
Expand Down Expand Up @@ -75,8 +73,8 @@ func downloadFile(ctx context.Context, url string, filepath string) (string, err
return outFile.Name(), nil
}

func downloadManifestData(url string) (tools.Build, error) {
var response tools.Build
func downloadManifestData(url string) (Build, error) {
var response Build
resp, err := http.Get(url) //nolint // we should have already verified that this is a proper valid url
if err != nil {
return response, fmt.Errorf("failed to download manifest [%s]\n %w", url, err)
Expand Down
11 changes: 5 additions & 6 deletions magefile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ import (
"github.com/elastic/elastic-agent/pkg/testing/multipass"
"github.com/elastic/elastic-agent/pkg/testing/ogc"
"github.com/elastic/elastic-agent/pkg/testing/runner"
"github.com/elastic/elastic-agent/pkg/testing/tools"
"github.com/elastic/elastic-agent/pkg/testing/tools/git"
pv "github.com/elastic/elastic-agent/pkg/testing/tools/product_versions"
"github.com/elastic/elastic-agent/pkg/testing/tools/snapshots"
Expand Down Expand Up @@ -1407,12 +1406,12 @@ func mapManifestPlatformToAgentPlatform(manifestPltf string) (string, bool) {
return mappedPltf, found
}

func filterPackagesByPlatform(pkgs map[string]tools.Package) map[string]tools.Package {
func filterPackagesByPlatform(pkgs map[string]manifest.Package) map[string]manifest.Package {
if mg.Verbose() {
log.Printf("unfiltered packages: %v", pkgs)
}
platforms := devtools.Platforms.Names()
filteredPackages := map[string]tools.Package{}
filteredPackages := map[string]manifest.Package{}
for pkgName, pkgDesc := range pkgs {
if mg.Verbose() {
log.Printf("checking if %s:%v should be included", pkgName, pkgDesc)
Expand All @@ -1434,7 +1433,7 @@ func filterPackagesByPlatform(pkgs map[string]tools.Package) map[string]tools.Pa
return filteredPackages
}

func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir string, projects ...string) (map[string]tools.Package, error) {
func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir string, projects ...string) (map[string]manifest.Package, error) {

build, err := manifest.DownloadManifest(manifestUrl)
if err != nil {
Expand All @@ -1450,7 +1449,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s

// sync access to the downloadedArtifacts map
mx := new(sync.Mutex)
downloadedArtifacts := map[string]tools.Package{}
downloadedArtifacts := map[string]manifest.Package{}
errGrp, errCtx := errgroup.WithContext(ctx)

for _, projectName := range projects {
Expand All @@ -1468,7 +1467,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s
log.Printf("packages to download: %v", filteredPackages)
}
for pkgName, pkgDesc := range filteredPackages {
downloadFunc := func(pkgName string, pkgDesc tools.Package) func() error {
downloadFunc := func(pkgName string, pkgDesc manifest.Package) func() error {
return func() error {
artifactDownloadPath := filepath.Join(draDownloadDir, pkgName)
err := manifest.DownloadPackage(errCtx, pkgDesc.URL, artifactDownloadPath)
Expand Down

0 comments on commit 899434d

Please sign in to comment.