[Ind Agent] Update packaging to properly package from manifest if given

dev-tools/mage/manifest/manifest.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/magefile/mage/mg"
@@ -38,6 +39,17 @@ var errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not all
 
 var AllowedManifestHosts = []string{"snapshots.elastic.co", "staging.elastic.co"}
 
+var ComponentSpec = map[string][]string{
+	"apm-server":            {"apm-server"},
+	"beats":                 {"agentbeat"},
+	"cloud-defend":          {"cloud-defend"},
+	"cloudbeat":             {"cloudbeat"},
+	"elastic-agent-shipper": {"elastic-agent-shipper"},
+	"endpoint-dev":          {"endpoint-security"},
+	"fleet-server":          {"fleet-server"},
+	"prodfiler":             {"pf-elastic-collector", "pf-elastic-symbolizer", "pf-host-agent"},
+}
+
 // DownloadManifest is going to download the given manifest file and return the ManifestResponse
 func DownloadManifest(manifest string) (tools.Build, error) {
 	manifestUrl, urlError := url.Parse(manifest)
@@ -68,32 +80,80 @@ func DownloadManifest(manifest string) (tools.Build, error) {
 }
 
 func resolveManifestPackage(project tools.Project, pkg string, reqPackage string, version string) []string {
+	var val tools.Package
+	var ok bool
+
+	// Try the normal/easy case first
 	packageName := fmt.Sprintf("%s-%s-%s", pkg, version, reqPackage)
-	val, ok := project.Packages[packageName]
+	val, ok = project.Packages[packageName]
 	if !ok {
-		return nil
+		// If we didn't find it, it may be an Independent Agent Release, where
+		// the opted-in projects will have a patch version one higher than
+		// the rest of the projects, so we need to seek that out
+		if mg.Verbose() {
+			log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", pkg, reqPackage)
+		}
+
+		var foundIt bool
+		for pkgName := range project.Packages {
+			if strings.HasPrefix(pkgName, pkg) {
+				firstSplit := strings.Split(pkgName, pkg+"-")
+				if len(firstSplit) < 2 {
+					continue
+				}
+
+				secondHalf := firstSplit[1]
+				// Make sure we're finding one w/ the same required package type
+				if strings.Contains(secondHalf, reqPackage) {
+
+					// Split again after the version with the required package string
+					secondSplit := strings.Split(secondHalf, "-"+reqPackage)
+					if len(secondSplit) < 2 {
+						continue
+					}
+
+					// The first element after the split should normally be the version
+					pkgVersion := secondSplit[0]
+					if mg.Verbose() {
+						log.Printf(">>>>>>>>>>> Using derived version for package [%s]: %s ", pkgName, pkgVersion)
+					}
+
+					// Create a project/package key with the package, derived version, and required package
+					foundPkgKey := fmt.Sprintf("%s-%s-%s", pkg, pkgVersion, reqPackage)
+					if mg.Verbose() {
+						log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey)
+					}
+
+					// Get the package value, if it exists
+					val, ok = project.Packages[foundPkgKey]
+					if !ok {
+						continue
+					}
+
+					if mg.Verbose() {
+						log.Printf(">>>>>>>>>>> Found package key [%s]", foundPkgKey)
+					}
+
+					foundIt = true
+				}
+			}
+		}
+
+		if !foundIt {
+			return nil
+		}
 	}
+
 	if mg.Verbose() {
 		log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash)
 	}
-	return []string{val.URL, val.ShaURL, val.AscURL}
 
+	return []string{val.URL, val.ShaURL, val.AscURL}
 }
 
 // DownloadComponentsFromManifest is going to download a set of components from the given manifest into the destination
 // dropPath folder in order to later use that folder for packaging
 func DownloadComponentsFromManifest(manifest string, platforms []string, platformPackages map[string]string, dropPath string) error {
-	componentSpec := map[string][]string{
-		"apm-server":            {"apm-server"},
-		"beats":                 {"agentbeat"},
-		"cloud-defend":          {"cloud-defend"},
-		"cloudbeat":             {"cloudbeat"},
-		"elastic-agent-shipper": {"elastic-agent-shipper"},
-		"endpoint-dev":          {"endpoint-security"},
-		"fleet-server":          {"fleet-server"},
-		"prodfiler":             {"pf-elastic-collector", "pf-elastic-symbolizer", "pf-host-agent"},
-	}
-
 	manifestResponse, err := DownloadManifest(manifest)
 	if err != nil {
 		return fmt.Errorf("failed to download remote manifest file %w", err)
@@ -111,7 +171,7 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 	majorMinorPatchVersion := parsedManifestVersion.VersionWithPrerelease()
 
 	errGrp, downloadsCtx := errgroup.WithContext(context.Background())
-	for component, pkgs := range componentSpec {
+	for component, pkgs := range ComponentSpec {
 		for _, platform := range platforms {
 			targetPath := filepath.Join(dropPath)
 			err := os.MkdirAll(targetPath, 0755)

dev-tools/mage/manifest/manifest_test.go

@@ -0,0 +1,138 @@
+package manifest
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+
+	"github.com/elastic/elastic-agent/pkg/testing/tools"
+	"github.com/stretchr/testify/assert"
+)
+
+// Helper to get the absolute path of the parent directory name of the current file
+func ParentDir() string {
+	_, filename, _, _ := runtime.Caller(1)
+	return path.Dir(filename)
+}
+
+func getManifestJsonData(t *testing.T, filePath string) tools.Build {
+	var response tools.Build
+
+	contents, err := os.Open(filePath)
+	assert.NoError(t, err)
+
+	err = json.NewDecoder(contents).Decode(&response)
+	assert.NoError(t, err)
+
+	return response
+}
+
+func TestBlah(t *testing.T) {
+	parentDir := ParentDir()
+
+	tcs := []struct {
+		name            string
+		filePath        string
+		componentName   string
+		packageName     string
+		requiredPackage string
+		expectedUrlList []string
+	}{
+		{
+			name:            "Unified Release Staging 8.14 apm-server",
+			filePath:        path.Join(parentDir, "test_payload", "manifest-8.14.2.json"),
+			componentName:   "apm-server",
+			packageName:     "apm-server",
+			requiredPackage: "linux-x86_64.tar.gz",
+			expectedUrlList: []string{
+				"https://staging.elastic.co/8.14.2-cfd42f49/downloads/apm-server/apm-server-8.14.2-linux-x86_64.tar.gz",
+				"https://staging.elastic.co/8.14.2-cfd42f49/downloads/apm-server/apm-server-8.14.2-linux-x86_64.tar.gz.sha512",
+				"https://staging.elastic.co/8.14.2-cfd42f49/downloads/apm-server/apm-server-8.14.2-linux-x86_64.tar.gz.asc",
+			},
+		},
+		{
+			name:            "Unified Release Snapshot 8.14 apm-server",
+			filePath:        path.Join(parentDir, "test_payload", "manifest-8.14.2-SNAPSHOT.json"),
+			componentName:   "apm-server",
+			packageName:     "apm-server",
+			requiredPackage: "linux-x86_64.tar.gz",
+			expectedUrlList: []string{
+				"https://snapshots.elastic.co/8.14.2-1ceac187/downloads/apm-server/apm-server-8.14.2-SNAPSHOT-linux-x86_64.tar.gz",
+				"https://snapshots.elastic.co/8.14.2-1ceac187/downloads/apm-server/apm-server-8.14.2-SNAPSHOT-linux-x86_64.tar.gz.sha512",
+				"https://snapshots.elastic.co/8.14.2-1ceac187/downloads/apm-server/apm-server-8.14.2-SNAPSHOT-linux-x86_64.tar.gz.asc",
+			},
+		},
+		{
+			name:            "Independent Agent Staging 8.14 apm-server",
+			filePath:        path.Join(parentDir, "test_payload", "manifest-8.14.0+build202406201002.json"),
+			componentName:   "apm-server",
+			packageName:     "apm-server",
+			requiredPackage: "linux-x86_64.tar.gz",
+			expectedUrlList: []string{
+				"https://staging.elastic.co/8.14.0-fe696c51/downloads/apm-server/apm-server-8.14.0-linux-x86_64.tar.gz",
+				"https://staging.elastic.co/8.14.0-fe696c51/downloads/apm-server/apm-server-8.14.0-linux-x86_64.tar.gz.sha512",
+				"https://staging.elastic.co/8.14.0-fe696c51/downloads/apm-server/apm-server-8.14.0-linux-x86_64.tar.gz.asc",
+			},
+		},
+		{
+			name:            "Unified Release Staging 8.14 endpoint-dev",
+			filePath:        path.Join(parentDir, "test_payload", "manifest-8.14.2.json"),
+			componentName:   "endpoint-dev",
+			packageName:     "endpoint-security",
+			requiredPackage: "linux-x86_64.tar.gz",
+			expectedUrlList: []string{
+				"https://staging.elastic.co/8.14.2-cfd42f49/downloads/endpoint-dev/endpoint-security-8.14.2-linux-x86_64.tar.gz",
+				"https://staging.elastic.co/8.14.2-cfd42f49/downloads/endpoint-dev/endpoint-security-8.14.2-linux-x86_64.tar.gz.sha512",
+				"https://staging.elastic.co/8.14.2-cfd42f49/downloads/endpoint-dev/endpoint-security-8.14.2-linux-x86_64.tar.gz.asc",
+			},
+		},
+		{
+			name:            "Unified Release Snapshot 8.14 endpoint-dev",
+			filePath:        path.Join(parentDir, "test_payload", "manifest-8.14.2-SNAPSHOT.json"),
+			componentName:   "endpoint-dev",
+			packageName:     "endpoint-security",
+			requiredPackage: "linux-x86_64.tar.gz",
+			expectedUrlList: []string{
+				"https://snapshots.elastic.co/8.14.2-1ceac187/downloads/endpoint-dev/endpoint-security-8.14.2-SNAPSHOT-linux-x86_64.tar.gz",
+				"https://snapshots.elastic.co/8.14.2-1ceac187/downloads/endpoint-dev/endpoint-security-8.14.2-SNAPSHOT-linux-x86_64.tar.gz.sha512",
+				"https://snapshots.elastic.co/8.14.2-1ceac187/downloads/endpoint-dev/endpoint-security-8.14.2-SNAPSHOT-linux-x86_64.tar.gz.asc",
+			},
+		},
+		{
+			name:            "Independent Agent Staging 8.14 endpoint-dev",
+			filePath:        path.Join(parentDir, "test_payload", "manifest-8.14.0+build202406201002.json"),
+			componentName:   "endpoint-dev",
+			packageName:     "endpoint-security",
+			requiredPackage: "linux-x86_64.tar.gz",
+			// Note how the version is one patch release higher than the manifest - this is expected
+			expectedUrlList: []string{
+				"https://staging.elastic.co/independent-agent/8.14.1+build202406201002/downloads/endpoint-dev/endpoint-security-8.14.1-linux-x86_64.tar.gz",
+				"https://staging.elastic.co/independent-agent/8.14.1+build202406201002/downloads/endpoint-dev/endpoint-security-8.14.1-linux-x86_64.tar.gz.sha512",
+				"https://staging.elastic.co/independent-agent/8.14.1+build202406201002/downloads/endpoint-dev/endpoint-security-8.14.1-linux-x86_64.tar.gz.asc",
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			manifestJson := getManifestJsonData(t, tc.filePath)
+			log.Printf("Manifest Version: [%s]", manifestJson.Version)
+
+			projects := manifestJson.Projects
+
+			// Verify the component name is in the ComponentSpec
+			_, ok := ComponentSpec[tc.componentName]
+			assert.True(t, ok)
+
+			urlList := resolveManifestPackage(projects[tc.componentName], tc.packageName, tc.requiredPackage, manifestJson.Version)
+
+			assert.Len(t, urlList, 3)
+			for _, url := range urlList {
+				assert.Contains(t, tc.expectedUrlList, url)
+			}
+		})
+	}
+}