Implement content packages discovery (#1235)

Add a new discovery parameter to `/search` and `/categories` endpoints, to filter packages based on their discovery fields. It supports a list of fields that has to be passed in the form `fields:<comma-separated list of fields>`. To pass the filter, packages must include discovery fields in their manifest, and all of them must be included in the list included in the parameter. For example the following request will return prerelease packages of content type, whose discovery subfields are a subset of ['http.method', 'apache.status.total_bytes']. ``` GET /search?prerelease=true&type=content&discovery=fields:http.method,apache.status.total_bytes ```
elastic · Oct 8, 2024 · 8899500 · 8899500
1 parent 9c30214
commit 8899500
Show file tree

Hide file tree

Showing 21 changed files with 209 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Add new `discovery` parameter to search and category endpoints. [#1235](https://github.com/elastic/package-registry/pull/1235)
+
 ### Deprecated
 
 ### Known Issues

diff --git a/categories.go b/categories.go
@@ -147,6 +147,14 @@ func newCategoriesFilterFromQuery(query url.Values) (*packages.Filter, error) {
 		}
 	}
 
+	if v := query.Get("discovery"); v != "" {
+		discovery, err := packages.NewDiscoveryFilter(v)
+		if err != nil {
+			return nil, fmt.Errorf("invalid 'discovery' query param: '%s': %w", v, err)
+		}
+		filter.Discovery = discovery
+	}
+
 	return &filter, nil
 }
 
@@ -219,7 +227,7 @@ func getCategoriesOutput(ctx context.Context, categories map[string]*packages.Ca
 	}
 	sort.Strings(keys)
 
-	var outputCategories []*packages.Category
+	outputCategories := []*packages.Category{}
 	for _, k := range keys {
 		c := categories[k]
 		if category, ok := packages.Categories[c.Title]; ok {

diff --git a/main_test.go b/main_test.go
@@ -77,6 +77,7 @@ func TestEndpoints(t *testing.T) {
 		{"/categories?spec.min=1.1&spec.max=2.10&prerelease=true", "/categories", "categories-spec-min-1.1.0-max-2.10.0.json", categoriesHandler(testLogger, indexer, testCacheTime)},
 		{"/categories?spec.max=2.10&prerelease=true", "/categories", "categories-spec-max-2.10.0.json", categoriesHandler(testLogger, indexer, testCacheTime)},
 		{"/categories?spec.max=2.10.1&prerelease=true", "/categories", "categories-spec-max-error.txt", categoriesHandler(testLogger, indexer, testCacheTime)},
+		{"/categories?discovery=fields:process.pid&prerelease=true", "/categories", "categories-discovery-fields-process-pid.txt", categoriesHandler(testLogger, indexer, testCacheTime)},
 		{"/search?kibana.version=6.5.2", "/search", "search-kibana652.json", searchHandler(testLogger, indexer, testCacheTime)},
 		{"/search?kibana.version=7.2.1", "/search", "search-kibana721.json", searchHandler(testLogger, indexer, testCacheTime)},
 		{"/search?kibana.version=8.0.0", "/search", "search-kibana800.json", searchHandler(testLogger, indexer, testCacheTime)},
@@ -101,6 +102,8 @@ func TestEndpoints(t *testing.T) {
 		{"/search?spec.min=1.1&spec.max=2.10&prerelease=true", "/search", "search-spec-min-1.1.0-max-2.10.0.json", searchHandler(testLogger, indexer, testCacheTime)},
 		{"/search?spec.max=2.10&prerelease=true", "/search", "search-spec-max-2.10.0.json", searchHandler(testLogger, indexer, testCacheTime)},
 		{"/search?spec.max=2.10.1&prerelease=true", "/search", "search-spec-max-error.txt", searchHandler(testLogger, indexer, testCacheTime)},
+		{"/search?prerelease=true&discovery=fields:process.pid", "/search", "search-discovery-fields-process-pid.txt", searchHandler(testLogger, indexer, testCacheTime)},
+		{"/search?prerelease=true&discovery=fields:non.existing.field", "/search", "search-discovery-fields-empty.txt", searchHandler(testLogger, indexer, testCacheTime)},
 		{"/favicon.ico", "", "favicon.ico", faviconHandleFunc},
 
 		// Removed flags, kept to ensure that they don't break requests from old versions.

diff --git a/packages/packages.go b/packages/packages.go
@@ -297,11 +297,63 @@ type Filter struct {
 	Capabilities   []string
 	SpecMin        *semver.Version
 	SpecMax        *semver.Version
+	Discovery      *discoveryFilter
 
 	// Deprecated, release tags to be removed.
 	Experimental bool
 }
 
+type discoveryFilter struct {
+	Fields discoveryFilterFields
+}
+
+func NewDiscoveryFilter(filter string) (*discoveryFilter, error) {
+	filterType, args, found := strings.Cut(filter, ":")
+	if !found {
+		return nil, fmt.Errorf("could not parse filter %q", filter)
+	}
+
+	var result discoveryFilter
+	switch filterType {
+	case "fields":
+		for _, name := range strings.Split(args, ",") {
+			result.Fields = append(result.Fields, DiscoveryField{
+				Name: name,
+			})
+		}
+	default:
+		return nil, fmt.Errorf("unknown discovery filter %q", filterType)
+	}
+
+	return &result, nil
+}
+
+func (f *discoveryFilter) Matches(p *Package) bool {
+	if f == nil {
+		return true
+	}
+	return f.Fields.Matches(p)
+}
+
+type discoveryFilterFields []DiscoveryField
+
+// Matches implements matching for a collection of fields used as discovery filter.
+// It matches if all fields in the package are included in the list of fields in the query.
+func (fields discoveryFilterFields) Matches(p *Package) bool {
+	// If the package doesn't define this filter, it doesn't match.
+	if p.Discovery == nil || len(p.Discovery.Fields) == 0 {
+		return false
+	}
+
+	for _, packageField := range p.Discovery.Fields {
+		if !slices.Contains([]DiscoveryField(fields), packageField) {
+			return false
+		}
+	}
+
+	return true
+}
+
 // Apply applies the filter to the list of packages, if the filter is nil, no filtering is done.
 func (f *Filter) Apply(ctx context.Context, packages Packages) (Packages, error) {
 	if f == nil {
@@ -352,6 +404,10 @@ func (f *Filter) Apply(ctx context.Context, packages Packages) (Packages, error)
 			}
 		}
 
+		if f.Discovery != nil && !f.Discovery.Matches(p) {
+			continue
+		}
+
 		if f.SpecMin != nil || f.SpecMax != nil {
 			valid, err := p.HasCompatibleSpec(f.SpecMin, f.SpecMax, f.KibanaVersion)
 			if err != nil {

diff --git a/packages/packages_test.go b/packages/packages_test.go
@@ -473,6 +473,10 @@ func TestPackagesSpecMinMaxFilter(t *testing.T) {
 			Version:       "2.0.0",
 			Type:          "integration",
 			KibanaVersion: "^7.17.0 || ^8.0.0",
+			DiscoveryFields: []string{
+				"host.ip",
+				"nginx.stubstatus.hostname",
+			},
 		},
 		{
 			FormatVersion: "1.0.0",
@@ -667,6 +671,26 @@ func TestPackagesSpecMinMaxFilter(t *testing.T) {
 				{Name: "redisenterprise", Version: "1.0.0"},
 			},
 		},
+		{
+			Title: "use fields discovery filter that no packages match",
+			Filter: Filter{
+				AllVersions: true,
+				Prerelease:  true,
+				Discovery:   mustBuildDiscoveryFilter("fields:apache.status.total_bytes"),
+			},
+			Expected: []filterTestPackage{},
+		},
+		{
+			Title: "use fields discovery filter for the nginx package",
+			Filter: Filter{
+				AllVersions: true,
+				Prerelease:  true,
+				Discovery:   mustBuildDiscoveryFilter("fields:host.ip,nginx.stubstatus.hostname"),
+			},
+			Expected: []filterTestPackage{
+				{Name: "nginx", Version: "2.0.0"},
+			},
+		},
 	}
 
 	for _, c := range cases {
@@ -678,14 +702,23 @@ func TestPackagesSpecMinMaxFilter(t *testing.T) {
 	}
 }
 
+func mustBuildDiscoveryFilter(filter string) *discoveryFilter {
+	f, err := NewDiscoveryFilter(filter)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
 type filterTestPackage struct {
-	FormatVersion string
-	Name          string
-	Version       string
-	Release       string
-	Type          string
-	KibanaVersion string
-	Capabilities  []string
+	FormatVersion   string
+	Name            string
+	Version         string
+	Release         string
+	Type            string
+	KibanaVersion   string
+	Capabilities    []string
+	DiscoveryFields []string
 }
 
 func (p filterTestPackage) Build() *Package {
@@ -727,6 +760,15 @@ func (p filterTestPackage) Build() *Package {
 		}
 	}
 
+	for _, name := range p.DiscoveryFields {
+		if build.Discovery == nil {
+			build.Discovery = &Discovery{}
+		}
+		build.Discovery.Fields = append(build.Discovery.Fields, DiscoveryField{
+			Name: name,
+		})
+	}
+
 	// set spec semver.Version variables
 	build.setRuntimeFields()
 	return &build

diff --git a/packages/testdata/marshaler/packages.json b/packages/testdata/marshaler/packages.json
@@ -1115,6 +1115,9 @@
     "type": "elastic",
     "github": "elastic/ecosystem"
    },
+   "categories": [
+    "web"
+   ],
    "discovery": {
     "fields": [
      {

diff --git a/search.go b/search.go
@@ -143,6 +143,14 @@ func newSearchFilterFromQuery(query url.Values) (*packages.Filter, error) {
 		}
 	}
 
+	if v := query.Get("discovery"); v != "" {
+		discovery, err := packages.NewDiscoveryFilter(v)
+		if err != nil {
+			return nil, fmt.Errorf("invalid 'discovery' query param: '%s': %w", v, err)
+		}
+		filter.Discovery = discovery
+	}
+
 	return &filter, nil
 }
 

diff --git a/testdata/generated/categories-discovery-fields-process-pid.txt b/testdata/generated/categories-discovery-fields-process-pid.txt
@@ -0,0 +1,9 @@
+[
+  {
+    "id": "web",
+    "title": "Web Server",
+    "count": 1,
+    "parent_id": "observability",
+    "parent_title": "Observability"
+  }
+]
diff --git a/testdata/generated/categories-experimental.json b/testdata/generated/categories-experimental.json
@@ -51,7 +51,7 @@
   {
     "id": "web",
     "title": "Web Server",
-    "count": 3,
+    "count": 4,
     "parent_id": "observability",
     "parent_title": "Observability"
   }

diff --git a/testdata/generated/categories-prerelease-capabilities-none.json b/testdata/generated/categories-prerelease-capabilities-none.json
@@ -51,7 +51,7 @@
   {
     "id": "web",
     "title": "Web Server",
-    "count": 3,
+    "count": 4,
     "parent_id": "observability",
     "parent_title": "Observability"
   }

diff --git a/testdata/generated/categories-prerelease-capabilities-observability-security.json b/testdata/generated/categories-prerelease-capabilities-observability-security.json
@@ -51,7 +51,7 @@
   {
     "id": "web",
     "title": "Web Server",
-    "count": 3,
+    "count": 4,
     "parent_id": "observability",
     "parent_title": "Observability"
   }

diff --git a/testdata/generated/categories-prerelease.json b/testdata/generated/categories-prerelease.json
@@ -51,7 +51,7 @@
   {
     "id": "web",
     "title": "Web Server",
-    "count": 3,
+    "count": 4,
     "parent_id": "observability",
     "parent_title": "Observability"
   }

diff --git a/testdata/generated/package/good_content/0.1.0/index.json b/testdata/generated/package/good_content/0.1.0/index.json
@@ -31,6 +31,9 @@
     "type": "elastic",
     "github": "elastic/ecosystem"
   },
+  "categories": [
+    "web"
+  ],
   "discovery": {
     "fields": [
       {

diff --git a/testdata/generated/search-content-packages.json b/testdata/generated/search-content-packages.json
@@ -32,6 +32,9 @@
       "type": "elastic",
       "github": "elastic/ecosystem"
     },
+    "categories": [
+      "web"
+    ],
     "discovery": {
       "fields": [
         {

diff --git a/testdata/generated/search-discovery-fields-empty.txt b/testdata/generated/search-discovery-fields-empty.txt
@@ -0,0 +1 @@
+[]
diff --git a/testdata/generated/search-discovery-fields-process-pid.txt b/testdata/generated/search-discovery-fields-process-pid.txt
@@ -0,0 +1,46 @@
+[
+  {
+    "name": "good_content",
+    "title": "Good content package",
+    "version": "0.1.0",
+    "release": "beta",
+    "source": {
+      "license": "Apache-2.0"
+    },
+    "description": "This package is a dummy example for packages with the content type. These packages contain resources that are useful with data ingested by other integrations. They are not used to configure data sources.\n",
+    "type": "content",
+    "download": "/epr/good_content/good_content-0.1.0.zip",
+    "path": "/package/good_content/0.1.0",
+    "icons": [
+      {
+        "src": "/img/system.svg",
+        "path": "/package/good_content/0.1.0/img/system.svg",
+        "title": "system",
+        "size": "1000x1000",
+        "type": "image/svg+xml"
+      }
+    ],
+    "conditions": {
+      "kibana": {
+        "version": "^8.16.0"
+      },
+      "elastic": {
+        "subscription": "basic"
+      }
+    },
+    "owner": {
+      "type": "elastic",
+      "github": "elastic/ecosystem"
+    },
+    "categories": [
+      "web"
+    ],
+    "discovery": {
+      "fields": [
+        {
+          "name": "process.pid"
+        }
+      ]
+    }
+  }
+]
diff --git a/testdata/generated/search-package-experimental.json b/testdata/generated/search-package-experimental.json
@@ -282,6 +282,9 @@
       "type": "elastic",
       "github": "elastic/ecosystem"
     },
+    "categories": [
+      "web"
+    ],
     "discovery": {
       "fields": [
         {

diff --git a/testdata/generated/search-package-prerelease.json b/testdata/generated/search-package-prerelease.json
@@ -290,6 +290,9 @@
       "type": "elastic",
       "github": "elastic/ecosystem"
     },
+    "categories": [
+      "web"
+    ],
     "discovery": {
       "fields": [
         {

diff --git a/testdata/generated/search-prerelease-capabilities-none.json b/testdata/generated/search-prerelease-capabilities-none.json
@@ -257,6 +257,9 @@
       "type": "elastic",
       "github": "elastic/ecosystem"
     },
+    "categories": [
+      "web"
+    ],
     "discovery": {
       "fields": [
         {

diff --git a/testdata/generated/search-prerelease-capabilities-observability-security.json b/testdata/generated/search-prerelease-capabilities-observability-security.json
@@ -265,6 +265,9 @@
       "type": "elastic",
       "github": "elastic/ecosystem"
     },
+    "categories": [
+      "web"
+    ],
     "discovery": {
       "fields": [
         {

diff --git a/testdata/package/good_content/0.1.0/manifest.yml b/testdata/package/good_content/0.1.0/manifest.yml
@@ -9,6 +9,7 @@ version: 0.1.0
 type: content
 source:
   license: "Apache-2.0"
+categories: ["web"]
 conditions:
   kibana:
     version: '^8.16.0' #TBD