diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aa03fa70..834d7f543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Add new `discovery` parameter to search and category endpoints. [#1235](https://github.com/elastic/package-registry/pull/1235) + ### Deprecated ### Known Issues diff --git a/categories.go b/categories.go index e5c344406..354d081b8 100644 --- a/categories.go +++ b/categories.go @@ -147,6 +147,14 @@ func newCategoriesFilterFromQuery(query url.Values) (*packages.Filter, error) { } } + if v := query.Get("discovery"); v != "" { + discovery, err := packages.NewDiscoveryFilter(v) + if err != nil { + return nil, fmt.Errorf("invalid 'discovery' query param: '%s': %w", v, err) + } + filter.Discovery = discovery + } + return &filter, nil } @@ -219,7 +227,7 @@ func getCategoriesOutput(ctx context.Context, categories map[string]*packages.Ca } sort.Strings(keys) - var outputCategories []*packages.Category + outputCategories := []*packages.Category{} for _, k := range keys { c := categories[k] if category, ok := packages.Categories[c.Title]; ok { diff --git a/main_test.go b/main_test.go index 680dfad54..ec654b275 100644 --- a/main_test.go +++ b/main_test.go @@ -77,6 +77,7 @@ func TestEndpoints(t *testing.T) { {"/categories?spec.min=1.1&spec.max=2.10&prerelease=true", "/categories", "categories-spec-min-1.1.0-max-2.10.0.json", categoriesHandler(testLogger, indexer, testCacheTime)}, {"/categories?spec.max=2.10&prerelease=true", "/categories", "categories-spec-max-2.10.0.json", categoriesHandler(testLogger, indexer, testCacheTime)}, {"/categories?spec.max=2.10.1&prerelease=true", "/categories", "categories-spec-max-error.txt", categoriesHandler(testLogger, indexer, testCacheTime)}, + {"/categories?discovery=fields:process.pid&prerelease=true", "/categories", "categories-discovery-fields-process-pid.txt", categoriesHandler(testLogger, indexer, testCacheTime)}, {"/search?kibana.version=6.5.2", "/search", "search-kibana652.json", searchHandler(testLogger, indexer, testCacheTime)}, {"/search?kibana.version=7.2.1", "/search", "search-kibana721.json", searchHandler(testLogger, indexer, testCacheTime)}, {"/search?kibana.version=8.0.0", "/search", "search-kibana800.json", searchHandler(testLogger, indexer, testCacheTime)}, @@ -101,6 +102,8 @@ func TestEndpoints(t *testing.T) { {"/search?spec.min=1.1&spec.max=2.10&prerelease=true", "/search", "search-spec-min-1.1.0-max-2.10.0.json", searchHandler(testLogger, indexer, testCacheTime)}, {"/search?spec.max=2.10&prerelease=true", "/search", "search-spec-max-2.10.0.json", searchHandler(testLogger, indexer, testCacheTime)}, {"/search?spec.max=2.10.1&prerelease=true", "/search", "search-spec-max-error.txt", searchHandler(testLogger, indexer, testCacheTime)}, + {"/search?prerelease=true&discovery=fields:process.pid", "/search", "search-discovery-fields-process-pid.txt", searchHandler(testLogger, indexer, testCacheTime)}, + {"/search?prerelease=true&discovery=fields:non.existing.field", "/search", "search-discovery-fields-empty.txt", searchHandler(testLogger, indexer, testCacheTime)}, {"/favicon.ico", "", "favicon.ico", faviconHandleFunc}, // Removed flags, kept to ensure that they don't break requests from old versions. diff --git a/packages/packages.go b/packages/packages.go index 7a0cff47d..1b11781fa 100644 --- a/packages/packages.go +++ b/packages/packages.go @@ -297,11 +297,63 @@ type Filter struct { Capabilities []string SpecMin *semver.Version SpecMax *semver.Version + Discovery *discoveryFilter // Deprecated, release tags to be removed. Experimental bool } +type discoveryFilter struct { + Fields discoveryFilterFields +} + +func NewDiscoveryFilter(filter string) (*discoveryFilter, error) { + filterType, args, found := strings.Cut(filter, ":") + if !found { + return nil, fmt.Errorf("could not parse filter %q", filter) + } + + var result discoveryFilter + switch filterType { + case "fields": + for _, name := range strings.Split(args, ",") { + result.Fields = append(result.Fields, DiscoveryField{ + Name: name, + }) + } + default: + return nil, fmt.Errorf("unknown discovery filter %q", filterType) + } + + return &result, nil +} + +func (f *discoveryFilter) Matches(p *Package) bool { + if f == nil { + return true + } + return f.Fields.Matches(p) +} + +type discoveryFilterFields []DiscoveryField + +// Matches implements matching for a collection of fields used as discovery filter. +// It matches if all fields in the package are included in the list of fields in the query. +func (fields discoveryFilterFields) Matches(p *Package) bool { + // If the package doesn't define this filter, it doesn't match. + if p.Discovery == nil || len(p.Discovery.Fields) == 0 { + return false + } + + for _, packageField := range p.Discovery.Fields { + if !slices.Contains([]DiscoveryField(fields), packageField) { + return false + } + } + + return true +} + // Apply applies the filter to the list of packages, if the filter is nil, no filtering is done. func (f *Filter) Apply(ctx context.Context, packages Packages) (Packages, error) { if f == nil { @@ -352,6 +404,10 @@ func (f *Filter) Apply(ctx context.Context, packages Packages) (Packages, error) } } + if f.Discovery != nil && !f.Discovery.Matches(p) { + continue + } + if f.SpecMin != nil || f.SpecMax != nil { valid, err := p.HasCompatibleSpec(f.SpecMin, f.SpecMax, f.KibanaVersion) if err != nil { diff --git a/packages/packages_test.go b/packages/packages_test.go index 5dd7b5f76..05521315b 100644 --- a/packages/packages_test.go +++ b/packages/packages_test.go @@ -473,6 +473,10 @@ func TestPackagesSpecMinMaxFilter(t *testing.T) { Version: "2.0.0", Type: "integration", KibanaVersion: "^7.17.0 || ^8.0.0", + DiscoveryFields: []string{ + "host.ip", + "nginx.stubstatus.hostname", + }, }, { FormatVersion: "1.0.0", @@ -667,6 +671,26 @@ func TestPackagesSpecMinMaxFilter(t *testing.T) { {Name: "redisenterprise", Version: "1.0.0"}, }, }, + { + Title: "use fields discovery filter that no packages match", + Filter: Filter{ + AllVersions: true, + Prerelease: true, + Discovery: mustBuildDiscoveryFilter("fields:apache.status.total_bytes"), + }, + Expected: []filterTestPackage{}, + }, + { + Title: "use fields discovery filter for the nginx package", + Filter: Filter{ + AllVersions: true, + Prerelease: true, + Discovery: mustBuildDiscoveryFilter("fields:host.ip,nginx.stubstatus.hostname"), + }, + Expected: []filterTestPackage{ + {Name: "nginx", Version: "2.0.0"}, + }, + }, } for _, c := range cases { @@ -678,14 +702,23 @@ func TestPackagesSpecMinMaxFilter(t *testing.T) { } } +func mustBuildDiscoveryFilter(filter string) *discoveryFilter { + f, err := NewDiscoveryFilter(filter) + if err != nil { + panic(err) + } + return f +} + type filterTestPackage struct { - FormatVersion string - Name string - Version string - Release string - Type string - KibanaVersion string - Capabilities []string + FormatVersion string + Name string + Version string + Release string + Type string + KibanaVersion string + Capabilities []string + DiscoveryFields []string } func (p filterTestPackage) Build() *Package { @@ -727,6 +760,15 @@ func (p filterTestPackage) Build() *Package { } } + for _, name := range p.DiscoveryFields { + if build.Discovery == nil { + build.Discovery = &Discovery{} + } + build.Discovery.Fields = append(build.Discovery.Fields, DiscoveryField{ + Name: name, + }) + } + // set spec semver.Version variables build.setRuntimeFields() return &build diff --git a/packages/testdata/marshaler/packages.json b/packages/testdata/marshaler/packages.json index 97c478041..26ee3018a 100644 --- a/packages/testdata/marshaler/packages.json +++ b/packages/testdata/marshaler/packages.json @@ -1115,6 +1115,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/search.go b/search.go index a010feab4..93cfa6571 100644 --- a/search.go +++ b/search.go @@ -143,6 +143,14 @@ func newSearchFilterFromQuery(query url.Values) (*packages.Filter, error) { } } + if v := query.Get("discovery"); v != "" { + discovery, err := packages.NewDiscoveryFilter(v) + if err != nil { + return nil, fmt.Errorf("invalid 'discovery' query param: '%s': %w", v, err) + } + filter.Discovery = discovery + } + return &filter, nil } diff --git a/testdata/generated/categories-discovery-fields-process-pid.txt b/testdata/generated/categories-discovery-fields-process-pid.txt new file mode 100644 index 000000000..5c34afb61 --- /dev/null +++ b/testdata/generated/categories-discovery-fields-process-pid.txt @@ -0,0 +1,9 @@ +[ + { + "id": "web", + "title": "Web Server", + "count": 1, + "parent_id": "observability", + "parent_title": "Observability" + } +] diff --git a/testdata/generated/categories-experimental.json b/testdata/generated/categories-experimental.json index 3187dc826..8e22f9266 100644 --- a/testdata/generated/categories-experimental.json +++ b/testdata/generated/categories-experimental.json @@ -51,7 +51,7 @@ { "id": "web", "title": "Web Server", - "count": 3, + "count": 4, "parent_id": "observability", "parent_title": "Observability" } diff --git a/testdata/generated/categories-prerelease-capabilities-none.json b/testdata/generated/categories-prerelease-capabilities-none.json index 22db3e84e..c8a3778ff 100644 --- a/testdata/generated/categories-prerelease-capabilities-none.json +++ b/testdata/generated/categories-prerelease-capabilities-none.json @@ -51,7 +51,7 @@ { "id": "web", "title": "Web Server", - "count": 3, + "count": 4, "parent_id": "observability", "parent_title": "Observability" } diff --git a/testdata/generated/categories-prerelease-capabilities-observability-security.json b/testdata/generated/categories-prerelease-capabilities-observability-security.json index c924a6a66..4ba4a850b 100644 --- a/testdata/generated/categories-prerelease-capabilities-observability-security.json +++ b/testdata/generated/categories-prerelease-capabilities-observability-security.json @@ -51,7 +51,7 @@ { "id": "web", "title": "Web Server", - "count": 3, + "count": 4, "parent_id": "observability", "parent_title": "Observability" } diff --git a/testdata/generated/categories-prerelease.json b/testdata/generated/categories-prerelease.json index 5b60b59e0..6796d6af8 100644 --- a/testdata/generated/categories-prerelease.json +++ b/testdata/generated/categories-prerelease.json @@ -51,7 +51,7 @@ { "id": "web", "title": "Web Server", - "count": 3, + "count": 4, "parent_id": "observability", "parent_title": "Observability" } diff --git a/testdata/generated/package/good_content/0.1.0/index.json b/testdata/generated/package/good_content/0.1.0/index.json index 82acef40c..34b698c23 100644 --- a/testdata/generated/package/good_content/0.1.0/index.json +++ b/testdata/generated/package/good_content/0.1.0/index.json @@ -31,6 +31,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/testdata/generated/search-content-packages.json b/testdata/generated/search-content-packages.json index bc7f5c587..815aea474 100644 --- a/testdata/generated/search-content-packages.json +++ b/testdata/generated/search-content-packages.json @@ -32,6 +32,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/testdata/generated/search-discovery-fields-empty.txt b/testdata/generated/search-discovery-fields-empty.txt new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/testdata/generated/search-discovery-fields-empty.txt @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/testdata/generated/search-discovery-fields-process-pid.txt b/testdata/generated/search-discovery-fields-process-pid.txt new file mode 100644 index 000000000..815aea474 --- /dev/null +++ b/testdata/generated/search-discovery-fields-process-pid.txt @@ -0,0 +1,46 @@ +[ + { + "name": "good_content", + "title": "Good content package", + "version": "0.1.0", + "release": "beta", + "source": { + "license": "Apache-2.0" + }, + "description": "This package is a dummy example for packages with the content type. These packages contain resources that are useful with data ingested by other integrations. They are not used to configure data sources.\n", + "type": "content", + "download": "/epr/good_content/good_content-0.1.0.zip", + "path": "/package/good_content/0.1.0", + "icons": [ + { + "src": "/img/system.svg", + "path": "/package/good_content/0.1.0/img/system.svg", + "title": "system", + "size": "1000x1000", + "type": "image/svg+xml" + } + ], + "conditions": { + "kibana": { + "version": "^8.16.0" + }, + "elastic": { + "subscription": "basic" + } + }, + "owner": { + "type": "elastic", + "github": "elastic/ecosystem" + }, + "categories": [ + "web" + ], + "discovery": { + "fields": [ + { + "name": "process.pid" + } + ] + } + } +] diff --git a/testdata/generated/search-package-experimental.json b/testdata/generated/search-package-experimental.json index 4c5f5547f..09c5c2c55 100644 --- a/testdata/generated/search-package-experimental.json +++ b/testdata/generated/search-package-experimental.json @@ -282,6 +282,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/testdata/generated/search-package-prerelease.json b/testdata/generated/search-package-prerelease.json index 28f50606a..822256fa3 100644 --- a/testdata/generated/search-package-prerelease.json +++ b/testdata/generated/search-package-prerelease.json @@ -290,6 +290,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/testdata/generated/search-prerelease-capabilities-none.json b/testdata/generated/search-prerelease-capabilities-none.json index 9709566db..46e784af7 100644 --- a/testdata/generated/search-prerelease-capabilities-none.json +++ b/testdata/generated/search-prerelease-capabilities-none.json @@ -257,6 +257,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/testdata/generated/search-prerelease-capabilities-observability-security.json b/testdata/generated/search-prerelease-capabilities-observability-security.json index cbf86303e..a2a976bd8 100644 --- a/testdata/generated/search-prerelease-capabilities-observability-security.json +++ b/testdata/generated/search-prerelease-capabilities-observability-security.json @@ -265,6 +265,9 @@ "type": "elastic", "github": "elastic/ecosystem" }, + "categories": [ + "web" + ], "discovery": { "fields": [ { diff --git a/testdata/package/good_content/0.1.0/manifest.yml b/testdata/package/good_content/0.1.0/manifest.yml index be8a26f0a..2a4a3a215 100644 --- a/testdata/package/good_content/0.1.0/manifest.yml +++ b/testdata/package/good_content/0.1.0/manifest.yml @@ -9,6 +9,7 @@ version: 0.1.0 type: content source: license: "Apache-2.0" +categories: ["web"] conditions: kibana: version: '^8.16.0' #TBD