Initial definition of content pack[age]s (#777)

On this initial version, they only support the distribution of kibana assets, like dashboards.

This package type introduces a discovery mechanism based on the data the package can
work with. For example the following definition would indicate that the package provides
resources that are useful when used with data that includes the field `process.pid`:
```
discovery:
  fields:
    - name: "process.pid"
```

The package registry could add support to look for packages that have support for certain
fields. For example a request like the following one could be used to discover packages that
include resources that can leverage the `process.pid` or the `user.id` fields.
```
GET /search?discovery=fields:process.pid,user.id
```

code/go/internal/loader/spec.go

@@ -5,7 +5,10 @@
 package loader
 
 import (
+	"errors"
+	"fmt"
 	"io/fs"
+	"os"
 
 	"github.com/Masterminds/semver/v3"
 
@@ -18,5 +21,12 @@ import (
 func LoadSpec(fsys fs.FS, version semver.Version, pkgType string) (spectypes.ItemSpec, error) {
 	fileSpecLoader := yamlschema.NewFileSchemaLoader()
 	loader := specschema.NewFolderSpecLoader(fsys, fileSpecLoader, version)
-	return loader.Load(pkgType)
+	spec, err := loader.Load(pkgType)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf("package type %q not supported (%w)", pkgType, err)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return spec, nil
 }

code/go/internal/spec_test.go

@@ -19,6 +19,7 @@ func TestLoadAllBundledVersions(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, version := range versions {
+		testForVersionType(t, version, "content")
 		testForVersionType(t, version, "input")
 		testForVersionType(t, version, "integration")
 	}

code/go/internal/validator/spec.go

@@ -196,12 +196,12 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules
 		{fn: semantic.WarnOn(semantic.ValidateMinimumKibanaVersion), until: semver.MustParse("3.0.0")},
 		{fn: semantic.ValidateMinimumKibanaVersion, since: semver.MustParse("3.0.0")},
 		{fn: semantic.ValidateFieldGroups},
-		{fn: semantic.ValidateFieldsLimits(rootSpec.MaxFieldsPerDataStream())},
-		{fn: semantic.ValidateUniqueFields, since: semver.MustParse("2.0.0")},
-		{fn: semantic.ValidateDimensionFields},
-		{fn: semantic.ValidateDateFields},
-		{fn: semantic.ValidateRequiredFields},
-		{fn: semantic.ValidateExternalFieldsWithDevFolder},
+		{fn: semantic.ValidateFieldsLimits(rootSpec.MaxFieldsPerDataStream()), types: []string{"integration", "input"}},
+		{fn: semantic.ValidateUniqueFields, since: semver.MustParse("2.0.0"), types: []string{"integration", "input"}},
+		{fn: semantic.ValidateDimensionFields, types: []string{"integration", "input"}},
+		{fn: semantic.ValidateDateFields, types: []string{"integration", "input"}},
+		{fn: semantic.ValidateRequiredFields, types: []string{"integration", "input"}},
+		{fn: semantic.ValidateExternalFieldsWithDevFolder, types: []string{"integration", "input"}},
 		{fn: semantic.WarnOn(semantic.ValidateVisualizationsUsedByValue), types: []string{"integration"}, until: semver.MustParse("3.0.0")},
 		{fn: semantic.ValidateVisualizationsUsedByValue, types: []string{"integration"}, since: semver.MustParse("3.0.0")},
 		{fn: semantic.ValidateILMPolicyPresent, since: semver.MustParse("2.0.0"), types: []string{"integration"}},

code/go/pkg/specerrors/filter.go

@@ -23,25 +23,33 @@ type Filter struct {
 type FilterResult struct {
 	Processed error
 	Removed   error
+
+	UnusedProcessors []Processor
 }
 
 // Run runs all the processors over all the validation errors and return the filtered ones
 func (r *Filter) Run(allErrors ValidationErrors) (FilterResult, error) {
 	newErrors := allErrors
 	var allFiltered ValidationErrors
 
+	var unused []Processor
 	for _, p := range r.processors {
 		result, err := p.Process(newErrors)
 		if err != nil {
 			return FilterResult{Processed: allErrors, Removed: nil}, err
 		}
 		newErrors = result.Processed
 		allFiltered.Append(result.Removed)
+
+		if len(result.Removed) == 0 {
+			unused = append(unused, p)
+		}
 	}
 
 	return FilterResult{
-		Processed: nilOrValidationErrors(newErrors),
-		Removed:   nilOrValidationErrors(allFiltered),
+		Processed:        nilOrValidationErrors(newErrors),
+		Removed:          nilOrValidationErrors(allFiltered),
+		UnusedProcessors: unused,
 	}, nil
 }
 

code/go/pkg/validator/validator_test.go

@@ -33,6 +33,7 @@ func TestValidateFile(t *testing.T) {
 		"good_v2":                            {},
 		"good_v3":                            {},
 		"good_input":                         {},
+		"good_content":                       {},
 		"deploy_custom_agent":                {},
 		"deploy_custom_agent_multi_services": {},
 		"deploy_docker":                      {},
@@ -232,6 +233,7 @@ func TestValidateFile(t *testing.T) {
 					filter := specerrors.NewFilter(filterConfig)
 					result, err := filter.Run(verrs)
 					require.NoError(t, err)
+					assert.Empty(t, result.UnusedProcessors, "There are unused exclusion checks in the validation.yml file")
 					errs = result.Processed
 				}
 			}

spec/changelog.yml

@@ -2,11 +2,24 @@
 ## This file documents changes in the package specification. It is NOT a package specification file.
 ## Newer entries go at the bottom of each in-development version.
 ##
+- version: 3.4.0-next
+  changes:
+  - description: Add support for content packages.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
 - version: 3.3.0-next
   changes:
   - description: Add support for `slo` assets.
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/767
+  - description: Add spec for img directory.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
+- version: 3.2.2-next
+  changes:
+  - description: Improved error message for unsupported package types.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
 - version: 3.2.1
   changes:
   - description: Improve error information for missing dataset field in manifest.

spec/content/kibana/spec.yml

@@ -0,0 +1,31 @@
+spec:
+  additionalContents: false
+  contents:
+  - description: Folder containing Kibana dashboard assets
+    type: folder
+    name: dashboard
+    required: false
+    contents:
+    - description: A dashboard asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: '^{PACKAGE_NAME}-.+\.json$'
+      forbiddenPatterns:
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+  - description: File containing saved object tag definitions for assets
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "tags.yml"
+    required: false
+    $ref: "../../integration/kibana/tags.spec.yml"
+  - description: Folder containing Kibana SLO assets
+    type: folder
+    name: slo
+    required: false
+    contents:
+    - description: An SLO asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: '^{PACKAGE_NAME}-.+\.json$'
+      forbiddenPatterns:
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden

spec/content/manifest.spec.yml

@@ -0,0 +1,70 @@
+##
+## Describes the specification for the content package's main manifest.yml file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: object
+  additionalProperties: false
+  definitions:
+    discovery:
+      description: >
+        Description of the data this package can be used with. It can be used to discover
+        the package from elements in the existing data.
+      type: object
+      additionalProperties: false
+      properties:
+        fields:
+          description: Description of the fields this package can be used with.
+          type: array
+          items:
+            type: object
+            properties:
+              name:
+                description: Name of the field.
+                type: string
+  properties:
+    format_version:
+      description: The version of the package specification format used by this package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    name:
+      description: The name of the package.
+      type: string
+      pattern: '^[a-z0-9_]+$'
+      examples:
+      - apache
+    title:
+      $ref: "../integration/manifest.spec.yml#/definitions/title"
+    description:
+      $ref: "../integration/manifest.spec.yml#/definitions/description"
+    version:
+      description: The version of the package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    source:
+      $ref: "../integration/manifest.spec.yml#/definitions/source"
+    type:
+      description: The type of package.
+      type: string
+      enum:
+      - content
+      examples:
+      - content
+    categories:
+      $ref: "../integration/manifest.spec.yml#/definitions/categories"
+    conditions:
+      $ref: "../integration/manifest.spec.yml#/definitions/conditions"
+    discovery:
+      $ref: "#/definitions/discovery"
+    icons:
+      $ref: "../integration/manifest.spec.yml#/definitions/icons"
+    screenshots:
+      $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
+    owner:
+      $ref: "../integration/manifest.spec.yml#/definitions/owner"
+  required:
+  - format_version
+  - name
+  - title
+  - description
+  - version
+  - type
+  - owner

spec/content/spec.yml

@@ -0,0 +1,60 @@
+##
+## Entrypoint of "content packages" specification.
+## The specification is considered "beta" at the moment, so it may change until we release it as GA.
+##
+## Describes the folders and files that make up a package.
+##
+spec:
+  additionalContents: false
+  totalContentsLimit: 65535
+  totalSizeLimit: 250MB
+  sizeLimit: 150MB
+  configurationSizeLimit: 5MB
+  relativePathSizeLimit: 3MB
+  release: beta
+  contents:
+  - description: The main package manifest file
+    type: file
+    contentMediaType: "application/x-yaml"
+    sizeLimit: 5MB
+    name: "manifest.yml"
+    required: true
+    $ref: "./manifest.spec.yml"
+  - description: The package's CHANGELOG file
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "changelog.yml"
+    required: true
+    $ref: "../integration/changelog.spec.yml"
+  - description: The package's license file
+    type: file
+    contentMediaType: "text/plain"
+    name: "LICENSE.txt"
+    required: false
+  - description: Folder containing documentation for the package
+    type: folder
+    name: docs
+    required: true
+    $ref: "../integration/docs/spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "../integration/img/spec.yml"
+  - description: Folder containing Kibana assets provided by the package
+    type: folder
+    name: kibana
+    required: false
+    $ref: "./kibana/spec.yml"
+  - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package.
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "validation.yml"
+    required: false
+    $ref: "../integration/validation.spec.yml"
+
+versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/contents" # Package type not available before this version.

spec/input/spec.yml

@@ -1,6 +1,5 @@
 ##
 ## Entrypoint of "input packages" specification.
-## The specification is considered "beta" at the moment, so it may change until we release it as GA.
 ##
 ## Describes the folders and files that make up a package.
 ##
@@ -69,8 +68,17 @@ spec:
     name: "validation.yml"
     required: false
     $ref: "../integration/validation.spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "../integration/img/spec.yml"
 
 versions:
+  - before: 3.3.0
+    patch:
+      - op: remove
+        path: "/contents/10" # Definition for img folder.
   - before: 2.10.0
     patch:
       - op: remove

spec/integration/docs/spec.yml

@@ -1,13 +1,13 @@
-  spec:
-    additionalContents: false
-    contents:
-    - description: Main README file
-      type: file
-      contentMediaType: "text/markdown"
-      name: "README.md"
-      required: true
-    - description: Other README files (can be used by policy templates)
-      type: file
-      contentMediaType: "text/markdown"
-      pattern: '^.+.md'
-      required: false
+spec:
+  additionalContents: false
+  contents:
+  - description: Main README file
+    type: file
+    contentMediaType: "text/markdown"
+    name: "README.md"
+    required: true
+  - description: Other README files (can be used by policy templates)
+    type: file
+    contentMediaType: "text/markdown"
+    pattern: '^.+.md'
+    required: false

spec/integration/img/spec.yml

@@ -0,0 +1,2 @@
+spec:
+  additionalContents: true # TODO: Add definition for this dir?

spec/integration/kibana/tags.spec.yml

@@ -1,5 +1,5 @@
 ##
-## Describes the specification for a routing rules yml file
+## Describes the specification for the additional kibana tags provided by the package.
 ##
 spec:
   # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability

spec/integration/spec.yml

@@ -72,3 +72,14 @@ spec:
     name: "validation.yml"
     required: false
     $ref: "./validation.spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "./img/spec.yml"
+
+versions:
+  - before: 3.3.0
+    patch:
+      - op: remove
+        path: "/contents/11" # Definition for img folder.

test/packages/good_content/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.1.0
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/777

test/packages/good_content/docs/README.md

@@ -0,0 +1 @@
+# Reference package of content type