[8.x] [Entity Store] [Asset Inventory] Universal entity definition (e…

…lastic#202888) (elastic#205536) # Backport This will backport the following commits from `main` to `8.x`: - [[Entity Store] [Asset Inventory] Universal entity definition (elastic#202888)](elastic#202888)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Tiago Vila Verde <[email protected]>
jbudz · Jan 3, 2025 · 4994699 · 4994699
1 parent 25ede9b
commit 4994699
Show file tree

Hide file tree

Showing 54 changed files with 1,130 additions and 1,130 deletions.
diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
@@ -45909,6 +45909,8 @@ components:
       enum:
         - user
         - host
+        - service
+        - universal
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object
@@ -45979,6 +45981,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     Security_Entity_Analytics_API_IndexPattern:
       type: string
@@ -50085,7 +50088,7 @@ components:
           items:
             type: string
           description: |
-            A list of "carbon copy" email addresses. Addresses can be specified in `user@host-name` format or in name `<user@host-name>` format 
+            A list of "carbon copy" email addresses. Addresses can be specified in `user@host-name` format or in name `<user@host-name>` format
         message:
           type: string
           description: The email message text. Markdown format is supported.

diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -35059,6 +35059,7 @@ components:
         - user
         - host
         - service
+        - universal
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object
@@ -35129,6 +35130,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     Security_Entity_Analytics_API_IndexPattern:
       type: string

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts b/...ity/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts
@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type IdField = z.infer<typeof IdField>;
-export const IdField = z.enum(['host.name', 'user.name', 'service.name']);
+export const IdField = z.enum(['host.name', 'user.name', 'service.name', 'related.entity']);
 export type IdFieldEnum = typeof IdField.enum;
 export const IdFieldEnum = IdField.enum;
 

diff --git a/...lugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml b/...lugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml
@@ -29,6 +29,7 @@ components:
         - 'host.name'
         - 'user.name'
         - 'service.name'
+        - 'related.entity'
     AssetCriticalityRecordIdParts:
       type: object
       properties:

diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type EntityType = z.infer<typeof EntityType>;
-export const EntityType = z.enum(['user', 'host', 'service']);
+export const EntityType = z.enum(['user', 'host', 'service', 'universal']);
 export type EntityTypeEnum = typeof EntityType.enum;
 export const EntityTypeEnum = EntityType.enum;
 

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@@ -12,6 +12,7 @@ components:
         - user
         - host
         - service
+        - universal
 
     EngineDescriptor:
       type: object

diff --git a/...olution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts b/...olution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts
@@ -46,7 +46,7 @@ describe('parseAssetCriticalityCsvRow', () => {
 
     // @ts-ignore result can now only be InvalidRecord
     expect(result.error).toMatchInlineSnapshot(
-      `"Invalid entity type \\"invalid\\", expected to be one of: user, host, service"`
+      `"Invalid entity type \\"invalid\\", expected to be one of: user, host, service, universal"`
     );
   });
 
@@ -57,7 +57,7 @@ describe('parseAssetCriticalityCsvRow', () => {
 
     // @ts-ignore result can now only be InvalidRecord
     expect(result.error).toMatchInlineSnapshot(
-      `"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service"`
+      `"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service, universal"`
     );
   });
 

diff --git a/...ions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts b/...ions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts
@@ -31,6 +31,7 @@ export const IDENTITY_FIELD_MAP: Record<EntityType, IdField> = {
   [EntityTypeEnum.host]: 'host.name',
   [EntityTypeEnum.user]: 'user.name',
   [EntityTypeEnum.service]: 'service.name',
+  [EntityTypeEnum.universal]: 'related.entity',
 };
 
 export const getAvailableEntityTypes = (): EntityType[] =>

diff --git a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts
@@ -266,8 +266,13 @@ export const allowedExperimentalValues = Object.freeze({
   /**
    * Enables CrowdStrike's RunScript RTR command
    */
-
   crowdstrikeRunScriptEnabled: false,
+
+  /**
+   * Enables the Asset Inventory Entity Store feature.
+   * Allows initializing the Universal Entity Store via the API.
+   */
+  assetInventoryStoreEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

diff --git a/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -1002,6 +1002,7 @@ components:
         - user
         - host
         - service
+        - universal
       type: string
     HostEntity:
       type: object
@@ -1072,6 +1073,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     IndexPattern:
       type: string

diff --git a/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -1002,6 +1002,7 @@ components:
         - user
         - host
         - service
+        - universal
       type: string
     HostEntity:
       type: object
@@ -1072,6 +1073,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     IndexPattern:
       type: string

diff --git a/...curity/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts b/...curity/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts
@@ -82,6 +82,7 @@ const entityTypeByIdField = {
   'host.name': 'host',
   'user.name': 'user',
   'service.name': 'service',
+  'related.entity': 'universal',
 } as const;
 
 export const getImplicitEntityFields = (record: AssetCriticalityUpsertWithDeleted) => {

diff --git a/...ution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts b/...ution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts
@@ -10,18 +10,24 @@ import {
   EngineComponentResourceEnum,
   type EngineComponentStatus,
 } from '../../../../../common/api/entity_analytics';
-import type { UnitedEntityDefinition } from '../united_entity_definitions';
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
 
 const getComponentTemplateName = (definitionId: string) => `${definitionId}-latest@platform`;
 
 interface Options {
-  unitedDefinition: UnitedEntityDefinition;
+  /**
+   * The entity engine description id
+   **/
+  id: string;
   esClient: ElasticsearchClient;
 }
 
-export const createEntityIndexComponentTemplate = ({ unitedDefinition, esClient }: Options) => {
-  const { entityManagerDefinition, indexMappings } = unitedDefinition;
-  const name = getComponentTemplateName(entityManagerDefinition.id);
+export const createEntityIndexComponentTemplate = (
+  description: EntityEngineInstallationDescriptor,
+  esClient: ElasticsearchClient
+) => {
+  const { id, indexMappings } = description;
+  const name = getComponentTemplateName(id);
   return esClient.cluster.putComponentTemplate({
     name,
     body: {
@@ -35,9 +41,8 @@ export const createEntityIndexComponentTemplate = ({ unitedDefinition, esClient
   });
 };
 
-export const deleteEntityIndexComponentTemplate = ({ unitedDefinition, esClient }: Options) => {
-  const { entityManagerDefinition } = unitedDefinition;
-  const name = getComponentTemplateName(entityManagerDefinition.id);
+export const deleteEntityIndexComponentTemplate = ({ id, esClient }: Options) => {
+  const name = getComponentTemplateName(id);
   return esClient.cluster.deleteComponentTemplate(
     { name },
     {

diff --git a/...y_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts b/...y_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts
@@ -6,12 +6,14 @@
  */
 
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
-import type { EnrichPutPolicyRequest } from '@elastic/elasticsearch/lib/api/types';
+import type { EntityType } from '../../../../../common/api/entity_analytics';
 import { EngineComponentResourceEnum } from '../../../../../common/api/entity_analytics';
 import { getEntitiesIndexName } from '../utils';
-import type { UnitedEntityDefinition } from '../united_entity_definitions';
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
 
-type DefinitionMetadata = Pick<UnitedEntityDefinition, 'namespace' | 'entityType' | 'version'>;
+type DefinitionMetadata = Pick<EntityEngineInstallationDescriptor, 'entityType' | 'version'> & {
+  namespace: string;
+};
 
 export const getFieldRetentionEnrichPolicyName = ({
   namespace,
@@ -21,69 +23,79 @@ export const getFieldRetentionEnrichPolicyName = ({
   return `entity_store_field_retention_${entityType}_${namespace}_v${version}`;
 };
 
-const getFieldRetentionEnrichPolicy = (
-  unitedDefinition: UnitedEntityDefinition
-): EnrichPutPolicyRequest => {
-  const { namespace, entityType, fieldRetentionDefinition } = unitedDefinition;
-  return {
-    name: getFieldRetentionEnrichPolicyName(unitedDefinition),
-    match: {
-      indices: getEntitiesIndexName(entityType, namespace),
-      match_field: fieldRetentionDefinition.matchField,
-      enrich_fields: fieldRetentionDefinition.fields.map(({ field }) => field),
-    },
-  };
-};
-
 export const createFieldRetentionEnrichPolicy = async ({
   esClient,
-  unitedDefinition,
+  description,
+  options,
 }: {
   esClient: ElasticsearchClient;
-  unitedDefinition: UnitedEntityDefinition;
+  description: EntityEngineInstallationDescriptor;
+  options: { namespace: string };
 }) => {
-  const policy = getFieldRetentionEnrichPolicy(unitedDefinition);
-  return esClient.enrich.putPolicy(policy);
+  return esClient.enrich.putPolicy({
+    name: getFieldRetentionEnrichPolicyName({
+      namespace: options.namespace,
+      entityType: description.entityType,
+      version: description.version,
+    }),
+    match: {
+      indices: getEntitiesIndexName(description.entityType, options.namespace),
+      match_field: description.identityField,
+      enrich_fields: description.fields.map(({ destination }) => destination),
+    },
+  });
 };
 
 export const executeFieldRetentionEnrichPolicy = async ({
   esClient,
-  unitedDefinition,
+  entityType,
+  version,
   logger,
+  options,
 }: {
-  unitedDefinition: DefinitionMetadata;
+  entityType: EntityType;
+  version: string;
   esClient: ElasticsearchClient;
   logger: Logger;
+  options: { namespace: string };
 }): Promise<{ executed: boolean }> => {
-  const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
+  const name = getFieldRetentionEnrichPolicyName({
+    namespace: options.namespace,
+    entityType,
+    version,
+  });
   try {
     await esClient.enrich.executePolicy({ name });
     return { executed: true };
   } catch (e) {
     if (e.statusCode === 404) {
       return { executed: false };
     }
-    logger.error(
-      `Error executing field retention enrich policy for ${unitedDefinition.entityType}: ${e.message}`
-    );
+    logger.error(`Error executing field retention enrich policy for ${entityType}: ${e.message}`);
     throw e;
   }
 };
 
 export const deleteFieldRetentionEnrichPolicy = async ({
-  unitedDefinition,
+  description,
+  options,
   esClient,
   logger,
   attempts = 5,
   delayMs = 2000,
 }: {
-  unitedDefinition: DefinitionMetadata;
+  description: EntityEngineInstallationDescriptor;
+  options: { namespace: string };
   esClient: ElasticsearchClient;
   logger: Logger;
   attempts?: number;
   delayMs?: number;
 }) => {
-  const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
+  const name = getFieldRetentionEnrichPolicyName({
+    namespace: options.namespace,
+    entityType: description.entityType,
+    version: description.version,
+  });
   let currentAttempt = 1;
   while (currentAttempt <= attempts) {
     try {

diff --git a/...ty_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts b/...ty_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts
@@ -6,6 +6,7 @@
  */
 
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 import {
   EngineComponentResourceEnum,
   type EngineComponentStatus,
@@ -14,6 +15,8 @@ import {
 import { getEntitiesIndexName } from '../utils';
 import { createOrUpdateIndex } from '../../utils/create_or_update_index';
 
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
+
 interface Options {
   entityType: EntityType;
   esClient: ElasticsearchClient;
@@ -58,3 +61,51 @@ export const getEntityIndexStatus = async ({
 
   return { id: index, installed: exists, resource: EngineComponentResourceEnum.index };
 };
+
+export type MappingProperties = NonNullable<MappingTypeMapping['properties']>;
+
+export const generateIndexMappings = (
+  description: EntityEngineInstallationDescriptor
+): MappingTypeMapping => {
+  const identityFieldMappings: MappingProperties = {
+    [description.identityField]: {
+      type: 'keyword',
+      fields: {
+        text: {
+          type: 'match_only_text',
+        },
+      },
+    },
+  };
+
+  const otherFieldMappings = description.fields
+    .filter(({ mapping }) => mapping)
+    .reduce((acc, { destination, mapping }) => {
+      acc[destination] = mapping;
+      return acc;
+    }, {} as MappingProperties);
+
+  return {
+    properties: { ...BASE_ENTITY_INDEX_MAPPING, ...identityFieldMappings, ...otherFieldMappings },
+  };
+};
+
+export const BASE_ENTITY_INDEX_MAPPING: MappingProperties = {
+  '@timestamp': {
+    type: 'date',
+  },
+  'asset.criticality': {
+    type: 'keyword',
+  },
+  'entity.name': {
+    type: 'keyword',
+    fields: {
+      text: {
+        type: 'match_only_text',
+      },
+    },
+  },
+  'entity.source': {
+    type: 'keyword',
+  },
+};
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,6 +12,7 @@ components: @@
             - user
             - host
             - service
+            - universal
         EngineDescriptor:
           type: object
@@ Expand Down @@