Add keyword builder pipeline

.buildkite/ftr_security_stateful_configs.yml

@@ -105,3 +105,4 @@ enabled:
   - x-pack/test/cloud_security_posture_functional/data_views/config.ts
   - x-pack/test/automatic_import_api_integration/apis/config_basic.ts
   - x-pack/test/automatic_import_api_integration/apis/config_graphs.ts
+  - x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts

.github/CODEOWNERS

@@ -2514,6 +2514,9 @@ x-pack/solutions/security/plugins/security_solution/public/common/components/ses
 x-pack/solutions/security/plugins/security_solution/public/cloud_defend @elastic/kibana-cloud-security-posture
 x-pack/solutions/security/plugins/security_solution/public/cloud_security_posture @elastic/kibana-cloud-security-posture
 x-pack/solutions/security/plugins/security_solution/public/kubernetes @elastic/kibana-cloud-security-posture
+x-pack/test/security_solution_api_integration/test_suites/asset_inventory @elastic/kibana-cloud-security-posture
+x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory @elastic/kibana-cloud-security-posture
+
 ## Fleet plugin (co-owned with Fleet team)
 x-pack/platform/plugins/shared/fleet/public/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture
 x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.mock.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { AssetInventoryDataClient } from './asset_inventory_data_client';
+
+const createAssetInventoryDataClientMock = () =>
+  ({
+    init: jest.fn(),
+    enable: jest.fn(),
+    delete: jest.fn(),
+  } as unknown as jest.Mocked<AssetInventoryDataClient>);
+
+export const AssetInventoryDataClientMock = { create: createAssetInventoryDataClientMock };

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger, ElasticsearchClient, IScopedClusterClient } from '@kbn/core/server';
+
+import type { ExperimentalFeatures } from '../../../common';
+
+import { createKeywordBuilderPipeline, deleteKeywordBuilderPipeline } from './ingest_pipelines';
+
+interface AssetInventoryClientOpts {
+  logger: Logger;
+  clusterClient: IScopedClusterClient;
+  experimentalFeatures: ExperimentalFeatures;
+}
+
+// AssetInventoryDataClient is responsible for managing the asset inventory,
+// including initializing and cleaning up resources such as Elasticsearch ingest pipelines.
+export class AssetInventoryDataClient {
+  private esClient: ElasticsearchClient;
+
+  constructor(private readonly options: AssetInventoryClientOpts) {
+    const { clusterClient } = options;
+    this.esClient = clusterClient.asCurrentUser;
+  }
+
+  // Enables the asset inventory by deferring the initialization to avoid blocking the main thread.
+  public async enable() {
+    // Utility function to defer execution to the next tick using setTimeout.
+    const run = <T>(fn: () => Promise<T>) =>
+      new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
+
+    // Defer and execute the initialization process.
+    await run(() => this.init());
+
+    return { succeeded: true };
+  }
+
+  // Initializes the asset inventory by validating experimental feature flags and triggering asynchronous setup.
+  public async init() {
+    const { experimentalFeatures, logger } = this.options;
+
+    if (!experimentalFeatures.assetInventoryStoreEnabled) {
+      throw new Error('Universal entity store is not enabled');
+    }
+
+    logger.debug(`Initializing asset inventory`);
+
+    this.asyncSetup().catch((e) =>
+      logger.error(`Error during async setup of asset inventory: ${e.message}`)
+    );
+  }
+
+  // Sets up the necessary resources for asset inventory, including creating Elasticsearch ingest pipelines.
+  private async asyncSetup() {
+    const { logger } = this.options;
+    try {
+      logger.debug('creating keyword builder pipeline');
+      await createKeywordBuilderPipeline({
+        logger,
+        esClient: this.esClient,
+      });
+      logger.debug('keyword builder pipeline created');
+    } catch (err) {
+      logger.error(`Error initializing asset inventory: ${err.message}`);
+      await this.delete();
+    }
+  }
+
+  // Cleans up the resources associated with the asset inventory, such as removing the ingest pipeline.
+  public async delete() {
+    const { logger } = this.options;
+
+    logger.debug(`Deleting asset inventory`);
+
+    try {
+      logger.debug(`Deleting asset inventory keyword builder pipeline`);
+
+      await deleteKeywordBuilderPipeline({
+        logger,
+        esClient: this.esClient,
+      }).catch((err) => {
+        logger.error('Error on deleting keyword builder pipeline', err);
+      });
+
+      logger.debug(`Deleted asset inventory`);
+      return { deleted: true };
+    } catch (err) {
+      logger.error(`Error deleting asset inventory: ${err.message}`);
+      throw err;
+    }
+  }
+}

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './keyword_builder_ingest_pipeline';

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/keyword_builder_ingest_pipeline.ts

@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
+
+const PIPELINE_ID = 'entity-keyword-builder@platform';
+
+export const buildIngestPipeline = (): IngestProcessorContainer[] => {
+  return [
+    {
+      script: {
+        lang: 'painless',
+        on_failure: [
+          {
+            set: {
+              field: 'error.message',
+              value:
+                'Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}',
+            },
+          },
+        ],
+
+        // There are two layers of language to string scape on this script.
+        // - One is in javascript
+        // - Another one is in painless.
+        //
+        // .e.g, in painless we want the following line:
+        //    entry.getKey().replace("\"", "\\\"");
+        //
+        // To do so we must scape each backslash in javascript, otherwise the backslashes will only scape the next character
+        // and the backslashes won't end up in the painless layer
+        //
+        // The code then becomes:
+        //    entry.getKey().replace("\\"", "\\\\\\"");
+        // That is one extra backslash per backslash (there is no need to scape quotes in the javascript layer)
+        source: `
+            String jsonFromMap(Map map) {
+              StringBuilder json = new StringBuilder("{");
+              boolean first = true;
+              for (entry in map.entrySet()) {
+                  if (!first) {
+                      json.append(",");
+                  }
+                  first = false;
+                  String key = entry.getKey().replace("\\"", "\\\\\\"");
+                  Object value = entry.getValue();
+                  json.append("\\"").append(key).append("\\":");
+                  
+                  if (value instanceof String) {
+                      String escapedValue = ((String) value).replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedValue).append("\\"");
+                  } else if (value instanceof Map) {
+                      json.append(jsonFromMap((Map) value));
+                  } else if (value instanceof List) {
+                      json.append(jsonFromList((List) value));
+                  } else if (value instanceof Boolean || value instanceof Number) {
+                      json.append(value.toString());
+                  } else {
+                      // For other types, treat as string
+                      String escapedValue = value.toString().replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedValue).append("\\"");
+                  }
+              }
+              json.append("}");
+              return json.toString();
+          }
+
+          String jsonFromList(List list) {
+              StringBuilder json = new StringBuilder("[");
+              boolean first = true;
+              for (item in list) {
+                  if (!first) {
+                      json.append(",");
+                  }
+                  first = false;
+                  if (item instanceof String) {
+                      String escapedItem = ((String) item).replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedItem).append("\\"");
+                  } else if (item instanceof Map) {
+                      json.append(jsonFromMap((Map) item));
+                  } else if (item instanceof List) {
+                      json.append(jsonFromList((List) item));
+                  } else if (item instanceof Boolean || item instanceof Number) {
+                      json.append(item.toString());
+                  } else {
+                      // For other types, treat as string
+                      String escapedItem = item.toString().replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedItem).append("\\"");
+                  }
+              }
+              json.append("]");
+              return json.toString();
+          }
+
+          if (ctx.entities?.metadata == null) {
+            return;
+          }
+
+          def keywords = [];
+          for (key in ctx.entities.metadata.keySet()) {
+            def value = ctx.entities.metadata[key];
+            def metadata = jsonFromMap([key: value]);  
+            keywords.add(metadata);
+          }
+
+          ctx['entities']['keyword'] = keywords;
+          `,
+      },
+    },
+    {
+      set: {
+        field: 'event.ingested',
+        value: '{{{_ingest.timestamp}}}',
+      },
+    },
+  ];
+};
+
+// developing the pipeline is a bit tricky, so we have a debug mode
+// set  xpack.securitySolution.entityAnalytics.entityStore.developer.pipelineDebugMode
+// to true to keep the enrich field and the context field in the document to help with debugging.
+export const createKeywordBuilderPipeline = async ({
+  logger,
+  esClient,
+}: {
+  logger: Logger;
+  esClient: ElasticsearchClient;
+}) => {
+  const pipeline = {
+    id: PIPELINE_ID,
+    body: {
+      _meta: {
+        managed_by: 'entity_store',
+        managed: true,
+      },
+      description: `Serialize entities.metadata into a keyword field`,
+      processors: buildIngestPipeline(),
+    },
+  };
+
+  logger.debug(`Attempting to create pipeline: ${JSON.stringify(pipeline)}`);
+
+  await esClient.ingest.putPipeline(pipeline);
+};
+
+export const deleteKeywordBuilderPipeline = ({
+  logger,
+  esClient,
+}: {
+  logger: Logger;
+  esClient: ElasticsearchClient;
+}) => {
+  logger.debug(`Attempting to delete pipeline: ${PIPELINE_ID}`);
+  return esClient.ingest.deletePipeline(
+    {
+      id: PIPELINE_ID,
+    },
+    {
+      ignore: [404],
+    }
+  );
+};

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/routes/delete.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { API_VERSIONS } from '../../../../common/constants';
+import type { AssetInventoryRoutesDeps } from '../types';
+
+export const deleteAssetInventoryRoute = (
+  router: AssetInventoryRoutesDeps['router'],
+  logger: Logger
+) => {
+  router.versioned
+    .delete({
+      access: 'public',
+      path: '/api/asset_inventory/delete',
+      security: {
+        authz: {
+          requiredPrivileges: ['securitySolution'],
+        },
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        // TODO: create validation
+        validate: false,
+      },
+
+      async (context, request, response) => {
+        const siemResponse = buildSiemResponse(response);
+
+        try {
+          const secSol = await context.securitySolution;
+          const body = await secSol.getAssetInventoryClient().delete();
+
+          return response.ok({ body });
+        } catch (e) {
+          logger.error('Error in DeleteEntityEngine:', e);
+          const error = transformError(e);
+          return siemResponse.error({
+            statusCode: error.statusCode,
+            body: error.message,
+          });
+        }
+      }
+    );
+};