[SIEM Migrations] Add missing fields to rule migrations results (#206833

) ## Summary Include all data from the migration process in the translated rule documents, so we are able to display the correct information in the table, allowing us also to sort and filter by these fields. The fields added are: - `integration_ids` -> new field mapped in the index (from `integration_id`), the field is set when we match a prebuilt rule too. - `risk_score` -> new field mapped in the index, the field is set when we match a prebuilt rule and set the default value otherwise. - `severity` -> the field is set when we match a prebuilt rule too. Defaults moved from the UI to the LLM graph result. Next steps: - Take the `risk_score` from the original rule for the custom translated rules - Infer `severity` from the original rule risk_score (and maybe other parameters) for the custom translated rules Other changes - The RuleMigrationSevice has been refactored to take all dependencies (clients, services) from the API context factory. This change makes all dependencies always available within the Rule migration service so we don't need to pass them by parameters in each single operation. - The Prebuilt rule retriever now stores all the prebuilt rules data in memory during the migration, so we can return all the prebuilt rule information when we execute semantic searches. This was necessary to set `rule_id`, `integration_ids`, `severity`, and `risk_score` fields correctly. ## Screenshots ![screenshot](https://github.com/user-attachments/assets/ee85879e-9d37-498c-9803-0fd3850c3cc5) --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Jan 17, 2025 · 7f1e24e · 7f1e24e
1 parent a042747
commit 7f1e24e
Show file tree

Hide file tree

Showing 28 changed files with 168 additions and 147 deletions.
diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts
@@ -60,8 +60,6 @@ export const DEFAULT_TRANSLATION_RISK_SCORE = 21;
 export const DEFAULT_TRANSLATION_SEVERITY: Severity = 'low';
 
 export const DEFAULT_TRANSLATION_FIELDS = {
-  risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
-  severity: DEFAULT_TRANSLATION_SEVERITY,
   from: 'now-360s',
   to: 'now',
   interval: '5m',

diff --git a/...ons/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/...ons/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts
@@ -90,6 +90,10 @@ export const ElasticRule = z.object({
    * The migrated rule severity.
    */
   severity: z.string().optional(),
+  /**
+   * The migrated rule risk_score value, integer between 0 and 100.
+   */
+  risk_score: z.number().optional(),
   /**
    * The translated elastic query.
    */
@@ -103,9 +107,9 @@ export const ElasticRule = z.object({
    */
   prebuilt_rule_id: NonEmptyString.optional(),
   /**
-   * The Elastic integration ID found to be most relevant to the splunk rule.
+   * The IDs of the Elastic integrations suggested to be installed for this rule.
    */
-  integration_id: z.string().optional(),
+  integration_ids: z.array(z.string()).optional(),
   /**
    * The Elastic rule id installed as a result.
    */

diff --git a/...ecurity/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/...ecurity/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml
@@ -72,6 +72,9 @@ components:
         severity:
           type: string
           description: The migrated rule severity.
+        risk_score:
+          type: number
+          description: The migrated rule risk_score value, integer between 0 and 100.
         query:
           type: string
           description: The translated elastic query.
@@ -83,9 +86,11 @@ components:
         prebuilt_rule_id:
           description: The Elastic prebuilt rule id matched.
           $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'
-        integration_id:
-          type: string
-          description: The Elastic integration ID found to be most relevant to the splunk rule.
+        integration_ids:
+          type: array
+          description: The IDs of the Elastic integrations suggested to be installed for this rule.
+          items:
+            type: string
         id:
           description: The Elastic rule id installed as a result.
           $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'

diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/utils.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/utils.ts
@@ -6,14 +6,24 @@
  */
 
 import type { Severity } from '../../api/detection_engine';
-import { DEFAULT_TRANSLATION_FIELDS, DEFAULT_TRANSLATION_SEVERITY } from '../constants';
+import { DEFAULT_TRANSLATION_FIELDS } from '../constants';
 import type { ElasticRule, ElasticRulePartial } from '../model/rule_migration.gen';
 
 export type MigrationPrebuiltRule = ElasticRulePartial &
-  Required<Pick<ElasticRulePartial, 'title' | 'description' | 'prebuilt_rule_id'>>;
+  Required<
+    Pick<
+      ElasticRulePartial,
+      'title' | 'description' | 'prebuilt_rule_id' | 'severity' | 'risk_score'
+    >
+  >;
 
 export type MigrationCustomRule = ElasticRulePartial &
-  Required<Pick<ElasticRulePartial, 'title' | 'description' | 'query' | 'query_language'>>;
+  Required<
+    Pick<
+      ElasticRulePartial,
+      'title' | 'description' | 'query' | 'query_language' | 'severity' | 'risk_score'
+    >
+  >;
 
 export const isMigrationPrebuiltRule = (rule?: ElasticRule): rule is MigrationPrebuiltRule =>
   !!(rule?.title && rule?.description && rule?.prebuilt_rule_id);
@@ -33,8 +43,8 @@ export const convertMigrationCustomRuleToSecurityRulePayload = (
     name: rule.title,
     description: rule.description,
     enabled,
-
+    severity: rule.severity as Severity,
+    risk_score: rule.risk_score,
     ...DEFAULT_TRANSLATION_FIELDS,
-    severity: (rule.severity as Severity) ?? DEFAULT_TRANSLATION_SEVERITY,
   };
 };
diff --git a/...y/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx b/...y/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx
@@ -262,24 +262,18 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
         if (!isLoading && ruleMigrations.length) {
           const ruleMigration = ruleMigrations.find((item) => item.id === ruleId);
           let matchedPrebuiltRule: RuleResponse | undefined;
-          const relatedIntegrations: RelatedIntegration[] = [];
+          let relatedIntegrations: RelatedIntegration[] = [];
           if (ruleMigration) {
             // Find matched prebuilt rule if any and prioritize its installed version
-            const matchedPrebuiltRuleVersion = ruleMigration.elastic_rule?.prebuilt_rule_id
-              ? prebuiltRules[ruleMigration.elastic_rule.prebuilt_rule_id]
-              : undefined;
-            matchedPrebuiltRule =
-              matchedPrebuiltRuleVersion?.current ?? matchedPrebuiltRuleVersion?.target;
+            const prebuiltRuleId = ruleMigration.elastic_rule?.prebuilt_rule_id;
+            const prebuiltRuleVersions = prebuiltRuleId ? prebuiltRules[prebuiltRuleId] : undefined;
+            matchedPrebuiltRule = prebuiltRuleVersions?.current ?? prebuiltRuleVersions?.target;
 
-            if (integrations) {
-              if (matchedPrebuiltRule?.related_integrations) {
-                relatedIntegrations.push(...matchedPrebuiltRule.related_integrations);
-              } else if (ruleMigration.elastic_rule?.integration_id) {
-                const integration = integrations[ruleMigration.elastic_rule.integration_id];
-                if (integration) {
-                  relatedIntegrations.push(integration);
-                }
-              }
+            const integrationIds = ruleMigration.elastic_rule?.integration_ids;
+            if (integrations && integrationIds) {
+              relatedIntegrations = integrationIds
+                .map((integrationId) => integrations[integrationId])
+                .filter((integration) => integration != null);
             }
           }
           return { ruleMigration, matchedPrebuiltRule, relatedIntegrations, isIntegrationsLoading };

diff --git a/...ity_solution/public/siem_migrations/rules/components/rules_table_columns/integrations.tsx b/...ity_solution/public/siem_migrations/rules/components/rules_table_columns/integrations.tsx
@@ -21,7 +21,7 @@ export const createIntegrationsColumn = ({
   ) => { relatedIntegrations?: RelatedIntegration[]; isIntegrationsLoading?: boolean } | undefined;
 }): TableColumn => {
   return {
-    field: 'elastic_rule.integration_id',
+    field: 'elastic_rule.integration_ids',
     name: i18n.COLUMN_INTEGRATIONS,
     render: (_, rule: RuleMigration) => {
       const migrationRuleData = getMigrationRuleData(rule.id);

diff --git a/...urity_solution/public/siem_migrations/rules/components/rules_table_columns/risk_score.tsx b/...urity_solution/public/siem_migrations/rules/components/rules_table_columns/risk_score.tsx
@@ -8,22 +8,17 @@
 import React from 'react';
 import { EuiText } from '@elastic/eui';
 import { type RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
-import {
-  DEFAULT_TRANSLATION_RISK_SCORE,
-  SiemMigrationStatus,
-} from '../../../../../common/siem_migrations/constants';
+import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants';
 import * as i18n from './translations';
 import { COLUMN_EMPTY_VALUE, type TableColumn } from './constants';
 
 export const createRiskScoreColumn = (): TableColumn => {
   return {
-    field: 'risk_score',
+    field: 'elastic_rule.risk_score',
     name: i18n.COLUMN_RISK_SCORE,
-    render: (_, rule: RuleMigration) => (
+    render: (riskScore, rule: RuleMigration) => (
       <EuiText data-test-subj="riskScore" size="s">
-        {rule.status === SiemMigrationStatus.FAILED
-          ? COLUMN_EMPTY_VALUE
-          : DEFAULT_TRANSLATION_RISK_SCORE}
+        {rule.status === SiemMigrationStatus.FAILED ? COLUMN_EMPTY_VALUE : riskScore}
       </EuiText>
     ),
     sortable: true,

diff --git a/...ecurity_solution/public/siem_migrations/rules/components/rules_table_columns/severity.tsx b/...ecurity_solution/public/siem_migrations/rules/components/rules_table_columns/severity.tsx
@@ -8,10 +8,7 @@
 import React from 'react';
 import type { Severity } from '@kbn/securitysolution-io-ts-alerting-types';
 import { type RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
-import {
-  DEFAULT_TRANSLATION_SEVERITY,
-  SiemMigrationStatus,
-} from '../../../../../common/siem_migrations/constants';
+import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants';
 import { SeverityBadge } from '../../../../common/components/severity_badge';
 import { COLUMN_EMPTY_VALUE, type TableColumn } from './constants';
 import * as i18n from './translations';
@@ -20,7 +17,7 @@ export const createSeverityColumn = (): TableColumn => {
   return {
     field: 'elastic_rule.severity',
     name: i18n.COLUMN_SEVERITY,
-    render: (value: Severity = DEFAULT_TRANSLATION_SEVERITY, rule: RuleMigration) =>
+    render: (value: Severity, rule: RuleMigration) =>
       rule.status === SiemMigrationStatus.FAILED ? (
         <>{COLUMN_EMPTY_VALUE}</>
       ) : (

diff --git a/...olutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/...olutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts
@@ -52,10 +52,6 @@ export const registerSiemRuleMigrationsStartRoute = (
             const ctx = await context.resolve(['core', 'actions', 'alerting', 'securitySolution']);
 
             const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient();
-            const inferenceClient = ctx.securitySolution.getInferenceClient();
-            const actionsClient = ctx.actions.getActionsClient();
-            const soClient = ctx.core.savedObjects.client;
-            const rulesClient = await ctx.alerting.getRulesClient();
 
             if (retry) {
               const { updated } = await ruleMigrationsClient.task.updateToRetry(
@@ -78,10 +74,6 @@ export const registerSiemRuleMigrationsStartRoute = (
               migrationId,
               connectorId,
               invocationConfig,
-              inferenceClient,
-              actionsClient,
-              soClient,
-              rulesClient,
             });
 
             if (!exists) {

diff --git a/...curity_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts b/...curity_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts
@@ -18,7 +18,7 @@ import type {
   Logger,
 } from '@kbn/core/server';
 import assert from 'assert';
-import type { Stored } from '../types';
+import type { Stored, SiemRuleMigrationsClientDependencies } from '../types';
 import type { IndexNameProvider } from './rule_migrations_data_client';
 
 const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const;
@@ -30,7 +30,8 @@ export class RuleMigrationsDataBaseClient {
     protected getIndexName: IndexNameProvider,
     protected currentUser: AuthenticatedUser,
     protected esScopedClient: IScopedClusterClient,
-    protected logger: Logger
+    protected logger: Logger,
+    protected dependencies: SiemRuleMigrationsClientDependencies
   ) {
     this.esClient = esScopedClient.asInternalUser;
   }

diff --git a/...ns/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts b/...ns/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts
@@ -6,12 +6,12 @@
  */
 
 import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server';
-import type { PackageService } from '@kbn/fleet-plugin/server';
 import { RuleMigrationsDataIntegrationsClient } from './rule_migrations_data_integrations_client';
 import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_prebuilt_rules_client';
 import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client';
 import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client';
 import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client';
+import type { SiemRuleMigrationsClientDependencies } from '../types';
 import type { AdapterId } from './rule_migrations_data_service';
 
 export type IndexNameProvider = () => Promise<string>;
@@ -29,32 +29,35 @@ export class RuleMigrationsDataClient {
     currentUser: AuthenticatedUser,
     esScopedClient: IScopedClusterClient,
     logger: Logger,
-    packageService?: PackageService
+    dependencies: SiemRuleMigrationsClientDependencies
   ) {
     this.rules = new RuleMigrationsDataRulesClient(
       indexNameProviders.rules,
       currentUser,
       esScopedClient,
-      logger
+      logger,
+      dependencies
     );
     this.resources = new RuleMigrationsDataResourcesClient(
       indexNameProviders.resources,
       currentUser,
       esScopedClient,
-      logger
+      logger,
+      dependencies
     );
     this.integrations = new RuleMigrationsDataIntegrationsClient(
       indexNameProviders.integrations,
       currentUser,
       esScopedClient,
       logger,
-      packageService
+      dependencies
     );
     this.prebuiltRules = new RuleMigrationsDataPrebuiltRulesClient(
       indexNameProviders.prebuiltrules,
       currentUser,
       esScopedClient,
-      logger
+      logger,
+      dependencies
     );
     this.lookups = new RuleMigrationsDataLookupsClient(currentUser, esScopedClient, logger);
   }

diff --git a/...olution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts b/...olution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts
@@ -5,15 +5,12 @@
  * 2.0.
  */
 
-import type { PackageService } from '@kbn/fleet-plugin/server';
-import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server';
 import type { PackageList } from '@kbn/fleet-plugin/common';
 import type { RuleMigrationIntegration } from '../types';
 import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
 
 /* This will be removed once the package registry changes is performed */
 import integrationsFile from './integrations_temp.json';
-import type { IndexNameProvider } from './rule_migrations_data_client';
 
 /* The minimum score required for a integration to be considered correct, might need to change this later */
 const MIN_SCORE = 40 as const;
@@ -26,18 +23,8 @@ const INTEGRATIONS = integrationsFile as RuleMigrationIntegration[];
  * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed.
  */
 export class RuleMigrationsDataIntegrationsClient extends RuleMigrationsDataBaseClient {
-  constructor(
-    getIndexName: IndexNameProvider,
-    currentUser: AuthenticatedUser,
-    esScopedClient: IScopedClusterClient,
-    logger: Logger,
-    private packageService?: PackageService
-  ) {
-    super(getIndexName, currentUser, esScopedClient, logger);
-  }
-
   async getIntegrationPackages(): Promise<PackageList | undefined> {
-    return this.packageService?.asInternalUser.getPackages();
+    return this.dependencies.packageService?.asInternalUser.getPackages();
   }
 
   /** Indexes an array of integrations to be used with ELSER semantic search queries */

diff --git a/...ution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts b/...ution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts
@@ -5,19 +5,15 @@
  * 2.0.
  */
 
-import type { RulesClient } from '@kbn/alerting-plugin/server';
-import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
+import type { RuleVersions } from '../../../detection_engine/prebuilt_rules/logic/diff/calculate_rule_diff';
 import { createPrebuiltRuleAssetsClient } from '../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client';
 import { createPrebuiltRuleObjectsClient } from '../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client';
 import { fetchRuleVersionsTriad } from '../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad';
 import type { RuleMigrationPrebuiltRule } from '../types';
 import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
 
-interface RetrievePrebuiltRulesParams {
-  soClient: SavedObjectsClientContract;
-  rulesClient: RulesClient;
-}
-
+export type { RuleVersions };
+export type PrebuildRuleVersionsMap = Map<string, RuleVersions>;
 /* The minimum score required for a integration to be considered correct, might need to change this later */
 const MIN_SCORE = 40 as const;
 /* The number of integrations the RAG will return, sorted by score */
@@ -29,17 +25,16 @@ const RETURNED_RULES = 5 as const;
 const BULK_MAX_SIZE = 500 as const;
 
 export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBaseClient {
-  /** Indexes an array of integrations to be used with ELSER semantic search queries */
-  async create({ soClient, rulesClient }: RetrievePrebuiltRulesParams): Promise<void> {
-    const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient);
-    const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient);
-
-    const ruleVersionsMap = await fetchRuleVersionsTriad({
-      ruleAssetsClient,
-      ruleObjectsClient,
-    });
+  async getRuleVersionsMap(): Promise<PrebuildRuleVersionsMap> {
+    const ruleAssetsClient = createPrebuiltRuleAssetsClient(this.dependencies.savedObjectsClient);
+    const ruleObjectsClient = createPrebuiltRuleObjectsClient(this.dependencies.rulesClient);
+    return fetchRuleVersionsTriad({ ruleAssetsClient, ruleObjectsClient });
+  }
 
+  /** Indexes an array of integrations to be used with ELSER semantic search queries */
+  async populate(ruleVersionsMap: PrebuildRuleVersionsMap): Promise<void> {
     const filteredRules: RuleMigrationPrebuiltRule[] = [];
+
     ruleVersionsMap.forEach((ruleVersions) => {
       const rule = ruleVersions.target || ruleVersions.current;
       if (rule) {
@@ -50,7 +45,6 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas
         filteredRules.push({
           rule_id: rule.rule_id,
           name: rule.name,
-          installed_rule_id: ruleVersions.current?.id,
           description: rule.description,
           elser_embedding: `${rule.name} - ${rule.description}`,
           ...(mitreAttackIds?.length && { mitre_attack_ids: mitreAttackIds }),
@@ -87,10 +81,7 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas
   }
 
   /** Based on a LLM generated semantic string, returns the 5 best results with a score above 40 */
-  async retrieveRules(
-    semanticString: string,
-    techniqueIds: string
-  ): Promise<RuleMigrationPrebuiltRule[]> {
+  async search(semanticString: string, techniqueIds: string): Promise<RuleMigrationPrebuiltRule[]> {
     const index = await this.getIndexName();
     const query = {
       bool: {
@@ -126,7 +117,7 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas
         size: RETURNED_RULES,
         min_score: MIN_SCORE,
       })
-      .then(this.processResponseHits.bind(this))
+      .then((response) => this.processResponseHits(response))
       .catch((error) => {
         this.logger.error(`Error querying prebuilt rule details for ELSER: ${error.message}`);
         throw error;