From eedc917b923a613babccaa57c328aa67ed937994 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 8 Jan 2025 20:26:22 +0100 Subject: [PATCH] [SIEM migrations] Implement ES|QL lookups and other fixes (#204960) ## Summary Adds support for ES|QL native LOOKUP JOIN operators for Splunk lookups. - Lookups import changes: - Stores the lookups files as indices using `lookup_` pattern (queries fail if the name contains `-`) - Indexes the lookups content data without duplicates (supports csv and json/ndjson) - Stores the lookup index name as the resource content that is passed to the translation agent - Fixes bug with `_lookup` suffix in the names coming from Splunk: queries use the `_lookup` suffix, but files in the. lookup editor don't have it) - Lookups translation changes: - Prompt for the `inline_query` node updated to support lookups, replacing the splunk lookup name with the new Elastic lookup index name. Placeholders for missing macros/lookups are now added in this node instead of the `translate_query` node. - Prompt for ES|QL translation updated to convert LOOKUP syntax and ignore macro/lookups placeholders - Other improvements on the agent graph: - All rule migration nodes in the graph now generate a "summary" explaining the reasoning behind each decision of the LLM, they are displayed in the comments section of each rule translation. - The inline query node was moved inside the translation sub-graph since it's only needed there. - Validation now is executed without placeholders, preventing it from running all the iterations without being able to fix it. - A deterministic node was added at the end to set the translation result and ensure minimum defaults are met. - Avoid inline_query LLM calls when a prebuilt rule matched or when the Splunk query is unsupported - Avoid prebuilt_rule matching LLM calls when no prebuilt rule is retrieved from the semantic search. - Avoid integration matching LLM calls when no integration is retrieved from the semantic search. - Other fixes - Fixes bug which was setting translation `FULL` when we missed the integration and index pattern (logs-*). Changed to `PARTIAL` - Fixes bug where the description was missing for custom translated rules, we now fallback to the splunk rule title if the description is missing - Added summary comment for prebuilt rule matching ### Screenshots #### New summary comments: ##### Prebuilt rule matching: - matching ![prebuilt matching](https://github.com/user-attachments/assets/63c86cd9-f06d-4664-89db-2fa36bdff838) - not matching ![prebuilt not matching](https://github.com/user-attachments/assets/3bd6bf7b-0564-416b-9b16-700b346dd95e) ##### Query inlining summary: ![Inlining summary](https://github.com/user-attachments/assets/6bf88e61-e269-4d4b-a01f-1a009c622982) ##### Integration matching: - matching: ![integration matching](https://github.com/user-attachments/assets/a77e01d9-3a2e-4629-a575-905b6995d55d) - not matching ![integration no match](https://github.com/user-attachments/assets/ce21b0e4-e3a3-4e2c-b6d2-2114f8a7f146) ##### ES|QL translation ![translation](https://github.com/user-attachments/assets/d0dd0879-c9ce-44f3-aa44-e3b724cd5898) Needs manual translation reason: ![unsupported](https://github.com/user-attachments/assets/45fd73b2-5fc0-4504-99bd-e263c01c3a11) #### Lookups UI: ![UI](https://github.com/user-attachments/assets/c7271e47-b0a5-4b31-b5cf-d99285e108bf) Lookup index example: ![lookup index](https://github.com/user-attachments/assets/88c275b8-96dd-4770-804b-164b3e3d4f8f) Translation ![lookup translation](https://github.com/user-attachments/assets/647a6003-e930-407b-aaf2-02bc1ea95de6) #### Test data [rules.json](https://github.com/user-attachments/files/18208912/rules.json) [all_macros.json](https://github.com/user-attachments/files/18208914/all_macros.json) [lookups.zip](https://github.com/user-attachments/files/18208904/lookups.zip) (uncompress before uploading) --- .../index-adapter/src/field_maps/types.ts | 13 +- .../common/siem_migrations/constants.ts | 2 - .../model/api/rules/rule_migration.gen.ts | 3 +- .../api/rules/rule_migration.schema.yaml | 2 +- .../model/rule_migration.gen.ts | 28 +++- .../model/rule_migration.schema.yaml | 24 ++- .../siem_migrations/rules/resources/index.ts | 31 ++-- .../splunk/splunk_identifier.test.ts | 30 ++-- .../resources/splunk/splunk_identifier.ts | 20 +-- .../siem_migrations/rules/resources/types.ts | 12 +- .../data_input_flyout/data_input_flyout.tsx | 6 +- .../steps/lookups/lookups_data_input.tsx | 7 +- .../lookups_file_upload.tsx | 2 +- .../sub_steps/missing_lookups_list/index.tsx | 11 +- .../missing_lookups_list.tsx | 36 ++-- .../components/data_input_flyout/types.ts | 4 +- .../migration_ready_panel.tsx | 4 +- .../upload_missing_panel.tsx | 4 +- .../hooks/use_get_missing_resources.ts | 4 +- .../rules/service/rule_migrations_service.ts | 4 +- .../rules/api/resources/missing.ts | 4 +- .../rules/api/resources/upsert.ts | 13 +- .../siem_migrations/rules/api/util/lookups.ts | 107 ++++++++++++ .../rules/data/rule_migrations_data_client.ts | 19 ++- ...ule_migrations_data_integrations_client.ts | 24 +-- .../rule_migrations_data_lookups_client.ts | 67 ++++++++ ...e_migrations_data_prebuilt_rules_client.ts | 2 +- .../rule_migrations_data_resources_client.ts | 6 +- .../data/rule_migrations_data_service.test.ts | 63 ++++--- .../data/rule_migrations_data_service.ts | 91 +++++++--- .../rules/data/rule_migrations_field_maps.ts | 11 +- .../siem_rule_migrations_service.test.ts | 2 +- .../rules/siem_rule_migrations_service.ts | 4 +- .../siem_migrations/rules/task/agent/graph.ts | 18 +- .../match_prebuilt_rule.ts | 23 ++- .../nodes/match_prebuilt_rule/prompts.ts | 27 ++- .../nodes/process_query/process_query.ts | 43 ----- .../task/agent/nodes/process_query/prompts.ts | 117 ------------- .../siem_migrations/rules/task/agent/state.ts | 4 +- .../agent/sub_graphs/translate_rule/graph.ts | 29 +++- .../nodes/ecs_mapping/ecs_mapping.ts | 5 +- .../filter_index_patterns.ts | 40 ----- .../nodes/inline_query}/index.ts | 2 +- .../nodes/inline_query/inline_query.ts | 68 ++++++++ .../nodes/inline_query/prompts.ts | 156 ++++++++++++++++++ .../nodes/retrieve_integrations/prompts.ts | 30 +++- .../retrieve_integrations.ts | 25 ++- .../nodes/translate_rule/prompts.ts | 13 +- .../nodes/translate_rule/translate_rule.ts | 19 +-- .../index.ts | 2 +- .../translation_result/translation_result.ts | 42 +++++ .../nodes/validation/validation.ts | 36 ++-- .../agent/sub_graphs/translate_rule/state.ts | 12 +- .../retrievers/integration_retriever.test.ts | 5 +- .../task/retrievers/integration_retriever.ts | 19 +-- .../retrievers/prebuilt_rules_retriever.ts | 26 +-- .../retrievers/rule_migrations_retriever.ts | 19 +-- .../rule_resource_retriever.test.ts | 24 +-- .../retrievers/rule_resource_retriever.ts | 20 +-- .../rules/task/rule_migrations_task_client.ts | 19 ++- .../rules/task/util/comments.ts | 11 ++ .../server/lib/siem_migrations/rules/types.ts | 6 +- 62 files changed, 939 insertions(+), 581 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts rename x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/{nodes/process_query => sub_graphs/translate_rule/nodes/inline_query}/index.ts (82%) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts rename x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/{filter_index_patterns => translation_result}/index.ts (78%) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts diff --git a/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts b/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts index 9c96fa2d796f6..1906f899e9131 100644 --- a/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts +++ b/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts @@ -61,11 +61,14 @@ export type FieldMap = Record< // This utility type flattens all the keys of a schema object and its nested objects as a union type. // Its purpose is to ensure that the FieldMap keys are always in sync with the schema object. // It assumes all optional fields of the schema are required in the field map, they can always be omitted from the resulting type. -export type SchemaFieldMapKeys< - T extends Record, - Key = keyof T -> = Key extends string - ? NonNullable extends Record +// We need to use any to avoid TS errors since interfaces do not satisfy Record, but they do satisfy Record. +/* eslint-disable @typescript-eslint/no-explicit-any */ +export type SchemaFieldMapKeys, Key = keyof T> = Key extends string + ? NonNullable extends any[] + ? NonNullable extends Array> + ? `${Key}` | `${Key}.${SchemaFieldMapKeys[number]>}` + : `${Key}` + : NonNullable extends Record ? `${Key}` | `${Key}.${SchemaFieldMapKeys>}` : `${Key}` : never; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts index 019766d96e78f..ee3059c242f7f 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts @@ -62,5 +62,3 @@ export const DEFAULT_TRANSLATION_FIELDS = { to: 'now', interval: '5m', } as const; - -export const EMPTY_RESOURCE_PLACEHOLDER = ''; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts index 661ef1d1cbb4b..b98d38cc08f36 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts @@ -27,6 +27,7 @@ import { RuleMigrationResourceData, RuleMigrationResourceType, RuleMigrationResource, + RuleMigrationResourceBase, } from '../../rule_migration.gen'; import { RelatedIntegration } from '../../../../api/detection_engine/model/rule_schema/common_attributes.gen'; import { NonEmptyString } from '../../../../api/model/primitives.gen'; @@ -147,7 +148,7 @@ export type GetRuleMigrationResourcesMissingRequestParamsInput = z.input< export type GetRuleMigrationResourcesMissingResponse = z.infer< typeof GetRuleMigrationResourcesMissingResponse >; -export const GetRuleMigrationResourcesMissingResponse = z.array(RuleMigrationResourceData); +export const GetRuleMigrationResourcesMissingResponse = z.array(RuleMigrationResourceBase); export type GetRuleMigrationStatsRequestParams = z.infer; export const GetRuleMigrationStatsRequestParams = z.object({ diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml index 823e2b32d2fce..d51225cef8969 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml @@ -583,4 +583,4 @@ paths: type: array description: The identified resources missing items: - $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData' + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceBase' diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 27db56aedb451..c55511f83a3ae 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -356,35 +356,49 @@ export const UpdateRuleMigrationData = z.object({ * The type of the rule migration resource. */ export type RuleMigrationResourceType = z.infer; -export const RuleMigrationResourceType = z.enum(['macro', 'list']); +export const RuleMigrationResourceType = z.enum(['macro', 'lookup']); export type RuleMigrationResourceTypeEnum = typeof RuleMigrationResourceType.enum; export const RuleMigrationResourceTypeEnum = RuleMigrationResourceType.enum; /** - * The rule migration resource data provided by the vendor. + * The rule migration resource basic information. */ -export type RuleMigrationResourceData = z.infer; -export const RuleMigrationResourceData = z.object({ +export type RuleMigrationResourceBase = z.infer; +export const RuleMigrationResourceBase = z.object({ type: RuleMigrationResourceType, /** * The resource name identifier. */ name: z.string(), +}); + +export type RuleMigrationResourceContent = z.infer; +export const RuleMigrationResourceContent = z.object({ /** - * The resource content value. + * The resource content value. Can be an empty string. */ - content: z.string().optional(), + content: z.string(), /** * The resource arbitrary metadata. */ metadata: z.object({}).optional(), }); +/** + * The rule migration resource data. + */ +export type RuleMigrationResourceData = z.infer; +export const RuleMigrationResourceData = RuleMigrationResourceBase.merge( + RuleMigrationResourceContent +); + /** * The rule migration resource document object. */ export type RuleMigrationResource = z.infer; -export const RuleMigrationResource = RuleMigrationResourceData.merge( +export const RuleMigrationResource = RuleMigrationResourceBase.merge( + RuleMigrationResourceContent.partial() +).merge( z.object({ /** * The rule resource migration id diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index f3a85b3e37443..0361c3a01d0cf 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -327,11 +327,11 @@ components: description: The type of the rule migration resource. enum: - macro # Reusable part a query that can be customized and called from multiple rules - - list # A list of values that can be used inside queries reused in different rules + - lookup # A list of values that can be used inside queries as data enrichment or data source - RuleMigrationResourceData: + RuleMigrationResourceBase: type: object - description: The rule migration resource data provided by the vendor. + description: The rule migration resource basic information. required: - type - name @@ -341,17 +341,31 @@ components: name: type: string description: The resource name identifier. + + RuleMigrationResourceContent: + type: object + required: + - content + properties: content: type: string - description: The resource content value. + description: The resource content value. Can be an empty string. metadata: type: object description: The resource arbitrary metadata. + RuleMigrationResourceData: + description: The rule migration resource data. + allOf: + - $ref: '#/components/schemas/RuleMigrationResourceBase' + - $ref: '#/components/schemas/RuleMigrationResourceContent' + RuleMigrationResource: description: The rule migration resource document object. allOf: - - $ref: '#/components/schemas/RuleMigrationResourceData' + - $ref: '#/components/schemas/RuleMigrationResourceBase' + - $ref: '#/components/schemas/RuleMigrationResourceContent' + x-modify: partial - type: object required: - id diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts index 8ec7adf050bf3..4b6472eb0e55c 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts @@ -9,8 +9,9 @@ import type { OriginalRule, OriginalRuleVendor, RuleMigrationResourceData, + RuleMigrationResourceBase, } from '../../model/rule_migration.gen'; -import type { ResourceIdentifiers, RuleResource } from './types'; +import type { ResourceIdentifiers } from './types'; import { splResourceIdentifiers } from './splunk'; const ruleResourceIdentifiers: Record = { @@ -29,48 +30,48 @@ export class ResourceIdentifier { this.identifiers = ruleResourceIdentifiers[vendor]; } - public fromOriginalRule(originalRule: OriginalRule): RuleResource[] { + public fromOriginalRule(originalRule: OriginalRule): RuleMigrationResourceBase[] { return this.identifiers.fromOriginalRule(originalRule); } - public fromResource(resource: RuleMigrationResourceData): RuleResource[] { + public fromResource(resource: RuleMigrationResourceData): RuleMigrationResourceBase[] { return this.identifiers.fromResource(resource); } - public fromOriginalRules(originalRules: OriginalRule[]): RuleResource[] { - const lists = new Set(); + public fromOriginalRules(originalRules: OriginalRule[]): RuleMigrationResourceBase[] { + const lookups = new Set(); const macros = new Set(); originalRules.forEach((rule) => { const resources = this.identifiers.fromOriginalRule(rule); resources.forEach((resource) => { if (resource.type === 'macro') { macros.add(resource.name); - } else if (resource.type === 'list') { - lists.add(resource.name); + } else if (resource.type === 'lookup') { + lookups.add(resource.name); } }); }); return [ - ...Array.from(macros).map((name) => ({ type: 'macro', name })), - ...Array.from(lists).map((name) => ({ type: 'list', name })), + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), ]; } - public fromResources(resources: RuleMigrationResourceData[]): RuleResource[] { - const lists = new Set(); + public fromResources(resources: RuleMigrationResourceData[]): RuleMigrationResourceBase[] { + const lookups = new Set(); const macros = new Set(); resources.forEach((resource) => { this.identifiers.fromResource(resource).forEach((identifiedResource) => { if (identifiedResource.type === 'macro') { macros.add(identifiedResource.name); - } else if (identifiedResource.type === 'list') { - lists.add(identifiedResource.name); + } else if (identifiedResource.type === 'lookup') { + lookups.add(identifiedResource.name); } }); }); return [ - ...Array.from(macros).map((name) => ({ type: 'macro', name })), - ...Array.from(lists).map((name) => ({ type: 'list', name })), + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), ]; } } diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts index 5d144e5b8a38f..36b4adc2769e4 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts @@ -47,9 +47,9 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ - { type: 'list', name: 'my_lookup_table' }, - { type: 'list', name: 'other_lookup_list' }, - { type: 'list', name: 'third_lookup' }, + { type: 'lookup', name: 'my_lookup_table' }, + { type: 'lookup', name: 'other_lookup_list' }, + { type: 'lookup', name: 'third' }, ]); }); @@ -60,9 +60,9 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, - { type: 'list', name: 'other_lookup_list' }, - { type: 'list', name: 'third_lookup' }, + { type: 'lookup', name: 'my_lookup_table' }, + { type: 'lookup', name: 'other_lookup_list' }, + { type: 'lookup', name: 'third' }, ]); }); @@ -72,11 +72,11 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ - { type: 'list', name: 'my_lookup_1' }, - { type: 'list', name: 'my_lookup_2' }, - { type: 'list', name: 'my_lookup_3' }, - { type: 'list', name: 'my_lookup_4' }, - { type: 'list', name: 'my_lookup_5' }, + { type: 'lookup', name: 'my_lookup_1' }, + { type: 'lookup', name: 'my_lookup_2' }, + { type: 'lookup', name: 'my_lookup_3' }, + { type: 'lookup', name: 'my_lookup_4' }, + { type: 'lookup', name: 'my_lookup_5' }, ]); }); @@ -96,7 +96,7 @@ describe('splResourceIdentifier', () => { { type: 'macro', name: 'macro_one' }, { type: 'macro', name: 'my_lookup_table' }, { type: 'macro', name: 'third_macro' }, - { type: 'list', name: 'real_lookup_list' }, + { type: 'lookup', name: 'real_lookup_list' }, ]); }); @@ -107,7 +107,7 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, + { type: 'lookup', name: 'my_lookup_table' }, ]); }); @@ -118,7 +118,7 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, + { type: 'lookup', name: 'my_lookup_table' }, ]); }); @@ -129,7 +129,7 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, + { type: 'lookup', name: 'my_lookup_table' }, ]); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts index 2ecc43321b11f..90856896193dc 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts @@ -11,17 +11,17 @@ * Please make sure to test all regular expressions them before using them. * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker */ +import type { RuleMigrationResourceBase } from '../../../model/rule_migration.gen'; +import type { ResourceIdentifier } from '../types'; -import type { ResourceIdentifier, RuleResource } from '../types'; - -const listRegex = /\b(?:lookup)\s+([\w-]+)\b/g; // Captures only the lookup name +const lookupRegex = /\b(?:lookup)\s+([\w-]+)\b/g; // Captures only the lookup name const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments export const splResourceIdentifier: ResourceIdentifier = (input) => { - // sanitize the query to avoid mismatching macro and list names inside comments or literal strings + // sanitize the query to avoid mismatching macro and lookup names inside comments or literal strings const sanitizedInput = sanitizeInput(input); - const resources: RuleResource[] = []; + const resources: RuleMigrationResourceBase[] = []; let macroMatch; while ((macroMatch = macrosRegex.exec(sanitizedInput)) !== null) { const macroName = macroMatch[1] as string; @@ -31,17 +31,17 @@ export const splResourceIdentifier: ResourceIdentifier = (input) => { resources.push({ type: 'macro', name: macroWithArgs }); } - let listMatch; - while ((listMatch = listRegex.exec(sanitizedInput)) !== null) { - resources.push({ type: 'list', name: listMatch[1] }); + let lookupMatch; + while ((lookupMatch = lookupRegex.exec(sanitizedInput)) !== null) { + resources.push({ type: 'lookup', name: lookupMatch[1].replace(/_lookup$/, '') }); } return resources; }; -// Comments should be removed before processing the query to avoid matching macro and list names inside them +// Comments should be removed before processing the query to avoid matching macro and lookup names inside them const commentRegex = /```.*?```/g; -// Literal strings should be replaced with a placeholder to avoid matching macro and list names inside them +// Literal strings should be replaced with a placeholder to avoid matching macro and lookup names inside them const doubleQuoteStrRegex = /".*?"/g; const singleQuoteStrRegex = /'.*?'/g; // lookup operator can have modifiers like local=true or update=false before the lookup name, we need to remove them diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts index 70c6e3b72124f..db4df43cc3df6 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts @@ -7,17 +7,13 @@ import type { OriginalRule, + RuleMigrationResourceBase, RuleMigrationResourceData, - RuleMigrationResourceType, } from '../../model/rule_migration.gen'; -export interface RuleResource { - type: RuleMigrationResourceType; - name: string; -} -export type ResourceIdentifier = (input: string) => RuleResource[]; +export type ResourceIdentifier = (input: string) => RuleMigrationResourceBase[]; export interface ResourceIdentifiers { - fromOriginalRule: (originalRule: OriginalRule) => RuleResource[]; - fromResource: (resource: RuleMigrationResourceData) => RuleResource[]; + fromOriginalRule: (originalRule: OriginalRule) => RuleMigrationResourceBase[]; + fromResource: (resource: RuleMigrationResourceData) => RuleMigrationResourceBase[]; } diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx index 9062e3a6b21e8..4f8e73f43f6c3 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx @@ -19,7 +19,7 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import type { - RuleMigrationResourceData, + RuleMigrationResourceBase, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { RulesDataInput } from './steps/rules/rules_data_input'; @@ -60,12 +60,12 @@ export const MigrationDataInputFlyout = React.memo { + (missingResources: RuleMigrationResourceBase[]) => { const newMissingResourcesIndexed = missingResources.reduce( (acc, { type, name }) => { if (type === 'macro') { acc.macros.push(name); - } else if (type === 'list') { + } else if (type === 'lookup') { acc.lookups.push(name); } return acc; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx index a8fca750ce5da..cfe88caafc652 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx @@ -15,7 +15,6 @@ import { EuiTitle, } from '@elastic/eui'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import { EMPTY_RESOURCE_PLACEHOLDER } from '../../../../../../../common/siem_migrations/constants'; import type { RuleMigrationResourceData, RuleMigrationTaskStats, @@ -101,14 +100,12 @@ export const LookupsDataInputSubSteps = React.memo((lookups) => { setUploadedLookups((prevUploadedLookups) => ({ ...prevUploadedLookups, - ...Object.fromEntries( - lookups.map((lookup) => [lookup.name, lookup.content ?? EMPTY_RESOURCE_PLACEHOLDER]) - ), + ...Object.fromEntries(lookups.map((lookup) => [lookup.name, lookup.content])), })); }, []); useEffect(() => { - if (missingLookups.every((lookupName) => uploadedLookups[lookupName])) { + if (missingLookups.every((lookupName) => uploadedLookups[lookupName] != null)) { setSubStep(END); onAllLookupsCreated(); } diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx index 6ea9562f24cce..0dc05493f7469 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx @@ -77,7 +77,7 @@ export const LookupsFileUpload = React.memo( } const name = file.name.replace(/\.[^/.]+$/, '').trim(); - resolve({ type: 'list', name, content }); + resolve({ type: 'lookup', name, content }); }; const handleReaderError = function () { diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx index ae1dbc0a03b3c..4109c2906203f 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx @@ -7,7 +7,6 @@ import React, { useCallback, useMemo } from 'react'; import type { EuiStepProps, EuiStepStatus } from '@elastic/eui'; -import { EMPTY_RESOURCE_PLACEHOLDER } from '../../../../../../../../../common/siem_migrations/constants'; import { useUpsertResources } from '../../../../../../service/hooks/use_upsert_resources'; import type { RuleMigrationTaskStats } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; import type { UploadedLookups, AddUploadedLookups } from '../../lookups_data_input'; @@ -32,11 +31,11 @@ export const useMissingLookupsListStep = ({ }: MissingLookupsListStepProps): EuiStepProps => { const { upsertResources, isLoading, error } = useUpsertResources(addUploadedLookups); - const clearLookup = useCallback( + const omitLookup = useCallback( (lookupName: string) => { - upsertResources(migrationStats.id, [ - { type: 'list', name: lookupName, content: EMPTY_RESOURCE_PLACEHOLDER }, - ]); + // Saving the lookup with an empty content to omit it. + // The translation will ignore this lookup and will not cause partial translations. + upsertResources(migrationStats.id, [{ type: 'lookup', name: lookupName, content: '' }]); }, [upsertResources, migrationStats] ); @@ -59,7 +58,7 @@ export const useMissingLookupsListStep = ({ onCopied={onCopied} missingLookups={missingLookups} uploadedLookups={uploadedLookups} - clearLookup={clearLookup} + omitLookup={omitLookup} /> ), }; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx index cd462a41bb6c1..fb521d9302a83 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx @@ -19,7 +19,6 @@ import { EuiToolTip, useEuiTheme, } from '@elastic/eui'; -import { EMPTY_RESOURCE_PLACEHOLDER } from '../../../../../../../../../common/siem_migrations/constants'; import type { UploadedLookups } from '../../lookups_data_input'; import * as i18n from './translations'; @@ -31,18 +30,18 @@ const scrollPanelCss = css` interface MissingLookupsListProps { missingLookups: string[]; uploadedLookups: UploadedLookups; - clearLookup: (lookupsName: string) => void; + omitLookup: (lookupsName: string) => void; onCopied: () => void; } export const MissingLookupsList = React.memo( - ({ missingLookups, uploadedLookups, clearLookup, onCopied }) => { + ({ missingLookups, uploadedLookups, omitLookup, onCopied }) => { const { euiTheme } = useEuiTheme(); return ( <> {missingLookups.map((lookupName) => { - const isMarkedAsEmpty = uploadedLookups[lookupName] === EMPTY_RESOURCE_PLACEHOLDER; + const isOmitted = uploadedLookups[lookupName] === ''; return ( ( justifyContent="flexStart" > - {uploadedLookups[lookupName] ? ( + {uploadedLookups[lookupName] != null ? ( ) : ( )} - + {lookupName} @@ -78,10 +74,10 @@ export const MissingLookupsList = React.memo( - @@ -115,7 +111,7 @@ const CopyLookupNameButton = React.memo( ( ); CopyLookupNameButton.displayName = 'CopyLookupNameButton'; -interface ClearLookupButtonProps { +interface OmitLookupButtonProps { lookupName: string; - clearLookup: (lookupName: string) => void; + omitLookup: (lookupName: string) => void; isDisabled: boolean; } -const ClearLookupButton = React.memo( - ({ lookupName, clearLookup, isDisabled: isDisabledDefault }) => { +const OmitLookupButton = React.memo( + ({ lookupName, omitLookup, isDisabled: isDisabledDefault }) => { const [isDisabled, setIsDisabled] = useState(isDisabledDefault); const onClick = useCallback(() => { setIsDisabled(true); - clearLookup(lookupName); - }, [clearLookup, lookupName]); + omitLookup(lookupName); + }, [omitLookup, lookupName]); const button = useMemo( () => ( @@ -158,4 +154,4 @@ const ClearLookupButton = React.memo( return {button}; } ); -ClearLookupButton.displayName = 'ClearLookupButton'; +OmitLookupButton.displayName = 'OmitLookupButton'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts index 1e5a8a0f7028c..5c33dcc5c1826 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts @@ -6,10 +6,10 @@ */ import type { - RuleMigrationResourceData, + RuleMigrationResourceBase, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; export type OnMigrationCreated = (migrationStats: RuleMigrationTaskStats) => void; export type OnResourcesCreated = () => void; -export type OnMissingResourcesFetched = (missingResources: RuleMigrationResourceData[]) => void; +export type OnMissingResourcesFetched = (missingResources: RuleMigrationResourceBase[]) => void; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx index 3c230cba4c34f..d10ac6f6eb551 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx @@ -8,7 +8,7 @@ import React, { useCallback, useEffect } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiButton, EuiPanel } from '@elastic/eui'; import { CenteredLoadingSpinner } from '../../../../common/components/centered_loading_spinner'; -import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { PanelText } from '../../../../common/components/panel_text'; import { useStartMigration } from '../../service/hooks/use_start_migration'; import type { RuleMigrationStats } from '../../types'; @@ -21,7 +21,7 @@ export interface MigrationReadyPanelProps { } export const MigrationReadyPanel = React.memo(({ migrationStats }) => { const { openFlyout } = useRuleMigrationDataInputContext(); - const [missingResources, setMissingResources] = React.useState([]); + const [missingResources, setMissingResources] = React.useState([]); const { getMissingResources, isLoading } = useGetMissingResources(setMissingResources); useEffect(() => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx index f1c6bdd71613a..569c726c9a87e 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx @@ -16,7 +16,7 @@ import { } from '@elastic/eui'; import { AssistantIcon } from '@kbn/ai-assistant-icon'; import type { SpacerSize } from '@elastic/eui/src/components/spacer/spacer'; -import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { PanelText } from '../../../../common/components/panel_text'; import { useGetMissingResources } from '../../service/hooks/use_get_missing_resources'; import * as i18n from './translations'; @@ -31,7 +31,7 @@ export const RuleMigrationsUploadMissingPanel = React.memo { const { euiTheme } = useEuiTheme(); const { openFlyout } = useRuleMigrationDataInputContext(); - const [missingResources, setMissingResources] = React.useState([]); + const [missingResources, setMissingResources] = React.useState([]); const { getMissingResources, isLoading } = useGetMissingResources(setMissingResources); useEffect(() => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts index a0679aa1e8bd2..5c4ec3925c5e3 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts @@ -7,7 +7,7 @@ import { useCallback, useReducer } from 'react'; import { i18n } from '@kbn/i18n'; -import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { useKibana } from '../../../../common/lib/kibana/kibana_react'; import { reducer, initialState } from './common/api_request_reducer'; @@ -17,7 +17,7 @@ export const RULES_DATA_INPUT_CREATE_MIGRATION_ERROR = i18n.translate( ); export type GetMissingResources = (migrationId: string) => void; -export type OnSuccess = (missingResources: RuleMigrationResourceData[]) => void; +export type OnSuccess = (missingResources: RuleMigrationResourceBase[]) => void; export const useGetMissingResources = (onSuccess: OnSuccess) => { const { siemMigrations, notifications } = useKibana().services; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts index cda500df3e215..1fcc14b5846ed 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts @@ -15,7 +15,7 @@ import { import type { RelatedIntegration } from '../../../../common/api/detection_engine'; import type { LangSmithOptions } from '../../../../common/siem_migrations/model/common.gen'; import type { - RuleMigrationResourceData, + RuleMigrationResourceBase, RuleMigrationTaskStats, } from '../../../../common/siem_migrations/model/rule_migration.gen'; import type { @@ -187,7 +187,7 @@ export class SiemRulesMigrationsService { return results; } - public async getMissingResources(migrationId: string): Promise { + public async getMissingResources(migrationId: string): Promise { return getMissingResources({ migrationId }); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts index 0c9ad11f4cce6..6b2108703bcc6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts @@ -7,7 +7,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import type { RuleMigrationResourceData } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; import { GetRuleMigrationResourcesMissingRequestParams, type GetRuleMigrationResourcesMissingResponse, @@ -49,7 +49,7 @@ export const registerSiemRuleMigrationsResourceGetMissingRoute = ( const options = { filters: { hasContent: false } }; const batches = ruleMigrationsClient.data.resources.searchBatches(migrationId, options); - const missingResources: RuleMigrationResourceData[] = []; + const missingResources: RuleMigrationResourceBase[] = []; let results = await batches.next(); while (results.length) { missingResources.push(...results.map(({ type, name }) => ({ type, name }))); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts index fde332aefbd3f..94a5c9368144f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts @@ -7,6 +7,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import partition from 'lodash/partition'; import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; import { UpsertRuleMigrationResourcesRequestBody, @@ -17,6 +18,7 @@ import { SIEM_RULE_MIGRATION_RESOURCES_PATH } from '../../../../../../common/sie import type { SecuritySolutionPluginRouter } from '../../../../../types'; import type { CreateRuleMigrationResourceInput } from '../../data/rule_migrations_data_resources_client'; import { withLicense } from '../util/with_license'; +import { processLookups } from '../util/lookups'; export const registerSiemRuleMigrationsResourceUpsertRoute = ( router: SecuritySolutionPluginRouter, @@ -58,14 +60,17 @@ export const registerSiemRuleMigrationsResourceUpsertRoute = ( return res.notFound({ body: { message: 'Migration not found' } }); } - // Upsert identified resource documents with content - const ruleMigrations = resources.map((resource) => ({ + const [lookups, macros] = partition(resources, { type: 'lookup' }); + const processedLookups = await processLookups(lookups, ruleMigrationsClient); + const resourcesUpsert = [...macros, ...processedLookups].map((resource) => ({ ...resource, migration_id: migrationId, })); - await ruleMigrationsClient.data.resources.upsert(ruleMigrations); - // Create identified resource documents without content to keep track of them + // Upsert the resources + await ruleMigrationsClient.data.resources.upsert(resourcesUpsert); + + // Create identified resource documents to keep track of them (without content) const resourceIdentifier = new ResourceIdentifier(rule.original_rule.vendor); const resourcesToCreate = resourceIdentifier .fromResources(resources) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts new file mode 100644 index 0000000000000..f18fc05dafbdb --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Papa from 'papaparse'; +import { initPromisePool } from '../../../../../utils/promise_pool'; +import type { RuleMigrationResourceData } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { SiemRuleMigrationsClient } from '../../siem_rule_migrations_service'; + +interface LookupWithData extends RuleMigrationResourceData { + data: object[] | null; +} + +export const processLookups = async ( + resources: RuleMigrationResourceData[], + ruleMigrationsClient: SiemRuleMigrationsClient +): Promise => { + const lookupsData: Record = {}; + + resources.forEach((resource) => { + if (resource.type === 'lookup' && !lookupsData[resource.name]) { + try { + lookupsData[resource.name] = { ...resource, data: parseContent(resource.content) }; + } catch (error) { + throw new Error(`Invalid content for lookup ${name}: ${error.message}`); + } + } + }); + + const lookups: RuleMigrationResourceData[] = []; + const result = await initPromisePool({ + concurrency: 10, + items: Object.entries(lookupsData), + executor: async ([name, { data, ...resource }]) => { + if (!data) { + lookups.push({ ...resource, content: '' }); // empty content will make lookup be ignored during translation + return; + } + const indexName = await ruleMigrationsClient.data.lookups.create(name, data); + lookups.push({ ...resource, content: indexName }); // lookup will be translated using the index name + }, + }); + const [error] = result.errors; + if (error) { + throw new Error(`Failed to process lookups: ${error.error}`); + } + + return lookups; +}; + +const parseContent = (fileContent: string): object[] | null => { + const trimmedContent = fileContent.trim(); + if (fileContent === '') { + return null; + } + let arrayContent: object[]; + + if (trimmedContent.startsWith('[')) { + arrayContent = parseJSONArray(trimmedContent); + } else if (trimmedContent.startsWith('{')) { + arrayContent = parseNDJSON(trimmedContent); + } else { + arrayContent = parseCSV(trimmedContent); + } + return arrayContent; +}; + +const parseCSV = (fileContent: string): object[] => { + const config: Papa.ParseConfig = { + header: true, // If header is false, rows are arrays; otherwise they are objects of data keyed by the field name. + skipEmptyLines: true, + }; + const { data, errors } = Papa.parse(fileContent, config); + if (errors.length > 0) { + throw new Error('Invalid CSV'); + } + return data as object[]; +}; + +const parseNDJSON = (fileContent: string): object[] => { + return fileContent + .split(/\n(?=\{)/) // split at newline followed by '{'. + .filter((entry) => entry.trim() !== '') // Remove empty entries. + .map(parseJSON); // Parse each entry as JSON. +}; + +const parseJSONArray = (fileContent: string): object[] => { + const parsedContent = parseJSON(fileContent); + if (!Array.isArray(parsedContent)) { + throw new Error('invalid JSON'); + } + return parsedContent; +}; + +const parseJSON = (fileContent: string) => { + try { + return JSON.parse(fileContent); + } catch (error) { + if (error instanceof RangeError) { + throw new Error('File is too large'); + } + throw new Error('Invalid JSON'); + } +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts index e479c42cce273..9198a521bcb96 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts @@ -5,12 +5,13 @@ * 2.0. */ -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { IScopedClusterClient, Logger } from '@kbn/core/server'; import type { PackageService } from '@kbn/fleet-plugin/server'; import { RuleMigrationsDataIntegrationsClient } from './rule_migrations_data_integrations_client'; import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_prebuilt_rules_client'; import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client'; import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client'; +import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client'; import type { AdapterId } from './rule_migrations_data_service'; export type IndexNameProvider = () => Promise; @@ -21,37 +22,43 @@ export class RuleMigrationsDataClient { public readonly resources: RuleMigrationsDataResourcesClient; public readonly integrations: RuleMigrationsDataIntegrationsClient; public readonly prebuiltRules: RuleMigrationsDataPrebuiltRulesClient; + public readonly lookups: RuleMigrationsDataLookupsClient; constructor( indexNameProviders: IndexNameProviders, username: string, - esClient: ElasticsearchClient, + esScopedClient: IScopedClusterClient, logger: Logger, packageService?: PackageService ) { this.rules = new RuleMigrationsDataRulesClient( indexNameProviders.rules, username, - esClient, + esScopedClient.asInternalUser, logger ); this.resources = new RuleMigrationsDataResourcesClient( indexNameProviders.resources, username, - esClient, + esScopedClient.asInternalUser, logger ); this.integrations = new RuleMigrationsDataIntegrationsClient( indexNameProviders.integrations, username, - esClient, + esScopedClient.asInternalUser, logger, packageService ); this.prebuiltRules = new RuleMigrationsDataPrebuiltRulesClient( indexNameProviders.prebuiltrules, username, - esClient, + esScopedClient.asInternalUser, + logger + ); + this.lookups = new RuleMigrationsDataLookupsClient( + username, + esScopedClient.asCurrentUser, logger ); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts index 947a206cd0c7a..54a4f14a667e4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts @@ -8,7 +8,7 @@ import type { PackageService } from '@kbn/fleet-plugin/server'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { PackageList } from '@kbn/fleet-plugin/common'; -import type { Integration } from '../types'; +import type { RuleMigrationIntegration } from '../types'; import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; /* This will be removed once the package registry changes is performed */ @@ -21,7 +21,7 @@ const MIN_SCORE = 40 as const; const RETURNED_INTEGRATIONS = 5 as const; /* This is a temp implementation to allow further development until https://github.com/elastic/package-registry/issues/1252 */ -const INTEGRATIONS = integrationsFile as Integration[]; +const INTEGRATIONS = integrationsFile as RuleMigrationIntegration[]; /* BULK_MAX_SIZE defines the number to break down the bulk operations by. * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ @@ -70,30 +70,18 @@ export class RuleMigrationsDataIntegrationsClient extends RuleMigrationsDataBase } /** Based on a LLM generated semantic string, returns the 5 best results with a score above 40 */ - async retrieveIntegrations(semanticString: string): Promise { + async retrieveIntegrations(semanticString: string): Promise { const index = await this.getIndexName(); const query = { bool: { should: [ - { - semantic: { - query: semanticString, - field: 'elser_embedding', - boost: 1.5, - }, - }, - { - multi_match: { - query: semanticString, - fields: ['title^2', 'description'], - boost: 3, - }, - }, + { semantic: { query: semanticString, field: 'elser_embedding', boost: 1.5 } }, + { multi_match: { query: semanticString, fields: ['title^2', 'description'], boost: 3 } }, ], }, }; const results = await this.esClient - .search({ + .search({ index, query, size: RETURNED_INTEGRATIONS, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts new file mode 100644 index 0000000000000..93763d6508cf0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sha256 } from 'js-sha256'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '@kbn/index-adapter'; + +export type LookupData = object[]; + +export class RuleMigrationsDataLookupsClient { + constructor( + protected username: string, + protected esClient: ElasticsearchClient, + protected logger: Logger + ) {} + + async create(lookupName: string, data: LookupData): Promise { + const indexName = `lookup_${lookupName}`; + try { + await this.executeEs(() => + this.esClient.indices.create({ + index: indexName, + settings: { index: { mode: 'lookup' } }, + mappings: { dynamic: 'runtime' }, + }) + ); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + this.logger.error(`Error creating lookup index ${indexName} - ${error.message}`); + throw error; + } + } + + if (data.length > 0) { + await this.indexData(indexName, data); + } + return indexName; + } + + async indexData(indexName: string, data: LookupData): Promise { + const body = data.flatMap((doc) => [ + { create: { _index: indexName, _id: this.generateDocumentHash(doc) } }, + doc, + ]); + + try { + await this.executeEs(() => this.esClient.bulk({ index: indexName, body })); + } catch (error) { + if (error?.statusCode !== 404) { + this.logger.error(`Error indexing data for lookup index ${indexName} - ${error.message}`); + throw error; + } + } + } + + private async executeEs(fn: () => Promise): Promise { + return retryTransientEsErrors(fn, { logger: this.logger }); + } + + private generateDocumentHash(document: object): string { + return sha256.create().update(JSON.stringify(document)).hex(); // document need to be created in a deterministic way + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts index ccd158c347c77..ccc0b31b31dc3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts @@ -50,7 +50,7 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas filteredRules.push({ rule_id: rule.rule_id, name: rule.name, - installedRuleId: ruleVersions.current?.id, + installed_rule_id: ruleVersions.current?.id, description: rule.description, elser_embedding: `${rule.name} - ${rule.description}`, ...(mitreAttackIds?.length && { mitre_attack_ids: mitreAttackIds }), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts index 97e51e9bafdb0..c2bd1ed2e50f8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts @@ -16,10 +16,8 @@ import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client export type CreateRuleMigrationResourceInput = Pick< RuleMigrationResource, - 'migration_id' | 'type' | 'name' | 'metadata' -> & { - content?: string; -}; + 'migration_id' | 'type' | 'name' | 'content' | 'metadata' +>; export interface RuleMigrationResourceFilters { type?: RuleMigrationResourceType; names?: string[]; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts index e991ce2684f3e..cb6c94ebae40f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts @@ -9,7 +9,7 @@ import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { securityServiceMock } from '@kbn/core-security-server-mocks'; import type { InstallParams } from '@kbn/index-adapter'; -import { IndexPatternAdapter } from '@kbn/index-adapter'; +import { IndexPatternAdapter, IndexAdapter } from '@kbn/index-adapter'; import { loggerMock } from '@kbn/logging-mocks'; import { Subject } from 'rxjs'; import type { IndexNameProviders } from './rule_migrations_data_client'; @@ -28,6 +28,7 @@ jest.mock('./rule_migrations_data_client', () => ({ const MockedIndexPatternAdapter = IndexPatternAdapter as unknown as jest.MockedClass< typeof IndexPatternAdapter >; +const MockedIndexAdapter = IndexAdapter as unknown as jest.MockedClass; const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; @@ -42,39 +43,42 @@ describe('SiemRuleMigrationsDataService', () => { describe('constructor', () => { it('should create IndexPatternAdapters', () => { new RuleMigrationsDataService(logger, kibanaVersion); - expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(4); + expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(2); + expect(MockedIndexAdapter).toHaveBeenCalledTimes(2); }); it('should create component templates', () => { new RuleMigrationsDataService(logger, kibanaVersion); - const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + const [rulesAdapter, resourcesAdapter] = MockedIndexPatternAdapter.mock.instances; + const [integrationsAdapter, prebuiltRulesAdapter] = MockedIndexAdapter.mock.instances; + expect(rulesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-rules` }) ); - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect(resourcesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-resources` }) ); - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect(integrationsAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-integrations` }) ); - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect(prebuiltRulesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltrules` }) ); }); it('should create index templates', () => { new RuleMigrationsDataService(logger, kibanaVersion); - const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + const [rulesAdapter, resourcesAdapter] = MockedIndexPatternAdapter.mock.instances; + const [integrationsAdapter, prebuiltRulesAdapter] = MockedIndexAdapter.mock.instances; + expect(rulesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-rules` }) ); - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect(resourcesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-resources` }) ); - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect(integrationsAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-integrations` }) ); - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect(prebuiltRulesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltrules` }) ); }); @@ -82,55 +86,50 @@ describe('SiemRuleMigrationsDataService', () => { describe('install', () => { it('should install index pattern', async () => { - const index = new RuleMigrationsDataService(logger, kibanaVersion); + const service = new RuleMigrationsDataService(logger, kibanaVersion); const params: Omit = { esClient, pluginStop$: new Subject(), }; - await index.install(params); + await service.install(params); const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; + const [indexAdapter] = MockedIndexAdapter.mock.instances; + expect(indexPatternAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); + expect(indexAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); }); }); describe('createClient', () => { const currentUser = securityServiceMock.createMockAuthenticatedUser(); - const createClientParams = { spaceId: 'space1', currentUser, esClient }; + const createClientParams = { + spaceId: 'space1', + currentUser, + esScopedClient: elasticsearchServiceMock.createStart().client.asScoped(), + }; it('should install space index pattern', async () => { - const index = new RuleMigrationsDataService(logger, kibanaVersion); + const service = new RuleMigrationsDataService(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), pluginStop$: new Subject(), }; - const [ - rulesIndexPatternAdapter, - resourcesIndexPatternAdapter, - integrationsIndexPatternAdapter, - prebuiltrulesIndexPatternAdapter, - ] = MockedIndexPatternAdapter.mock.instances; + const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter] = + MockedIndexPatternAdapter.mock.instances; (rulesIndexPatternAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); - await index.install(params); - index.createClient(createClientParams); + await service.install(params); + service.createClient(createClientParams); await mockIndexNameProviders.rules(); await mockIndexNameProviders.resources(); - await mockIndexNameProviders.integrations(); - await mockIndexNameProviders.prebuiltrules(); expect(rulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); expect(rulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); expect(resourcesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); expect(resourcesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); - - expect(integrationsIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); - expect(integrationsIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); - - expect(prebuiltrulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); - expect(prebuiltrulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts index 6681f0c3903b0..5cacaf5592407 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts @@ -4,8 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; -import { IndexPatternAdapter, type FieldMap, type InstallParams } from '@kbn/index-adapter'; +import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server'; +import { + IndexAdapter, + IndexPatternAdapter, + type FieldMap, + type InstallParams, +} from '@kbn/index-adapter'; import type { PackageService } from '@kbn/fleet-plugin/server'; import type { IndexNameProvider, IndexNameProviders } from './rule_migrations_data_client'; import { RuleMigrationsDataClient } from './rule_migrations_data_client'; @@ -19,29 +24,56 @@ import { const TOTAL_FIELDS_LIMIT = 2500; export const INDEX_PATTERN = '.kibana-siem-rule-migrations'; -export type AdapterId = 'rules' | 'resources' | 'integrations' | 'prebuiltrules'; +export interface Adapters { + rules: IndexPatternAdapter; + resources: IndexPatternAdapter; + integrations: IndexAdapter; + prebuiltrules: IndexAdapter; +} + +export type AdapterId = keyof Adapters; interface CreateClientParams { spaceId: string; currentUser: AuthenticatedUser; - esClient: ElasticsearchClient; + esScopedClient: IScopedClusterClient; packageService?: PackageService; } +interface CreateAdapterParams { + adapterId: AdapterId; + fieldMap: FieldMap; +} export class RuleMigrationsDataService { - private readonly adapters: Record; + private readonly adapters: Adapters; constructor(private logger: Logger, private kibanaVersion: string) { this.adapters = { - rules: this.createAdapter({ id: 'rules', fieldMap: ruleMigrationsFieldMap }), - resources: this.createAdapter({ id: 'resources', fieldMap: ruleMigrationResourcesFieldMap }), - integrations: this.createAdapter({ id: 'integrations', fieldMap: integrationsFieldMap }), - prebuiltrules: this.createAdapter({ id: 'prebuiltrules', fieldMap: prebuiltRulesFieldMap }), + rules: this.createIndexPatternAdapter({ + adapterId: 'rules', + fieldMap: ruleMigrationsFieldMap, + }), + resources: this.createIndexPatternAdapter({ + adapterId: 'resources', + fieldMap: ruleMigrationResourcesFieldMap, + }), + integrations: this.createIndexAdapter({ + adapterId: 'integrations', + fieldMap: integrationsFieldMap, + }), + prebuiltrules: this.createIndexAdapter({ + adapterId: 'prebuiltrules', + fieldMap: prebuiltRulesFieldMap, + }), }; } - private createAdapter({ id, fieldMap }: { id: AdapterId; fieldMap: FieldMap }) { - const name = `${INDEX_PATTERN}-${id}`; + private getAdapterIndexName(adapterId: AdapterId) { + return `${INDEX_PATTERN}-${adapterId}`; + } + + private createIndexPatternAdapter({ adapterId, fieldMap }: CreateAdapterParams) { + const name = this.getAdapterIndexName(adapterId); const adapter = new IndexPatternAdapter(name, { kibanaVersion: this.kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, @@ -51,6 +83,17 @@ export class RuleMigrationsDataService { return adapter; } + private createIndexAdapter({ adapterId, fieldMap }: CreateAdapterParams) { + const name = this.getAdapterIndexName(adapterId); + const adapter = new IndexAdapter(name, { + kibanaVersion: this.kibanaVersion, + totalFieldsLimit: TOTAL_FIELDS_LIMIT, + }); + adapter.setComponentTemplate({ name, fieldMap }); + adapter.setIndexTemplate({ name, componentTemplateRefs: [name] }); + return adapter; + } + public async install(params: Omit): Promise { await Promise.all([ this.adapters.rules.install({ ...params, logger: this.logger }), @@ -60,27 +103,35 @@ export class RuleMigrationsDataService { ]); } - public createClient({ spaceId, currentUser, esClient, packageService }: CreateClientParams) { + public createClient({ + spaceId, + currentUser, + esScopedClient, + packageService, + }: CreateClientParams) { const indexNameProviders: IndexNameProviders = { - rules: this.createIndexNameProvider('rules', spaceId), - resources: this.createIndexNameProvider('resources', spaceId), - integrations: this.createIndexNameProvider('integrations', spaceId), - prebuiltrules: this.createIndexNameProvider('prebuiltrules', spaceId), + rules: this.createIndexNameProvider(this.adapters.rules, spaceId), + resources: this.createIndexNameProvider(this.adapters.resources, spaceId), + integrations: async () => this.getAdapterIndexName('integrations'), + prebuiltrules: async () => this.getAdapterIndexName('prebuiltrules'), }; return new RuleMigrationsDataClient( indexNameProviders, currentUser.username, - esClient, + esScopedClient, this.logger, packageService ); } - private createIndexNameProvider(adapter: AdapterId, spaceId: string): IndexNameProvider { + private createIndexNameProvider( + adapter: IndexPatternAdapter, + spaceId: string + ): IndexNameProvider { return async () => { - await this.adapters[adapter].createIndex(spaceId); // This will resolve instantly when the index is already created - return this.adapters[adapter].getIndexName(spaceId); + await adapter.createIndex(spaceId); // This will resolve instantly when the index is already created + return adapter.getIndexName(spaceId); }; } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts index c03b7ef52e784..9369b3b5e1bc7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts @@ -10,6 +10,7 @@ import type { RuleMigration, RuleMigrationResource, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationIntegration, RuleMigrationPrebuiltRule } from '../types'; export const ruleMigrationsFieldMap: FieldMap>> = { '@timestamp': { type: 'date', required: false }, @@ -52,22 +53,22 @@ export const ruleMigrationResourcesFieldMap: FieldMap< updated_by: { type: 'keyword', required: false }, }; -export const integrationsFieldMap: FieldMap = { - '@timestamp': { type: 'date', required: true }, +export const integrationsFieldMap: FieldMap> = { + id: { type: 'keyword', required: true }, title: { type: 'text', required: true }, description: { type: 'text', required: true }, data_streams: { type: 'nested', array: true, required: true }, 'data_streams.dataset': { type: 'keyword', required: true }, 'data_streams.title': { type: 'text', required: true }, 'data_streams.index_pattern': { type: 'keyword', required: true }, - elser_embeddings: { type: 'semantic_text', required: true }, + elser_embedding: { type: 'semantic_text', required: true }, }; -export const prebuiltRulesFieldMap: FieldMap = { - '@timestamp': { type: 'date', required: true }, +export const prebuiltRulesFieldMap: FieldMap> = { name: { type: 'text', required: true }, description: { type: 'text', required: true }, elser_embedding: { type: 'semantic_text', required: true }, rule_id: { type: 'keyword', required: true }, + installed_rule_id: { type: 'keyword', required: true }, mitre_attack_ids: { type: 'keyword', array: true, required: false }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 2e701f0fc7b3e..a25fa7d576114 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -95,7 +95,7 @@ describe('SiemRuleMigrationsService', () => { expect(mockDataCreateClient).toHaveBeenCalledWith({ spaceId: createClientParams.spaceId, currentUser: createClientParams.currentUser, - esClient: esClusterClient.asInternalUser, + esScopedClient: esClusterClient.asScoped(), }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index 3be54a7e3d896..e3c6d4a13b15b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -68,11 +68,11 @@ export class SiemRuleMigrationsService { assert(currentUser, 'Current user must be authenticated'); assert(this.esClusterClient, 'ES client not available, please call setup first'); - const esClient = this.esClusterClient.asInternalUser; + const esScopedClient = this.esClusterClient.asScoped(request); const dataClient = this.dataService.createClient({ spaceId, currentUser, - esClient, + esScopedClient, packageService, }); const taskClient = this.taskService.createClient({ currentUser, dataClient }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index b1165ce982293..988e919943ba1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -6,10 +6,8 @@ */ import { END, START, StateGraph } from '@langchain/langgraph'; -import { RuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; import { getCreateSemanticQueryNode } from './nodes/create_semantic_query'; import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; -import { getProcessQueryNode } from './nodes/process_query'; import { migrateRuleState } from './state'; import { getTranslateRuleGraph } from './sub_graphs/translate_rule'; @@ -35,19 +33,19 @@ export function getRuleMigrationAgent({ logger, }); const createSemanticQueryNode = getCreateSemanticQueryNode({ model }); - const processQueryNode = getProcessQueryNode({ model, ruleMigrationsRetriever }); const siemMigrationAgentGraph = new StateGraph(migrateRuleState) // Nodes - .addNode('processQuery', processQueryNode) .addNode('createSemanticQuery', createSemanticQueryNode) .addNode('matchPrebuiltRule', matchPrebuiltRuleNode) .addNode('translationSubGraph', translationSubGraph) // Edges .addEdge(START, 'createSemanticQuery') .addEdge('createSemanticQuery', 'matchPrebuiltRule') - .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional, ['processQuery', END]) - .addEdge('processQuery', 'translationSubGraph') + .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional, [ + 'translationSubGraph', + END, + ]) .addEdge('translationSubGraph', END); const graph = siemMigrationAgentGraph.compile(); @@ -55,15 +53,9 @@ export function getRuleMigrationAgent({ return graph; } -/* - * If the original splunk rule has no prebuilt rule match, we will start processing the query, unless it is related to input/outputlookups. - */ const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { if (state.elastic_rule?.prebuilt_rule_id) { return END; } - if (state.translation_result === RuleTranslationResult.UNTRANSLATABLE) { - return END; - } - return 'processQuery'; + return 'translationSubGraph'; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts index d7537fdc72dd0..fd3dfc983443e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -12,6 +12,7 @@ import type { RuleMigrationsRetriever } from '../../../retrievers'; import type { ChatModel } from '../../../util/actions_client_chat'; import type { GraphNode } from '../../types'; import { MATCH_PREBUILT_RULE_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../util/comments'; interface GetMatchPrebuiltRuleNodeParams { model: ChatModel; @@ -21,6 +22,7 @@ interface GetMatchPrebuiltRuleNodeParams { interface GetMatchedRuleResponse { match: string; + summary: string; } export const getMatchPrebuiltRuleNode = ({ @@ -35,6 +37,9 @@ export const getMatchPrebuiltRuleNode = ({ query, techniqueIds.join(',') ); + if (prebuiltRules.length === 0) { + return { comments: ['## Prebuilt Rule Matching Summary\nNo related prebuilt rule found.'] }; + } const outputParser = new JsonOutputParser(); const mostRelevantRule = MATCH_PREBUILT_RULE_PROMPT.pipe(model).pipe(outputParser); @@ -58,30 +63,24 @@ export const getMatchPrebuiltRuleNode = ({ rules: JSON.stringify(elasticSecurityRules, null, 2), splunk_rule: JSON.stringify(splunkRule, null, 2), })) as GetMatchedRuleResponse; + + const comments = response.summary ? [cleanMarkdown(response.summary)] : undefined; + if (response.match) { const matchedRule = prebuiltRules.find((r) => r.name === response.match); if (matchedRule) { return { + comments, elastic_rule: { title: matchedRule.name, description: matchedRule.description, - id: matchedRule.installedRuleId, + id: matchedRule.installed_rule_id, prebuilt_rule_id: matchedRule.rule_id, }, translation_result: RuleTranslationResult.FULL, }; } } - const lookupTypes = ['inputlookup', 'outputlookup']; - if ( - state.original_rule?.query && - lookupTypes.some((type) => state.original_rule.query.includes(type)) - ) { - logger.debug( - `Rule: ${state.original_rule?.title} did not match any prebuilt rule, but contains inputlookup, dropping` - ); - return { translation_result: RuleTranslationResult.UNTRANSLATABLE }; - } - return {}; + return { comments }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts index 12fb7ec70febf..c109a8dc3dd8d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts @@ -10,7 +10,8 @@ export const MATCH_PREBUILT_RULE_PROMPT = ChatPromptTemplate.fromMessages([ [ 'system', `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. -You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Detection Rule that covers the same threat, if any. +You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Prebuilt Rule for the same purpose, if any. + Here are some context for you to reference for your task, read it carefully as you will get questions about it later: @@ -22,18 +23,27 @@ Here are some context for you to reference for your task, read it carefully as y ], [ 'human', - `See the below description of the relevant splunk rule and try to match it with any of the elastic detection rules with similar names. + `See the below description of the splunk rule, try to find a Elastic Prebuilt Rule with similar purpose. {splunk_rule} -- Always reply with a JSON object with the key "match" and the value being the most relevant matched elastic detection rule name. Do not reply with anything else. -- Only reply with exact matches, if you are unsure or do not find a very confident match, always reply with an empty string value in the match key, do not guess or reply with anything else. -- If there is one Elastic rule in the list that covers the same usecase, set the name of the matching rule as a value of the match key. Do not reply with anything else. -- If there are multiple rules in the list that cover the same usecase, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". +- Carefully analyze the Splunk Detection Rule data provided by the user. +- Match the Splunk rule to the most relevant Elastic Prebuilt Rules from the list provided above. +- If no related Elastic Prebuilt Rule is found, reply with an empty string. +- Provide a concise reasoning summary for your decision, explaining why the selected Prebuilt Rule is the best fit, or why no suitable match was found. + +- Always reply with a JSON object with the key "match" and the value being the most relevant matched elastic detection rule name, and a "summary" entry with the reasons behind the match. Do not reply with anything else. +- Only reply with exact matches, if you are unsure or do not find a very confident match, always reply with an empty string value in the match key, do not guess or reply with anything else. +- If there is only one match, answer with the name of the rule in the "match" key. Do not reply with anything else. +- If there are multiple matches, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". +- Finally, write a "summary" in markdown format with the reasoning behind the decision. Starting with "## Prebuilt Rule Matching Summary\n". +- Make sure the JSON object is formatted correctly and the values properly escaped. + + U: Title: Linux Auditd Add User Account Type @@ -41,7 +51,10 @@ Description: The following analytic detects the suspicious add user account type A: Please find the match JSON object below: \`\`\`json -{{"match": "Linux User Account Creation"}} +{{ + "match": "Linux User Account Creation", + "summary": "## Prebuilt Rule Matching Summary\\\nThe Splunk rule \"Linux Auditd Add User Account Type\" is matched with the Elastic rule \"Linux User Account Creation\" because both rules cover the same use case of detecting user account creation on Linux systems." +}} \`\`\` `, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts deleted file mode 100644 index e4b7e64e85b00..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { StringOutputParser } from '@langchain/core/output_parsers'; -import { isEmpty } from 'lodash/fp'; -import type { RuleMigrationsRetriever } from '../../../retrievers'; -import type { ChatModel } from '../../../util/actions_client_chat'; -import type { GraphNode } from '../../types'; -import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts'; - -interface GetProcessQueryNodeParams { - model: ChatModel; - ruleMigrationsRetriever: RuleMigrationsRetriever; -} - -export const getProcessQueryNode = ({ - model, - ruleMigrationsRetriever, -}: GetProcessQueryNodeParams): GraphNode => { - return async (state) => { - let query = state.original_rule.query; - const resources = await ruleMigrationsRetriever.resources.getResources(state.original_rule); - if (!isEmpty(resources)) { - const replaceQueryParser = new StringOutputParser(); - const replaceQueryResourcePrompt = - REPLACE_QUERY_RESOURCE_PROMPT.pipe(model).pipe(replaceQueryParser); - const resourceContext = getResourcesContext(resources); - const response = await replaceQueryResourcePrompt.invoke({ - query: state.original_rule.query, - macros: resourceContext.macros, - }); - const splQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1] ?? ''; - if (splQuery) { - query = splQuery; - } - } - return { inline_query: query }; - }; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts deleted file mode 100644 index 68eaaeffd11b1..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ChatPromptTemplate } from '@langchain/core/prompts'; -import type { RuleMigrationResources } from '../../../retrievers/rule_resource_retriever'; - -interface ResourceContext { - macros: string; - lists: string; -} - -export const getResourcesContext = (resources: RuleMigrationResources): ResourceContext => { - const result: ResourceContext = { macros: '', lists: '' }; - - // Process macros - if (resources.macro?.length) { - const macrosMap = resources.macro.reduce((acc, macro) => { - acc[macro.name] = macro.content; - return acc; - }, {} as Record); - - result.macros = JSON.stringify(macrosMap, null, 2); - } - - // Process lists - if (resources.list?.length) { - const listsMap = resources.list.reduce((acc, list) => { - acc[list.name] = list.content; - return acc; - }, {} as Record); - - result.lists = JSON.stringify(listsMap, null, 2); - } - - return result; -}; - -export const REPLACE_QUERY_RESOURCE_PROMPT = ChatPromptTemplate.fromMessages([ - [ - 'system', - `You are an agent expert in Splunk SPL (Search Processing Language). -Your task is to inline a set of macros syntax using its values in a SPL query. -Here are some context for you to reference for your task, read it carefully as you will get questions about it later: - - - - -Always follow the below guidelines when replacing macros: -- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments. - -Having the following macros: - \`someSource\`: sourcetype="somesource" - \`searchTitle(1)\`: search title="$value$" - \`searchTitle\`: search title=* - \`searchType\`: search type=* -And the following SPL query: - \`\`\`spl - \`someSource\` \`someFilter\` - | \`searchTitle("sometitle")\` - | \`searchType("sometype")\` - | table * - \`\`\` -The correct replacement would be: - \`\`\`spl - sourcetype="somesource" \`someFilter\` - | search title="sometitle" - | \`searchType("sometype")\` - | table * - \`\`\` - -`, - ], - [ - 'human', - `Go through the SPL query and identify all the macros that are used. - -{macros} - - - -\`\`\`spl -{query} -\`\`\` - - -Divide the query up into separate section and go through each section one at a time to identify the macros used that need to be replaced using one of two scenarios: -- The macro is provided in the list of available macros: Replace it using its actual content. -- The macro is not in the list of available macros: Do not replace it, keep it in the query as it is. - - -- You will be provided with a SPL query and also the related macros used in the query. -- You have to replace the macros syntax in the SPL query and use their values inline, if provided. -- The original and modified queries must be equivalent. -- You must respond only with the modified query inside a \`\`\`spl code block, nothing else similar to the example response below. - - - -A: Please find the modified SPL query below: -\`\`\`spl -sourcetype="linux:audit" \`linux_auditd_normalized_proctitle_process\` -| rename host as dest -| where LIKE (process_exec, "%chown root%") -| stats count min(_time) as firstTime max(_time) as lastTime by process_exec proctitle normalized_proctitle_delimiter dest -| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(firstTime) -| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(lastTime) -| search * -\`\`\` - - -`, - ], - ['ai', 'Please find the modified SPL query below:'], -]); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts index 66b3c0c8e7a71..cf07e676715a7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -7,6 +7,7 @@ import type { BaseMessage } from '@langchain/core/messages'; import { Annotation, messagesStateReducer } from '@langchain/langgraph'; +import { uniq } from 'lodash/fp'; import type { RuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; import type { ElasticRule, @@ -33,7 +34,8 @@ export const migrateRuleState = Annotation.Root({ }), translation_result: Annotation(), comments: Annotation({ - reducer: (current, value) => (value ? (current ?? []).concat(value) : current), + // Translation subgraph causes the original main graph comments to be concatenated again, we need to deduplicate them. + reducer: (current, value) => uniq(value ? (current ?? []).concat(value) : current), default: () => [], }), response: Annotation(), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts index 99d7d1439d63e..b18f60c006549 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts @@ -9,11 +9,12 @@ import { END, START, StateGraph } from '@langchain/langgraph'; import { isEmpty } from 'lodash/fp'; import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; import { getEcsMappingNode } from './nodes/ecs_mapping'; -import { getFilterIndexPatternsNode } from './nodes/filter_index_patterns'; +import { translationResultNode } from './nodes/translation_result'; import { getFixQueryErrorsNode } from './nodes/fix_query_errors'; import { getRetrieveIntegrationsNode } from './nodes/retrieve_integrations'; import { getTranslateRuleNode } from './nodes/translate_rule'; import { getValidationNode } from './nodes/validation'; +import { getInlineQueryNode } from './nodes/inline_query'; import { translateRuleState } from './state'; import type { TranslateRuleGraphParams, TranslateRuleState } from './types'; @@ -32,22 +33,27 @@ export function getTranslateRuleGraph({ connectorId, logger, }); + const inlineQueryNode = getInlineQueryNode({ model, ruleMigrationsRetriever }); const validationNode = getValidationNode({ logger }); const fixQueryErrorsNode = getFixQueryErrorsNode({ inferenceClient, connectorId, logger }); const retrieveIntegrationsNode = getRetrieveIntegrationsNode({ model, ruleMigrationsRetriever }); const ecsMappingNode = getEcsMappingNode({ inferenceClient, connectorId, logger }); - const filterIndexPatternsNode = getFilterIndexPatternsNode({ logger }); const translateRuleGraph = new StateGraph(translateRuleState) // Nodes + .addNode('inlineQuery', inlineQueryNode) + .addNode('retrieveIntegrations', retrieveIntegrationsNode) .addNode('translateRule', translateRuleNode) .addNode('validation', validationNode) .addNode('fixQueryErrors', fixQueryErrorsNode) - .addNode('retrieveIntegrations', retrieveIntegrationsNode) .addNode('ecsMapping', ecsMappingNode) - .addNode('filterIndexPatterns', filterIndexPatternsNode) + .addNode('translationResult', translationResultNode) // Edges - .addEdge(START, 'retrieveIntegrations') + .addEdge(START, 'inlineQuery') + .addConditionalEdges('inlineQuery', translatableRouter, [ + 'retrieveIntegrations', + 'translationResult', + ]) .addEdge('retrieveIntegrations', 'translateRule') .addEdge('translateRule', 'validation') .addEdge('fixQueryErrors', 'validation') @@ -55,15 +61,22 @@ export function getTranslateRuleGraph({ .addConditionalEdges('validation', validationRouter, [ 'fixQueryErrors', 'ecsMapping', - 'filterIndexPatterns', + 'translationResult', ]) - .addEdge('filterIndexPatterns', END); + .addEdge('translationResult', END); const graph = translateRuleGraph.compile(); graph.name = 'Translate Rule Graph'; return graph; } +const translatableRouter = (state: TranslateRuleState) => { + if (!state.inline_query) { + return 'translationResult'; + } + return 'retrieveIntegrations'; +}; + const validationRouter = (state: TranslateRuleState) => { if ( state.validation_errors.iterations <= MAX_VALIDATION_ITERATIONS && @@ -76,5 +89,5 @@ const validationRouter = (state: TranslateRuleState) => { return 'ecsMapping'; } } - return 'filterIndexPatterns'; + return 'translationResult'; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts index 07753432e5dbc..cddf85dea2d31 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts @@ -12,6 +12,7 @@ import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_ca import type { GraphNode } from '../../types'; import { SIEM_RULE_MIGRATION_CIM_ECS_MAP } from './cim_ecs_map'; import { ESQL_TRANSLATE_ECS_MAPPING_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetEcsMappingNodeParams { inferenceClient: InferenceClient; @@ -47,7 +48,7 @@ export const getEcsMappingNode = ({ return { response, - comments: [ecsSummary], + comments: [cleanMarkdown(ecsSummary)], translation_finalized: true, translation_result: translationResult, elastic_rule: { @@ -59,7 +60,7 @@ export const getEcsMappingNode = ({ }; const getTranslationResult = (esqlQuery: string): RuleTranslationResult => { - if (esqlQuery.match(/\[(macro):[\s\S]*\]/)) { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { return RuleTranslationResult.PARTIAL; } return RuleTranslationResult.FULL; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts deleted file mode 100644 index b7cbcabff2ca2..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; -import type { GraphNode } from '../../types'; - -interface GetFilterIndexPatternsNodeParams { - logger: Logger; -} - -/** - * When rule translation happens without any related integrations found we reuse the logs-* pattern to make validation easier. - * However we want to replace this with a value to notify the end user that it needs to be replaced. - */ -export const getFilterIndexPatternsNode = ({ - logger, -}: GetFilterIndexPatternsNodeParams): GraphNode => { - return async (state) => { - const query = state.elastic_rule?.query; - - if (query && query.includes('logs-*')) { - logger.debug('Replacing logs-* with a placeholder value'); - const newQuery = query.replace('logs-*', '[indexPattern:logs-*]'); - return { - elastic_rule: { - ...state.elastic_rule, - query: newQuery, - translation_result: RuleTranslationResult.PARTIAL, - }, - }; - } - - return {}; - }; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/index.ts similarity index 82% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/index.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/index.ts index 6feb852eba474..c466306e99074 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/index.ts @@ -4,4 +4,4 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -export { getProcessQueryNode } from './process_query'; +export { getInlineQueryNode } from './inline_query'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts new file mode 100644 index 0000000000000..e5d16f59658c3 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { isEmpty } from 'lodash/fp'; +import type { RuleMigrationsRetriever } from '../../../../../retrievers'; +import type { ChatModel } from '../../../../../util/actions_client_chat'; +import type { GraphNode } from '../../../../types'; +import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; + +interface GetInlineQueryNodeParams { + model: ChatModel; + ruleMigrationsRetriever: RuleMigrationsRetriever; +} + +export const getInlineQueryNode = ({ + model, + ruleMigrationsRetriever, +}: GetInlineQueryNodeParams): GraphNode => { + return async (state) => { + let query = state.original_rule.query; + + // Check before to avoid unnecessary LLM calls + let unsupportedComment = getUnsupportedComment(query); + if (unsupportedComment) { + return { comments: [unsupportedComment] }; + } + + const resources = await ruleMigrationsRetriever.resources.getResources(state.original_rule); + if (!isEmpty(resources)) { + const replaceQueryParser = new StringOutputParser(); + const replaceQueryResourcePrompt = + REPLACE_QUERY_RESOURCE_PROMPT.pipe(model).pipe(replaceQueryParser); + const resourceContext = getResourcesContext(resources); + const response = await replaceQueryResourcePrompt.invoke({ + query: state.original_rule.query, + macros: resourceContext.macros, + lookups: resourceContext.lookups, + }); + const splQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; + const inliningSummary = response.match(/## Inlining Summary[\s\S]*$/)?.[0] ?? ''; + if (splQuery) { + query = splQuery; + } + + // Check after replacing in case the replacements made it untranslatable + unsupportedComment = getUnsupportedComment(query); + if (unsupportedComment) { + return { comments: [unsupportedComment] }; + } + + return { inline_query: query, comments: [cleanMarkdown(inliningSummary)] }; + } + return { inline_query: query }; + }; +}; + +const getUnsupportedComment = (query: string): string | undefined => { + const unsupportedText = '## Translation Summary\nCan not create custom translation.\n'; + if (query.includes(' inputlookup')) { + return `${unsupportedText}Reason: \`inputlookup\` command is not supported.`; + } +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts new file mode 100644 index 0000000000000..49f2b2b57f479 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatPromptTemplate } from '@langchain/core/prompts'; +import type { RuleMigrationResources } from '../../../../../retrievers/rule_resource_retriever'; + +interface ResourceContext { + macros: string; + lookups: string; +} + +export const getResourcesContext = (resources: RuleMigrationResources): ResourceContext => { + const result: ResourceContext = { macros: '', lookups: '' }; + + // Process macros + if (resources.macro?.length) { + const macrosMap = resources.macro.reduce((acc, macro) => { + acc[macro.name] = macro.content; + return acc; + }, {} as Record); + + result.macros = JSON.stringify(macrosMap, null, 2); + } + + // Process lookups + if (resources.lookup?.length) { + const lookupsMap = resources.lookup.reduce((acc, lookup) => { + acc[lookup.name] = lookup.content; + return acc; + }, {} as Record); + + result.lookups = JSON.stringify(lookupsMap, null, 2); + } + + return result; +}; + +export const REPLACE_QUERY_RESOURCE_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an agent expert in Splunk SPL (Search Processing Language). +Your task is to replace macros and lookups syntax in a SPL query, using the actual content of the macros and lookup names provided to you. And write a summary of the replacement steps made. +Here are some context for you to reference for your task, read it carefully as you will get questions about it later: + + + + +You have to replace the macros syntax in the SPL query and use their value inline, if provided. + +Always follow the below guidelines when replacing macros: +- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments. +- Divide the query up into separate sections and go through each section one at a time to identify the macros used that need to be replaced, using one of two scenarios: + - The macro is provided in the list of available macros: Replace it using its actual content. + - The macro is not in the list of available macros: add a placeholder ("missing placeholder" from now on) in the query with the format [macro:(argumentCount)] including the [] keys, + Example: \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)] + +Having the following macros: + \`someSource\`: sourcetype="somesource" + \`searchTitle(1)\`: search title="$value$" + \`searchTitle\`: search title=* + \`searchType\`: search type=* + +And the following SPL query: + \`\`\`spl + \`someSource\` \`someFilter\` + | \`searchTitle("sometitle")\` + | \`searchType("sometype")\` + | \`anotherMacro("someParam","someOtherParam", 10)\` + | table * + \`\`\` + +The correct replacement would be: + \`\`\`spl + sourcetype="somesource" \`someFilter\` + | search title="sometitle" + | \`searchType("sometype")\` + | [macro:anotherMacro(3)] + | table * + \`\`\` + + + +You have to replace the lookup names in the SPL query with the Elastic name, if provided. + +Always follow the below guidelines when replacing lookups: +- Divide the query up into separate sections and go through each section one at a time to identify the lookups used that need to be replaced, using one of two scenarios: + - The lookup is provided in the list of available lookups: Replace the lookup name using its Elastic name provided. + - Remember the "_lookup" suffix in the lookup name in the query should be ignored when checking the list of available lookups + - The lookup is not in the list of available lookups: add a placeholder ("missing placeholder" from now on) in the query with the format [lookup:] including the [] keys, + Example: "lookup users uid OUTPUTNEW username, department" -> "[lookup:users]" + - The lookup is in the list but has empty name: omit the lookup from the query entirely, as if it was empty. To do so you can use the "eval" command to set the fields to empty strings. + +Having the following lookups: + "some_list": "lookup_some_list" + "another": "lookup_another-2" + "lookupName3": "" + +And the following SPL query: + \`\`\`spl + | lookup some_list name OUTPUT title + | lookup another_lookup name OUTPUT description + | lookup yet_another_lookup id OUTPUTNEW someField + | lookup lookupName3 uuid OUTPUTNEW group, name + \`\`\` + +The correct replacement would be: + \`\`\`spl + | lookup lookup_some_list name OUTPUT title + | lookup lookup_another-2 name OUTPUTNEW description + | [lookup:yet_another] + | eval group="", name="" + \`\`\` + + + +- The original and modified queries must be equivalent, except for the "missing placeholders". +- You must respond with the modified query inside a \`\`\`spl code block, followed by a summary of the replacement steps made in markdown format, starting with "## Inlining Summary". + + +`, + ], + [ + 'human', + `Go through the SPL query and replace all the macros and lookups provided: + +{macros} + + + +{lookups} + + + +\`\`\`spl +{query} +\`\`\` + + + +- First, the modified SPL query inside an \`\`\`spl code block. +- At the end, a step by step explanation of the replacements made in markdown format: + - Start with "## Inlining Summary" title + - Followed by information about the original query + - Then, the steps taken to replace the macros and lookups + - Finally, the modified SPL query + - Inside SPL language code blocks, Please add a line break before each pipe (|) character in the query. + - Make sure the Markdown is formatted correctly and the values properly escaped. + +`, + ], + ['ai', 'Please find the modified SPL query below:'], +]); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts index d562bb31b1628..786f737990661 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts @@ -9,10 +9,13 @@ import { ChatPromptTemplate } from '@langchain/core/prompts'; export const MATCH_INTEGRATION_PROMPT = ChatPromptTemplate.fromMessages([ [ 'system', - `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. -You will be provided with a Splunk Detection Rule name by the user, your goal is to try to find the most relevant Elastic Integration from the integration list below if any, and return either the most relevant. If none seems relevant you should always return empty. -Here are some context for you to reference for your task, read it carefully as you will get questions about it later: + `You are a Cybersecurity expert specializing in SIEM solutions. Your task is to assist in migrating detection rules from Splunk Security to Elastic Security by identifying the most appropriate Elastic Integration for a given Splunk rule. +Elastic Integrations are pre-built packages that ingest data from various sources into Elastic Security. +They enable Elastic Security detection rules to monitor environments, detect threats, and ensure a strong security posture. +Your goal is to identify the Elastic Integration that aligns best with the data source referenced in the provided Splunk rule. + +Here is the Elastic integrations context for you to reference for your task, read it carefully as you will get questions about it later: {integrations} @@ -28,19 +31,30 @@ Here are some context for you to reference for your task, read it carefully as y -- Always reply with a JSON object with the key "match" and the value being the most relevant matched integration title. Do not reply with anything else. -- Only reply with exact matches, if you are unsure or do not find a very confident match, always reply with an empty string value in the match key, do not guess or reply with anything else. -- If there is one elastic integration in the list that covers the relevant usecase, set the title of the matching integration as a value of the match key. Do not reply with anything else. -- If there are multiple elastic integrations in the list that cover the same usecase, answer with the most specific of them, for example if the rule is related to "Sysmon" then the Sysmon integration is more specific than Windows. +- Carefully analyze the Splunk Detection Rule data provided by the user. +- Match the data source in the Splunk rule to the most relevant Elastic Integration from the list provided above. +- If no related integration is found, reply with an empty string. +- Provide a concise reasoning summary for your decision, explaining why the selected integration is the best fit or why no suitable match was found. + +- Always reply with a JSON object with the key "match" and the value being the most relevant matched integration title, and a "summary" entry with the reasons behind the match. Do not reply with anything else. +- Only reply with exact matches or an empty string inside the "match" value, do not guess or reply with anything else. +- If there are multiple elastic integrations in the list that match, answer the most specific of them, for example if the rule is related to "Sysmon" then the Sysmon integration is more specific than Windows. +- Finally, write a "summary" in markdown format with the reasoning behind the integration matching, or otherwise, why none of the integrations suggested matched. Starting with "## Integration Matching Summary\n". +- Make sure the JSON object is formatted correctly and the values properly escaped. + + U: Linux Auditd Add User Account Type A: Please find the match JSON object below: \`\`\`json -{{"match": "auditd_manager"}} +{{ + "match": "auditd_manager", + "summary": "## Integration Matching Summary\\\nThe Splunk rule \"Linux Auditd Add User Account Type\" is matched with the \"auditd_manager\" integration because it ingests data from auditd logs which is the right data to detect user account creation on Linux systems." +}} \`\`\` `, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts index 74c9055bd7665..11c9069fc77ac 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts @@ -10,6 +10,7 @@ import type { RuleMigrationsRetriever } from '../../../../../retrievers'; import type { ChatModel } from '../../../../../util/actions_client_chat'; import type { GraphNode } from '../../types'; import { MATCH_INTEGRATION_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetRetrieveIntegrationsNodeParams { model: ChatModel; @@ -18,6 +19,7 @@ interface GetRetrieveIntegrationsNodeParams { interface GetMatchedIntegrationResponse { match: string; + summary: string; } export const getRetrieveIntegrationsNode = ({ @@ -28,16 +30,17 @@ export const getRetrieveIntegrationsNode = ({ const query = state.semantic_query; const integrations = await ruleMigrationsRetriever.integrations.getIntegrations(query); + if (integrations.length === 0) { + return { comments: ['## Integration Matching Summary\nNo related integration found.'] }; + } const outputParser = new JsonOutputParser(); const mostRelevantIntegration = MATCH_INTEGRATION_PROMPT.pipe(model).pipe(outputParser); - const elasticSecurityIntegrations = integrations.map((integration) => { - return { - title: integration.title, - description: integration.description, - }; - }); + const integrationsInfo = integrations.map((integration) => ({ + title: integration.title, + description: integration.description, + })); const splunkRule = { title: state.original_rule.title, description: state.original_rule.description, @@ -46,16 +49,20 @@ export const getRetrieveIntegrationsNode = ({ /* * Takes the most relevant integration from the array of integration(s) returned by the semantic query, returns either the most relevant or none. */ + const integrationsJson = JSON.stringify(integrationsInfo, null, 2); const response = (await mostRelevantIntegration.invoke({ - integrations: JSON.stringify(elasticSecurityIntegrations, null, 2), + integrations: integrationsJson, splunk_rule: JSON.stringify(splunkRule, null, 2), })) as GetMatchedIntegrationResponse; + + const comments = response.summary ? [cleanMarkdown(response.summary)] : undefined; + if (response.match) { const matchedIntegration = integrations.find((r) => r.title === response.match); if (matchedIntegration) { - return { integration: matchedIntegration }; + return { integration: matchedIntegration, comments }; } } - return {}; + return { comments }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts index 626251c3c8259..23a6829e6235d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts @@ -16,13 +16,8 @@ Here are some context for you to reference for your task, read it carefully as y {splunk_rule} - -If, in the SPL query, you find a macro call, mention it in the summary and add a placeholder in the query with the format [macro:(argumentCount)] including the [] keys, - Examples: - - \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)] - -If in an SPL query you identify a looku list call, it should be translated the following way: +If in an SPL query you identify a lookup call, it should be translated the following way: \`\`\`spl ... | lookup users uid OUTPUTNEW username, department \`\`\` @@ -43,13 +38,11 @@ Go through each step and part of the splunk rule and query while following the b - Go through each part of the SPL query and determine the steps required to produce the same end results using ES|QL. Only focus on translating the structure without modifying any of the field names. - Do NOT map any of the fields to the Elastic Common Schema (ECS), this will happen in a later step. - Always remember to translate any lookup list using the lookup_syntax above -- Always remember to replace macro call with the appropriate placeholder as defined in the macro info. - - Analyze the SPL query and identify the key components. - Do NOT translate the field names of the SPL query. -- Always start the resulting ES|QL query by filtering using FROM and with these index patterns: {indexPatterns}. +- Always start the resulting ES|QL query by filtering using FROM and with these index pattern: {indexPatterns}. - Always remember to translate any lookup list using the lookup_syntax above - Always remember to replace macro call with the appropriate placeholder as defined in the macro info. @@ -57,5 +50,7 @@ Go through each step and part of the splunk rule and query while following the b - First, the ES|QL query inside an \`\`\`esql code block. - At the end, the summary of the translation process followed in markdown, starting with "## Translation Summary". + - Inside SPL language code blocks, Please add a line break before each pipe (|) character in the query. + - Make sure the Markdown is formatted correctly and the values properly escaped. `); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts index 346df02714b67..d360b6a1c246d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts @@ -7,10 +7,10 @@ import type { Logger } from '@kbn/core/server'; import type { InferenceClient } from '@kbn/inference-plugin/server'; -import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_caller'; import type { GraphNode } from '../../types'; import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetTranslateRuleNodeParams { inferenceClient: InferenceClient; @@ -42,30 +42,17 @@ export const getTranslateRuleNode = ({ }); const response = await esqlKnowledgeBaseCaller(prompt); - const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; const translationSummary = response.match(/## Translation Summary[\s\S]*$/)?.[0] ?? ''; - const translationResult = getTranslationResult(esqlQuery); - return { response, - comments: [translationSummary], - translation_result: translationResult, + comments: [cleanMarkdown(translationSummary)], elastic_rule: { - title: state.original_rule.title, integration_id: integrationId, - description: state.original_rule.description, - severity: 'low', query: esqlQuery, query_language: 'esql', }, }; }; }; - -const getTranslationResult = (esqlQuery: string): RuleTranslationResult => { - if (esqlQuery.match(/\[(macro):[\s\S]*\]/)) { - return RuleTranslationResult.PARTIAL; - } - return RuleTranslationResult.FULL; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts similarity index 78% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/index.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts index 6e7762633b7fd..919c3c4f18b17 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts @@ -4,4 +4,4 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -export { getFilterIndexPatternsNode } from './filter_index_patterns'; +export { translationResultNode } from './translation_result'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts new file mode 100644 index 0000000000000..c70b4d3445ec7 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; +import type { GraphNode } from '../../types'; + +export const translationResultNode: GraphNode = async (state) => { + // Set defaults + const elasticRule = { + title: state.original_rule.title, + description: state.original_rule.description || state.original_rule.title, + severity: 'low', + ...state.elastic_rule, + }; + + const query = elasticRule.query; + let translationResult; + + if (!query) { + translationResult = RuleTranslationResult.UNTRANSLATABLE; + } else { + if (query.startsWith('FROM logs-*')) { + elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]'); + translationResult = RuleTranslationResult.PARTIAL; + } else if (state.validation_errors?.esql_errors) { + translationResult = RuleTranslationResult.PARTIAL; + } else if (query.match(/\[(macro|lookup):.*?\]/)) { + translationResult = RuleTranslationResult.PARTIAL; + } else { + translationResult = RuleTranslationResult.FULL; + } + } + + return { + elastic_rule: elasticRule, + translation_result: translationResult, + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts index 6f97678a04558..7d5e2104a3ae4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts @@ -23,21 +23,33 @@ export const getValidationNode = ({ logger }: GetValidationNodeParams): GraphNod const query = state.elastic_rule.query; // We want to prevent infinite loops, so we increment the iterations counter for each validation run. - const currentIteration = ++state.validation_errors.iterations; + const currentIteration = state.validation_errors.iterations + 1; let esqlErrors: string = ''; - if (!isEmpty(query)) { - const { errors, isEsqlQueryAggregating, hasMetadataOperator } = parseEsqlQuery(query); - if (!isEmpty(errors)) { - esqlErrors = JSON.stringify(errors); - } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { - esqlErrors = - 'Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.'; + try { + const sanitizedQuery = query ? removePlaceHolders(query) : ''; + if (!isEmpty(sanitizedQuery)) { + const { errors, isEsqlQueryAggregating, hasMetadataOperator } = + parseEsqlQuery(sanitizedQuery); + if (!isEmpty(errors)) { + esqlErrors = JSON.stringify(errors); + } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { + esqlErrors = `Queries that do't use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`; + } } + if (esqlErrors) { + logger.debug(`ESQL query validation failed: ${esqlErrors}`); + } + } catch (error) { + esqlErrors = error.message; + logger.info(`Error parsing ESQL query: ${error.message}`); } - if (esqlErrors) { - logger.debug(`ESQL query validation failed: ${esqlErrors}`); - } - return { validation_errors: { iterations: currentIteration, esql_errors: esqlErrors } }; }; }; + +function removePlaceHolders(query: string): string { + return query + .replace(/\[indexPattern\]/g, 'logs-*') // Replace the indexPattern placeholder with logs-* + .replaceAll(/\[(macro|lookup):.*?\]/g, '') // Removes any macro or lookup placeholders + .replaceAll(/\n(\s*?\|\s*?\n)*/g, '\n'); // Removes any empty lines with | (pipe) alone after removing the placeholders +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts index 873f1880d2252..9282163ab0002 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts @@ -9,11 +9,11 @@ import type { BaseMessage } from '@langchain/core/messages'; import { Annotation, messagesStateReducer } from '@langchain/langgraph'; import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; import type { - ElasticRule, + ElasticRulePartial, OriginalRule, RuleMigration, } from '../../../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { Integration } from '../../../../types'; +import type { RuleMigrationIntegration } from '../../../../types'; import type { TranslateRuleValidationErrors } from './types'; export const translateRuleState = Annotation.Root({ @@ -22,9 +22,9 @@ export const translateRuleState = Annotation.Root({ default: () => [], }), original_rule: Annotation(), - integration: Annotation({ + integration: Annotation({ reducer: (current, value) => value ?? current, - default: () => ({} as Integration), + default: () => ({} as RuleMigrationIntegration), }), translation_finalized: Annotation({ reducer: (current, value) => value ?? current, @@ -38,9 +38,9 @@ export const translateRuleState = Annotation.Root({ reducer: (current, value) => value ?? current, default: () => '', }), - elastic_rule: Annotation({ + elastic_rule: Annotation({ reducer: (state, action) => ({ ...state, ...action }), - default: () => ({} as ElasticRule), + default: () => ({}), }), validation_errors: Annotation({ reducer: (current, value) => value ?? current, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts index 2aa01a9c9c41d..2adb942e0c419 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts @@ -7,6 +7,7 @@ import { MockRuleMigrationsDataClient } from '../../data/__mocks__/mocks'; import { IntegrationRetriever } from './integration_retriever'; +import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever'; describe('IntegrationRetriever', () => { let integrationRetriever: IntegrationRetriever; @@ -19,7 +20,9 @@ describe('IntegrationRetriever', () => { elser_embedding: 'elser_embedding', }; beforeEach(() => { - integrationRetriever = new IntegrationRetriever(mockRuleMigrationsDataClient); + integrationRetriever = new IntegrationRetriever({ + data: mockRuleMigrationsDataClient, + } as RuleMigrationsRetrieverClients); mockRuleMigrationsDataClient.integrations.retrieveIntegrations.mockImplementation( async (_: string) => { return mockIntegrationItem; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts index 7913e2c438081..18ddb51a5b83b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts @@ -5,19 +5,18 @@ * 2.0. */ -import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; -import type { Integration } from '../../types'; +import type { RuleMigrationIntegration } from '../../types'; +import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever'; export class IntegrationRetriever { - constructor(private readonly dataClient: RuleMigrationsDataClient) {} + constructor(private readonly clients: RuleMigrationsRetrieverClients) {} - public async getIntegrations(semanticString: string): Promise { - return this.integrationRetriever(semanticString); + public async populateIndex() { + // TODO: use Fleet API client for integration retrieval as an argument once feature is available + return this.clients.data.integrations.create(); } - private integrationRetriever = async (semanticString: string): Promise => { - const integrations = await this.dataClient.integrations.retrieveIntegrations(semanticString); - - return integrations; - }; + public async getIntegrations(semanticString: string): Promise { + return this.clients.data.integrations.retrieveIntegrations(semanticString); + } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts index 0350bb43fe5ab..03fd24d4e6b4d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts @@ -5,25 +5,27 @@ * 2.0. */ -import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; import type { RuleMigrationPrebuiltRule } from '../../types'; +import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever'; export class PrebuiltRulesRetriever { - constructor(private readonly dataClient: RuleMigrationsDataClient) {} + constructor(private readonly clients: RuleMigrationsRetrieverClients) {} + + // TODO: + // 1. Implement the `initialize` method to retrieve prebuilt rules and keep them in memory. + // 2. Improve the `retrieveRules` method to return the real prebuilt rules instead of the ELSER index doc. + + public async populateIndex() { + return this.clients.data.prebuiltRules.create({ + rulesClient: this.clients.rules, + soClient: this.clients.savedObjects, + }); + } public async getRules( semanticString: string, techniqueIds: string ): Promise { - return this.prebuiltRulesRetriever(semanticString, techniqueIds); + return this.clients.data.prebuiltRules.retrieveRules(semanticString, techniqueIds); } - - private prebuiltRulesRetriever = async ( - semanticString: string, - techniqueIds: string - ): Promise => { - const rules = await this.dataClient.prebuiltRules.retrieveRules(semanticString, techniqueIds); - - return rules; - }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 29852558cda48..5616bfd4fb26b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -12,7 +12,7 @@ import { IntegrationRetriever } from './integration_retriever'; import { PrebuiltRulesRetriever } from './prebuilt_rules_retriever'; import { RuleResourceRetriever } from './rule_resource_retriever'; -interface RuleMigrationsRetrieverDeps { +export interface RuleMigrationsRetrieverClients { data: RuleMigrationsDataClient; rules: RulesClient; savedObjects: SavedObjectsClientContract; @@ -23,22 +23,17 @@ export class RuleMigrationsRetriever { public readonly integrations: IntegrationRetriever; public readonly prebuiltRules: PrebuiltRulesRetriever; - constructor(migrationId: string, private readonly clients: RuleMigrationsRetrieverDeps) { - this.resources = new RuleResourceRetriever(migrationId, this.clients.data); - this.integrations = new IntegrationRetriever(this.clients.data); - this.prebuiltRules = new PrebuiltRulesRetriever(this.clients.data); + constructor(migrationId: string, clients: RuleMigrationsRetrieverClients) { + this.resources = new RuleResourceRetriever(migrationId, clients.data); + this.integrations = new IntegrationRetriever(clients); + this.prebuiltRules = new PrebuiltRulesRetriever(clients); } public async initialize() { await Promise.all([ this.resources.initialize(), - // Populates the indices used for RAG searches on prebuilt rules and integrations. - this.clients.data.prebuiltRules.create({ - rulesClient: this.clients.rules, - soClient: this.clients.savedObjects, - }), - // Will use Fleet API client for integration retrieval as an argument once feature is available - this.clients.data.integrations.create(), + this.prebuiltRules.populateIndex(), + this.integrations.populateIndex(), ]).catch((error) => { throw new Error(`Failed to initialize RuleMigrationsRetriever: ${error}`); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts index 1e02eec2315e7..b0c50aa086ed3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts @@ -55,10 +55,10 @@ describe('RuleResourceRetriever', () => { expect(result).toEqual({}); }); - it('returns matching macro and list resources', async () => { + it('returns matching macro and lookup resources', async () => { const mockExistingResources = { macro: { macro1: { name: 'macro1', type: 'macro' } }, - list: { list1: { name: 'list1', type: 'list' } }, + lookup: { lookup1: { name: 'lookup1', type: 'lookup' } }, }; // Inject existing resources manually // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -66,7 +66,7 @@ describe('RuleResourceRetriever', () => { const mockResourcesIdentified = [ { name: 'macro1', type: 'macro' as const }, - { name: 'list1', type: 'list' as const }, + { name: 'lookup1', type: 'lookup' as const }, ]; MockResourceIdentifier.mockImplementation(() => ({ fromOriginalRule: jest.fn().mockReturnValue(mockResourcesIdentified), @@ -78,7 +78,7 @@ describe('RuleResourceRetriever', () => { const result = await retriever.getResources(originalRule); expect(result).toEqual({ macro: [{ name: 'macro1', type: 'macro' }], - list: [{ name: 'list1', type: 'list' }], + lookup: [{ name: 'lookup1', type: 'lookup' }], }); }); @@ -90,9 +90,9 @@ describe('RuleResourceRetriever', () => { macro1: { name: 'macro1', type: 'macro' }, macro2: { name: 'macro2', type: 'macro' }, }, - list: { - list1: { name: 'list1', type: 'list' }, - list2: { name: 'list2', type: 'list' }, + lookup: { + lookup1: { name: 'lookup1', type: 'lookup' }, + lookup2: { name: 'lookup2', type: 'lookup' }, }, }; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -100,12 +100,12 @@ describe('RuleResourceRetriever', () => { const mockResourcesIdentifiedFromRule = [ { name: 'macro1', type: 'macro' as const }, - { name: 'list1', type: 'list' as const }, + { name: 'lookup1', type: 'lookup' as const }, ]; const mockNestedResources = [ { name: 'macro2', type: 'macro' as const }, - { name: 'list2', type: 'list' as const }, + { name: 'lookup2', type: 'lookup' as const }, ]; MockResourceIdentifier.mockImplementation(() => ({ @@ -119,9 +119,9 @@ describe('RuleResourceRetriever', () => { { name: 'macro1', type: 'macro' }, { name: 'macro2', type: 'macro' }, ], - list: [ - { name: 'list1', type: 'list' }, - { name: 'list2', type: 'list' }, + lookup: [ + { name: 'lookup1', type: 'lookup' }, + { name: 'lookup2', type: 'lookup' }, ], }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts index b89939e199e5a..6fb9951c1985b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts @@ -21,7 +21,7 @@ export type RuleMigrationResources = Partial< >; interface ExistingResources { macro: Record; - list: Record; + lookup: Record; } export class RuleResourceRetriever { @@ -35,10 +35,10 @@ export class RuleResourceRetriever { public async initialize(): Promise { const batches = this.dataClient.resources.searchBatches( this.migrationId, - { filters: { hasContent: true } } + { filters: { hasContent: true } } // filters out missing (undefined) content resources, empty strings content will be included ); - const existingRuleResources: ExistingResources = { macro: {}, list: {} }; + const existingRuleResources: ExistingResources = { macro: {}, lookup: {} }; let resources; do { resources = await batches.next(); @@ -60,19 +60,19 @@ export class RuleResourceRetriever { const resourcesIdentifiedFromRule = resourceIdentifier.fromOriginalRule(originalRule); const macrosFound = new Map(); - const listsFound = new Map(); + const lookupsFound = new Map(); resourcesIdentifiedFromRule.forEach((resource) => { const existingResource = existingResources[resource.type][resource.name]; if (existingResource) { if (resource.type === 'macro') { macrosFound.set(resource.name, existingResource); - } else if (resource.type === 'list') { - listsFound.set(resource.name, existingResource); + } else if (resource.type === 'lookup') { + lookupsFound.set(resource.name, existingResource); } } }); - const resourcesFound = [...macrosFound.values(), ...listsFound.values()]; + const resourcesFound = [...macrosFound.values(), ...lookupsFound.values()]; if (!resourcesFound.length) { return {}; } @@ -88,8 +88,8 @@ export class RuleResourceRetriever { nestedResourcesFound.push(existingResource); if (resource.type === 'macro') { macrosFound.set(resource.name, existingResource); - } else if (resource.type === 'list') { - listsFound.set(resource.name, existingResource); + } else if (resource.type === 'lookup') { + lookupsFound.set(resource.name, existingResource); } } }); @@ -97,7 +97,7 @@ export class RuleResourceRetriever { return { ...(macrosFound.size > 0 ? { macro: Array.from(macrosFound.values()) } : {}), - ...(listsFound.size > 0 ? { list: Array.from(listsFound.values()) } : {}), + ...(lookupsFound.size > 0 ? { lookup: Array.from(lookupsFound.values()) } : {}), }; } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index 663a8f5218f33..3c932e97977e7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -111,15 +111,19 @@ export class RuleMigrationsTaskClient { await Promise.all( ruleMigrations.map(async (ruleMigration) => { this.logger.debug(`Starting migration of rule "${ruleMigration.original_rule.title}"`); + if (ruleMigration.elastic_rule?.id) { + await this.data.rules.saveCompleted(ruleMigration); + return; // skip already installed rules + } try { const start = Date.now(); - const invocation = agent.invoke( - { original_rule: ruleMigration.original_rule }, - config - ); + const invocationData = { original_rule: ruleMigration.original_rule }; + // using withAbortRace is a workaround for the issue with the langGraph signal not working properly - const migrationResult = await withAbortRace(invocation); + const migrationResult = await withAbortRace( + agent.invoke(invocationData, config) + ); const duration = (Date.now() - start) / 1000; this.logger.debug( @@ -193,16 +197,16 @@ export class RuleMigrationsTaskClient { rules: rulesClient, savedObjects: soClient, }); + await ruleMigrationsRetriever.initialize(); - const agent = getRuleMigrationAgent({ + return getRuleMigrationAgent({ connectorId, model, inferenceClient, ruleMigrationsRetriever, logger: this.logger, }); - return agent; } /** Updates all the rules in a migration to be re-executed */ @@ -217,6 +221,7 @@ export class RuleMigrationsTaskClient { await this.data.rules.updateStatus(migrationId, filter, SiemMigrationStatus.PENDING, { refresh: true, }); + // await this.data.rules.updateRetry(migrationId); return { updated: true }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts new file mode 100644 index 0000000000000..4dad4235ad4e0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const cleanMarkdown = (markdown: string): string => { + // Use languages known by the code block plugin + return markdown.replaceAll('```esql', '```sql').replaceAll('```spl', '```splunk-spl'); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index f13a407ee2500..d1df32ebbcb61 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -15,9 +15,9 @@ export type Stored = T & { id: string }; export type StoredRuleMigration = Stored; export type StoredRuleMigrationResource = Stored; -export interface Integration { - title: string; +export interface RuleMigrationIntegration { id: string; + title: string; description: string; data_streams: Array<{ dataset: string; title: string; index_pattern: string }>; elser_embedding: string; @@ -25,7 +25,7 @@ export interface Integration { export interface RuleMigrationPrebuiltRule { rule_id: string; - installedRuleId?: string; + installed_rule_id?: string; name: string; description: string; elser_embedding: string;