From ad9fd9071b8834cdf0d6a6d76c96c4b9fa370308 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 19 Dec 2024 15:38:55 +0100 Subject: [PATCH 1/9] lookus saved as indices --- .../common/siem_migrations/constants.ts | 2 - .../model/api/rules/rule_migration.gen.ts | 3 +- .../api/rules/rule_migration.schema.yaml | 2 +- .../model/rule_migration.gen.ts | 28 ++- .../model/rule_migration.schema.yaml | 24 ++- .../siem_migrations/rules/resources/index.ts | 31 ++-- .../splunk/splunk_identifier.test.ts | 44 +++-- .../resources/splunk/splunk_identifier.ts | 26 +-- .../siem_migrations/rules/resources/types.ts | 12 +- .../data_input_flyout/data_input_flyout.tsx | 6 +- .../steps/lookups/lookups_data_input.tsx | 7 +- .../lookups_file_upload.tsx | 2 +- .../sub_steps/missing_lookups_list/index.tsx | 11 +- .../missing_lookups_list.tsx | 36 ++-- .../components/data_input_flyout/types.ts | 4 +- .../hooks/use_get_missing_resources.ts | 4 +- .../rules/service/rule_migrations_service.ts | 4 +- .../rules/api/resources/missing.ts | 4 +- .../rules/api/resources/upsert.ts | 13 +- .../siem_migrations/rules/api/util/lookups.ts | 107 ++++++++++++ .../rules/data/rule_migrations_data_client.ts | 19 +- .../rule_migrations_data_lookups_client.ts | 67 +++++++ .../rule_migrations_data_resources_client.ts | 6 +- .../data/rule_migrations_data_service.test.ts | 67 ++++--- .../data/rule_migrations_data_service.ts | 86 ++++++--- .../rules/siem_rule_migrations_service.ts | 4 +- .../siem_migrations/rules/task/agent/graph.ts | 15 +- .../match_prebuilt_rule.ts | 10 -- .../task/agent/nodes/process_query/prompts.ts | 117 ------------- .../agent/sub_graphs/translate_rule/graph.ts | 19 +- .../nodes/ecs_mapping/ecs_mapping.ts | 2 +- .../filter_index_patterns.ts | 40 ----- .../nodes/inline_query}/index.ts | 2 +- .../nodes/inline_query/inline_query.ts} | 13 +- .../nodes/inline_query/prompts.ts | 163 ++++++++++++++++++ .../retrieve_integrations.ts | 3 + .../nodes/translate_rule/prompts.ts | 74 +++++--- .../nodes/translate_rule/translate_rule.ts | 2 +- .../index.ts | 2 +- .../translation_result/translation_result.ts | 40 +++++ .../nodes/validation/validation.ts | 16 +- .../rule_resource_retriever.test.ts | 24 +-- .../retrievers/rule_resource_retriever.ts | 20 +-- 43 files changed, 757 insertions(+), 424 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts rename x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/{nodes/process_query => sub_graphs/translate_rule/nodes/inline_query}/index.ts (82%) rename x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/{nodes/process_query/process_query.ts => sub_graphs/translate_rule/nodes/inline_query/inline_query.ts} (78%) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts rename x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/{filter_index_patterns => translation_result}/index.ts (78%) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts index 88ed777c21d69..c65c3cd9e28c8 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts @@ -60,5 +60,3 @@ export const DEFAULT_TRANSLATION_FIELDS = { to: 'now', interval: '5m', } as const; - -export const EMPTY_RESOURCE_PLACEHOLDER = ''; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts index 47c06e1e02c7a..75cd7c81e37e1 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts @@ -27,6 +27,7 @@ import { RuleMigrationResourceData, RuleMigrationResourceType, RuleMigrationResource, + RuleMigrationResourceBase, } from '../../rule_migration.gen'; import { NonEmptyString } from '../../../../api/model/primitives.gen'; import { ConnectorId, LangSmithOptions } from '../../common.gen'; @@ -138,7 +139,7 @@ export type GetRuleMigrationResourcesMissingRequestParamsInput = z.input< export type GetRuleMigrationResourcesMissingResponse = z.infer< typeof GetRuleMigrationResourcesMissingResponse >; -export const GetRuleMigrationResourcesMissingResponse = z.array(RuleMigrationResourceData); +export const GetRuleMigrationResourcesMissingResponse = z.array(RuleMigrationResourceBase); export type GetRuleMigrationStatsRequestParams = z.infer; export const GetRuleMigrationStatsRequestParams = z.object({ diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml index 69e43b57dabd3..07d16c77f7171 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml @@ -512,4 +512,4 @@ paths: type: array description: The identified resources missing items: - $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData' + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceBase' diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 27db56aedb451..c55511f83a3ae 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -356,35 +356,49 @@ export const UpdateRuleMigrationData = z.object({ * The type of the rule migration resource. */ export type RuleMigrationResourceType = z.infer; -export const RuleMigrationResourceType = z.enum(['macro', 'list']); +export const RuleMigrationResourceType = z.enum(['macro', 'lookup']); export type RuleMigrationResourceTypeEnum = typeof RuleMigrationResourceType.enum; export const RuleMigrationResourceTypeEnum = RuleMigrationResourceType.enum; /** - * The rule migration resource data provided by the vendor. + * The rule migration resource basic information. */ -export type RuleMigrationResourceData = z.infer; -export const RuleMigrationResourceData = z.object({ +export type RuleMigrationResourceBase = z.infer; +export const RuleMigrationResourceBase = z.object({ type: RuleMigrationResourceType, /** * The resource name identifier. */ name: z.string(), +}); + +export type RuleMigrationResourceContent = z.infer; +export const RuleMigrationResourceContent = z.object({ /** - * The resource content value. + * The resource content value. Can be an empty string. */ - content: z.string().optional(), + content: z.string(), /** * The resource arbitrary metadata. */ metadata: z.object({}).optional(), }); +/** + * The rule migration resource data. + */ +export type RuleMigrationResourceData = z.infer; +export const RuleMigrationResourceData = RuleMigrationResourceBase.merge( + RuleMigrationResourceContent +); + /** * The rule migration resource document object. */ export type RuleMigrationResource = z.infer; -export const RuleMigrationResource = RuleMigrationResourceData.merge( +export const RuleMigrationResource = RuleMigrationResourceBase.merge( + RuleMigrationResourceContent.partial() +).merge( z.object({ /** * The rule resource migration id diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index f3a85b3e37443..0361c3a01d0cf 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -327,11 +327,11 @@ components: description: The type of the rule migration resource. enum: - macro # Reusable part a query that can be customized and called from multiple rules - - list # A list of values that can be used inside queries reused in different rules + - lookup # A list of values that can be used inside queries as data enrichment or data source - RuleMigrationResourceData: + RuleMigrationResourceBase: type: object - description: The rule migration resource data provided by the vendor. + description: The rule migration resource basic information. required: - type - name @@ -341,17 +341,31 @@ components: name: type: string description: The resource name identifier. + + RuleMigrationResourceContent: + type: object + required: + - content + properties: content: type: string - description: The resource content value. + description: The resource content value. Can be an empty string. metadata: type: object description: The resource arbitrary metadata. + RuleMigrationResourceData: + description: The rule migration resource data. + allOf: + - $ref: '#/components/schemas/RuleMigrationResourceBase' + - $ref: '#/components/schemas/RuleMigrationResourceContent' + RuleMigrationResource: description: The rule migration resource document object. allOf: - - $ref: '#/components/schemas/RuleMigrationResourceData' + - $ref: '#/components/schemas/RuleMigrationResourceBase' + - $ref: '#/components/schemas/RuleMigrationResourceContent' + x-modify: partial - type: object required: - id diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts index 8ec7adf050bf3..4b6472eb0e55c 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts @@ -9,8 +9,9 @@ import type { OriginalRule, OriginalRuleVendor, RuleMigrationResourceData, + RuleMigrationResourceBase, } from '../../model/rule_migration.gen'; -import type { ResourceIdentifiers, RuleResource } from './types'; +import type { ResourceIdentifiers } from './types'; import { splResourceIdentifiers } from './splunk'; const ruleResourceIdentifiers: Record = { @@ -29,48 +30,48 @@ export class ResourceIdentifier { this.identifiers = ruleResourceIdentifiers[vendor]; } - public fromOriginalRule(originalRule: OriginalRule): RuleResource[] { + public fromOriginalRule(originalRule: OriginalRule): RuleMigrationResourceBase[] { return this.identifiers.fromOriginalRule(originalRule); } - public fromResource(resource: RuleMigrationResourceData): RuleResource[] { + public fromResource(resource: RuleMigrationResourceData): RuleMigrationResourceBase[] { return this.identifiers.fromResource(resource); } - public fromOriginalRules(originalRules: OriginalRule[]): RuleResource[] { - const lists = new Set(); + public fromOriginalRules(originalRules: OriginalRule[]): RuleMigrationResourceBase[] { + const lookups = new Set(); const macros = new Set(); originalRules.forEach((rule) => { const resources = this.identifiers.fromOriginalRule(rule); resources.forEach((resource) => { if (resource.type === 'macro') { macros.add(resource.name); - } else if (resource.type === 'list') { - lists.add(resource.name); + } else if (resource.type === 'lookup') { + lookups.add(resource.name); } }); }); return [ - ...Array.from(macros).map((name) => ({ type: 'macro', name })), - ...Array.from(lists).map((name) => ({ type: 'list', name })), + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), ]; } - public fromResources(resources: RuleMigrationResourceData[]): RuleResource[] { - const lists = new Set(); + public fromResources(resources: RuleMigrationResourceData[]): RuleMigrationResourceBase[] { + const lookups = new Set(); const macros = new Set(); resources.forEach((resource) => { this.identifiers.fromResource(resource).forEach((identifiedResource) => { if (identifiedResource.type === 'macro') { macros.add(identifiedResource.name); - } else if (identifiedResource.type === 'list') { - lists.add(identifiedResource.name); + } else if (identifiedResource.type === 'lookup') { + lookups.add(identifiedResource.name); } }); }); return [ - ...Array.from(macros).map((name) => ({ type: 'macro', name })), - ...Array.from(lists).map((name) => ({ type: 'list', name })), + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), ]; } } diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts index 5d144e5b8a38f..58c48632a51c8 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts @@ -47,12 +47,26 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ - { type: 'list', name: 'my_lookup_table' }, - { type: 'list', name: 'other_lookup_list' }, - { type: 'list', name: 'third_lookup' }, + { type: 'lookup', name: 'my_lookup_table' }, + { type: 'lookup', name: 'other_lookup_list' }, + { type: 'lookup', name: 'third' }, ]); }); + it('should extract lookup from inputlookup tables correctly', () => { + const query = 'inputlookup my_lookup_table WHERE field="value"'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([{ type: 'lookup', name: 'my_lookup_table' }]); + }); + + it('should extract lookup from inputlookup with modifiers correctly', () => { + const query = 'inputlookup append=T start=10 my_lookup_table WHERE field="value"'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([{ type: 'lookup', name: 'my_lookup_table' }]); + }); + it('should extract both macros and lookup tables correctly', () => { const query = '`macro_one` some search command | lookup my_lookup_table field AS alias OUTPUT new_field | lookup other_lookup_list | lookup third_lookup'; @@ -60,9 +74,9 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, - { type: 'list', name: 'other_lookup_list' }, - { type: 'list', name: 'third_lookup' }, + { type: 'lookup', name: 'my_lookup_table' }, + { type: 'lookup', name: 'other_lookup_list' }, + { type: 'lookup', name: 'third' }, ]); }); @@ -72,11 +86,11 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ - { type: 'list', name: 'my_lookup_1' }, - { type: 'list', name: 'my_lookup_2' }, - { type: 'list', name: 'my_lookup_3' }, - { type: 'list', name: 'my_lookup_4' }, - { type: 'list', name: 'my_lookup_5' }, + { type: 'lookup', name: 'my_lookup_1' }, + { type: 'lookup', name: 'my_lookup_2' }, + { type: 'lookup', name: 'my_lookup_3' }, + { type: 'lookup', name: 'my_lookup_4' }, + { type: 'lookup', name: 'my_lookup_5' }, ]); }); @@ -96,7 +110,7 @@ describe('splResourceIdentifier', () => { { type: 'macro', name: 'macro_one' }, { type: 'macro', name: 'my_lookup_table' }, { type: 'macro', name: 'third_macro' }, - { type: 'list', name: 'real_lookup_list' }, + { type: 'lookup', name: 'real_lookup_list' }, ]); }); @@ -107,7 +121,7 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, + { type: 'lookup', name: 'my_lookup_table' }, ]); }); @@ -118,7 +132,7 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, + { type: 'lookup', name: 'my_lookup_table' }, ]); }); @@ -129,7 +143,7 @@ describe('splResourceIdentifier', () => { const result = splResourceIdentifier(query); expect(result).toEqual([ { type: 'macro', name: 'macro_one' }, - { type: 'list', name: 'my_lookup_table' }, + { type: 'lookup', name: 'my_lookup_table' }, ]); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts index 2ecc43321b11f..35373ba155460 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts @@ -12,16 +12,17 @@ * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker */ -import type { ResourceIdentifier, RuleResource } from '../types'; +import type { RuleMigrationResourceBase } from '../../../model/rule_migration.gen'; +import type { ResourceIdentifier } from '../types'; -const listRegex = /\b(?:lookup)\s+([\w-]+)\b/g; // Captures only the lookup name +const lookupRegex = /\b(?:lookup|inputlookup)\s+([\w-]+)\b/g; // Captures only the lookup name const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments export const splResourceIdentifier: ResourceIdentifier = (input) => { - // sanitize the query to avoid mismatching macro and list names inside comments or literal strings + // sanitize the query to avoid mismatching macro and lookup names inside comments or literal strings const sanitizedInput = sanitizeInput(input); - const resources: RuleResource[] = []; + const resources: RuleMigrationResourceBase[] = []; let macroMatch; while ((macroMatch = macrosRegex.exec(sanitizedInput)) !== null) { const macroName = macroMatch[1] as string; @@ -31,26 +32,29 @@ export const splResourceIdentifier: ResourceIdentifier = (input) => { resources.push({ type: 'macro', name: macroWithArgs }); } - let listMatch; - while ((listMatch = listRegex.exec(sanitizedInput)) !== null) { - resources.push({ type: 'list', name: listMatch[1] }); + let lookupMatch; + while ((lookupMatch = lookupRegex.exec(sanitizedInput)) !== null) { + resources.push({ type: 'lookup', name: lookupMatch[1].replace(/_lookup$/, '') }); // Remove _lookup suffix } return resources; }; -// Comments should be removed before processing the query to avoid matching macro and list names inside them +// Comments should be removed before processing the query to avoid matching macro and lookup names inside them const commentRegex = /```.*?```/g; -// Literal strings should be replaced with a placeholder to avoid matching macro and list names inside them +// Literal strings should be replaced with a placeholder to avoid matching macro and lookup names inside them const doubleQuoteStrRegex = /".*?"/g; const singleQuoteStrRegex = /'.*?'/g; +// inputlookup operator can have modifiers like appent=T before the lookup name, we need to remove them +const inputlookupModifiers = /\binputlookup\b(\s+\w+=\w+)*/gi; // lookup operator can have modifiers like local=true or update=false before the lookup name, we need to remove them -const lookupModifiers = /\blookup\b\s+((local|update)=\s*(?:true|false)\s*)+/gi; +const lookupModifiers = /\blookup\b(\s+\w+=\w+)*/gi; const sanitizeInput = (query: string) => { return query .replaceAll(commentRegex, '') .replaceAll(doubleQuoteStrRegex, '"literal"') .replaceAll(singleQuoteStrRegex, "'literal'") - .replaceAll(lookupModifiers, 'lookup '); + .replaceAll(lookupModifiers, 'lookup ') + .replaceAll(inputlookupModifiers, 'inputlookup '); }; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts index 70c6e3b72124f..db4df43cc3df6 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts @@ -7,17 +7,13 @@ import type { OriginalRule, + RuleMigrationResourceBase, RuleMigrationResourceData, - RuleMigrationResourceType, } from '../../model/rule_migration.gen'; -export interface RuleResource { - type: RuleMigrationResourceType; - name: string; -} -export type ResourceIdentifier = (input: string) => RuleResource[]; +export type ResourceIdentifier = (input: string) => RuleMigrationResourceBase[]; export interface ResourceIdentifiers { - fromOriginalRule: (originalRule: OriginalRule) => RuleResource[]; - fromResource: (resource: RuleMigrationResourceData) => RuleResource[]; + fromOriginalRule: (originalRule: OriginalRule) => RuleMigrationResourceBase[]; + fromResource: (resource: RuleMigrationResourceData) => RuleMigrationResourceBase[]; } diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx index 9062e3a6b21e8..4f8e73f43f6c3 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx @@ -19,7 +19,7 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import type { - RuleMigrationResourceData, + RuleMigrationResourceBase, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { RulesDataInput } from './steps/rules/rules_data_input'; @@ -60,12 +60,12 @@ export const MigrationDataInputFlyout = React.memo { + (missingResources: RuleMigrationResourceBase[]) => { const newMissingResourcesIndexed = missingResources.reduce( (acc, { type, name }) => { if (type === 'macro') { acc.macros.push(name); - } else if (type === 'list') { + } else if (type === 'lookup') { acc.lookups.push(name); } return acc; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx index a8fca750ce5da..cfe88caafc652 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/lookups_data_input.tsx @@ -15,7 +15,6 @@ import { EuiTitle, } from '@elastic/eui'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import { EMPTY_RESOURCE_PLACEHOLDER } from '../../../../../../../common/siem_migrations/constants'; import type { RuleMigrationResourceData, RuleMigrationTaskStats, @@ -101,14 +100,12 @@ export const LookupsDataInputSubSteps = React.memo((lookups) => { setUploadedLookups((prevUploadedLookups) => ({ ...prevUploadedLookups, - ...Object.fromEntries( - lookups.map((lookup) => [lookup.name, lookup.content ?? EMPTY_RESOURCE_PLACEHOLDER]) - ), + ...Object.fromEntries(lookups.map((lookup) => [lookup.name, lookup.content])), })); }, []); useEffect(() => { - if (missingLookups.every((lookupName) => uploadedLookups[lookupName])) { + if (missingLookups.every((lookupName) => uploadedLookups[lookupName] != null)) { setSubStep(END); onAllLookupsCreated(); } diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx index 6ea9562f24cce..0dc05493f7469 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/lookups_file_upload/lookups_file_upload.tsx @@ -77,7 +77,7 @@ export const LookupsFileUpload = React.memo( } const name = file.name.replace(/\.[^/.]+$/, '').trim(); - resolve({ type: 'list', name, content }); + resolve({ type: 'lookup', name, content }); }; const handleReaderError = function () { diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx index ae1dbc0a03b3c..4109c2906203f 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/index.tsx @@ -7,7 +7,6 @@ import React, { useCallback, useMemo } from 'react'; import type { EuiStepProps, EuiStepStatus } from '@elastic/eui'; -import { EMPTY_RESOURCE_PLACEHOLDER } from '../../../../../../../../../common/siem_migrations/constants'; import { useUpsertResources } from '../../../../../../service/hooks/use_upsert_resources'; import type { RuleMigrationTaskStats } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; import type { UploadedLookups, AddUploadedLookups } from '../../lookups_data_input'; @@ -32,11 +31,11 @@ export const useMissingLookupsListStep = ({ }: MissingLookupsListStepProps): EuiStepProps => { const { upsertResources, isLoading, error } = useUpsertResources(addUploadedLookups); - const clearLookup = useCallback( + const omitLookup = useCallback( (lookupName: string) => { - upsertResources(migrationStats.id, [ - { type: 'list', name: lookupName, content: EMPTY_RESOURCE_PLACEHOLDER }, - ]); + // Saving the lookup with an empty content to omit it. + // The translation will ignore this lookup and will not cause partial translations. + upsertResources(migrationStats.id, [{ type: 'lookup', name: lookupName, content: '' }]); }, [upsertResources, migrationStats] ); @@ -59,7 +58,7 @@ export const useMissingLookupsListStep = ({ onCopied={onCopied} missingLookups={missingLookups} uploadedLookups={uploadedLookups} - clearLookup={clearLookup} + omitLookup={omitLookup} /> ), }; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx index cd462a41bb6c1..fb521d9302a83 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/lookups/sub_steps/missing_lookups_list/missing_lookups_list.tsx @@ -19,7 +19,6 @@ import { EuiToolTip, useEuiTheme, } from '@elastic/eui'; -import { EMPTY_RESOURCE_PLACEHOLDER } from '../../../../../../../../../common/siem_migrations/constants'; import type { UploadedLookups } from '../../lookups_data_input'; import * as i18n from './translations'; @@ -31,18 +30,18 @@ const scrollPanelCss = css` interface MissingLookupsListProps { missingLookups: string[]; uploadedLookups: UploadedLookups; - clearLookup: (lookupsName: string) => void; + omitLookup: (lookupsName: string) => void; onCopied: () => void; } export const MissingLookupsList = React.memo( - ({ missingLookups, uploadedLookups, clearLookup, onCopied }) => { + ({ missingLookups, uploadedLookups, omitLookup, onCopied }) => { const { euiTheme } = useEuiTheme(); return ( <> {missingLookups.map((lookupName) => { - const isMarkedAsEmpty = uploadedLookups[lookupName] === EMPTY_RESOURCE_PLACEHOLDER; + const isOmitted = uploadedLookups[lookupName] === ''; return ( ( justifyContent="flexStart" > - {uploadedLookups[lookupName] ? ( + {uploadedLookups[lookupName] != null ? ( ) : ( )} - + {lookupName} @@ -78,10 +74,10 @@ export const MissingLookupsList = React.memo( - @@ -115,7 +111,7 @@ const CopyLookupNameButton = React.memo( ( ); CopyLookupNameButton.displayName = 'CopyLookupNameButton'; -interface ClearLookupButtonProps { +interface OmitLookupButtonProps { lookupName: string; - clearLookup: (lookupName: string) => void; + omitLookup: (lookupName: string) => void; isDisabled: boolean; } -const ClearLookupButton = React.memo( - ({ lookupName, clearLookup, isDisabled: isDisabledDefault }) => { +const OmitLookupButton = React.memo( + ({ lookupName, omitLookup, isDisabled: isDisabledDefault }) => { const [isDisabled, setIsDisabled] = useState(isDisabledDefault); const onClick = useCallback(() => { setIsDisabled(true); - clearLookup(lookupName); - }, [clearLookup, lookupName]); + omitLookup(lookupName); + }, [omitLookup, lookupName]); const button = useMemo( () => ( @@ -158,4 +154,4 @@ const ClearLookupButton = React.memo( return {button}; } ); -ClearLookupButton.displayName = 'ClearLookupButton'; +OmitLookupButton.displayName = 'OmitLookupButton'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts index 1e5a8a0f7028c..5c33dcc5c1826 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts @@ -6,10 +6,10 @@ */ import type { - RuleMigrationResourceData, + RuleMigrationResourceBase, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; export type OnMigrationCreated = (migrationStats: RuleMigrationTaskStats) => void; export type OnResourcesCreated = () => void; -export type OnMissingResourcesFetched = (missingResources: RuleMigrationResourceData[]) => void; +export type OnMissingResourcesFetched = (missingResources: RuleMigrationResourceBase[]) => void; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts index a0679aa1e8bd2..5c4ec3925c5e3 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts @@ -7,7 +7,7 @@ import { useCallback, useReducer } from 'react'; import { i18n } from '@kbn/i18n'; -import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { useKibana } from '../../../../common/lib/kibana/kibana_react'; import { reducer, initialState } from './common/api_request_reducer'; @@ -17,7 +17,7 @@ export const RULES_DATA_INPUT_CREATE_MIGRATION_ERROR = i18n.translate( ); export type GetMissingResources = (migrationId: string) => void; -export type OnSuccess = (missingResources: RuleMigrationResourceData[]) => void; +export type OnSuccess = (missingResources: RuleMigrationResourceBase[]) => void; export const useGetMissingResources = (onSuccess: OnSuccess) => { const { siemMigrations, notifications } = useKibana().services; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts index 83ead556b09cc..1bcdfadfd8527 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts @@ -14,7 +14,7 @@ import { } from '@kbn/elastic-assistant/impl/assistant_context/constants'; import type { LangSmithOptions } from '../../../../common/siem_migrations/model/common.gen'; import type { - RuleMigrationResourceData, + RuleMigrationResourceBase, RuleMigrationTaskStats, } from '../../../../common/siem_migrations/model/rule_migration.gen'; import type { @@ -155,7 +155,7 @@ export class SiemRulesMigrationsService { return results; } - public async getMissingResources(migrationId: string): Promise { + public async getMissingResources(migrationId: string): Promise { return getMissingResources({ migrationId }); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts index 0c9ad11f4cce6..6b2108703bcc6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts @@ -7,7 +7,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import type { RuleMigrationResourceData } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; import { GetRuleMigrationResourcesMissingRequestParams, type GetRuleMigrationResourcesMissingResponse, @@ -49,7 +49,7 @@ export const registerSiemRuleMigrationsResourceGetMissingRoute = ( const options = { filters: { hasContent: false } }; const batches = ruleMigrationsClient.data.resources.searchBatches(migrationId, options); - const missingResources: RuleMigrationResourceData[] = []; + const missingResources: RuleMigrationResourceBase[] = []; let results = await batches.next(); while (results.length) { missingResources.push(...results.map(({ type, name }) => ({ type, name }))); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts index fde332aefbd3f..94a5c9368144f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts @@ -7,6 +7,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import partition from 'lodash/partition'; import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; import { UpsertRuleMigrationResourcesRequestBody, @@ -17,6 +18,7 @@ import { SIEM_RULE_MIGRATION_RESOURCES_PATH } from '../../../../../../common/sie import type { SecuritySolutionPluginRouter } from '../../../../../types'; import type { CreateRuleMigrationResourceInput } from '../../data/rule_migrations_data_resources_client'; import { withLicense } from '../util/with_license'; +import { processLookups } from '../util/lookups'; export const registerSiemRuleMigrationsResourceUpsertRoute = ( router: SecuritySolutionPluginRouter, @@ -58,14 +60,17 @@ export const registerSiemRuleMigrationsResourceUpsertRoute = ( return res.notFound({ body: { message: 'Migration not found' } }); } - // Upsert identified resource documents with content - const ruleMigrations = resources.map((resource) => ({ + const [lookups, macros] = partition(resources, { type: 'lookup' }); + const processedLookups = await processLookups(lookups, ruleMigrationsClient); + const resourcesUpsert = [...macros, ...processedLookups].map((resource) => ({ ...resource, migration_id: migrationId, })); - await ruleMigrationsClient.data.resources.upsert(ruleMigrations); - // Create identified resource documents without content to keep track of them + // Upsert the resources + await ruleMigrationsClient.data.resources.upsert(resourcesUpsert); + + // Create identified resource documents to keep track of them (without content) const resourceIdentifier = new ResourceIdentifier(rule.original_rule.vendor); const resourcesToCreate = resourceIdentifier .fromResources(resources) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts new file mode 100644 index 0000000000000..f18fc05dafbdb --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/lookups.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Papa from 'papaparse'; +import { initPromisePool } from '../../../../../utils/promise_pool'; +import type { RuleMigrationResourceData } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { SiemRuleMigrationsClient } from '../../siem_rule_migrations_service'; + +interface LookupWithData extends RuleMigrationResourceData { + data: object[] | null; +} + +export const processLookups = async ( + resources: RuleMigrationResourceData[], + ruleMigrationsClient: SiemRuleMigrationsClient +): Promise => { + const lookupsData: Record = {}; + + resources.forEach((resource) => { + if (resource.type === 'lookup' && !lookupsData[resource.name]) { + try { + lookupsData[resource.name] = { ...resource, data: parseContent(resource.content) }; + } catch (error) { + throw new Error(`Invalid content for lookup ${name}: ${error.message}`); + } + } + }); + + const lookups: RuleMigrationResourceData[] = []; + const result = await initPromisePool({ + concurrency: 10, + items: Object.entries(lookupsData), + executor: async ([name, { data, ...resource }]) => { + if (!data) { + lookups.push({ ...resource, content: '' }); // empty content will make lookup be ignored during translation + return; + } + const indexName = await ruleMigrationsClient.data.lookups.create(name, data); + lookups.push({ ...resource, content: indexName }); // lookup will be translated using the index name + }, + }); + const [error] = result.errors; + if (error) { + throw new Error(`Failed to process lookups: ${error.error}`); + } + + return lookups; +}; + +const parseContent = (fileContent: string): object[] | null => { + const trimmedContent = fileContent.trim(); + if (fileContent === '') { + return null; + } + let arrayContent: object[]; + + if (trimmedContent.startsWith('[')) { + arrayContent = parseJSONArray(trimmedContent); + } else if (trimmedContent.startsWith('{')) { + arrayContent = parseNDJSON(trimmedContent); + } else { + arrayContent = parseCSV(trimmedContent); + } + return arrayContent; +}; + +const parseCSV = (fileContent: string): object[] => { + const config: Papa.ParseConfig = { + header: true, // If header is false, rows are arrays; otherwise they are objects of data keyed by the field name. + skipEmptyLines: true, + }; + const { data, errors } = Papa.parse(fileContent, config); + if (errors.length > 0) { + throw new Error('Invalid CSV'); + } + return data as object[]; +}; + +const parseNDJSON = (fileContent: string): object[] => { + return fileContent + .split(/\n(?=\{)/) // split at newline followed by '{'. + .filter((entry) => entry.trim() !== '') // Remove empty entries. + .map(parseJSON); // Parse each entry as JSON. +}; + +const parseJSONArray = (fileContent: string): object[] => { + const parsedContent = parseJSON(fileContent); + if (!Array.isArray(parsedContent)) { + throw new Error('invalid JSON'); + } + return parsedContent; +}; + +const parseJSON = (fileContent: string) => { + try { + return JSON.parse(fileContent); + } catch (error) { + if (error instanceof RangeError) { + throw new Error('File is too large'); + } + throw new Error('Invalid JSON'); + } +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts index c06c889482360..0d10c1ae4f930 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts @@ -5,11 +5,12 @@ * 2.0. */ -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { IScopedClusterClient, Logger } from '@kbn/core/server'; import { RuleMigrationsDataIntegrationsClient } from './rule_migrations_data_integrations_client'; import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_prebuilt_rules_client'; import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client'; import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client'; +import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client'; import type { AdapterId } from './rule_migrations_data_service'; export type IndexNameProvider = () => Promise; @@ -20,35 +21,41 @@ export class RuleMigrationsDataClient { public readonly resources: RuleMigrationsDataResourcesClient; public readonly integrations: RuleMigrationsDataIntegrationsClient; public readonly prebuiltRules: RuleMigrationsDataPrebuiltRulesClient; + public readonly lookups: RuleMigrationsDataLookupsClient; constructor( indexNameProviders: IndexNameProviders, username: string, - esClient: ElasticsearchClient, + esScopedClient: IScopedClusterClient, logger: Logger ) { this.rules = new RuleMigrationsDataRulesClient( indexNameProviders.rules, username, - esClient, + esScopedClient.asInternalUser, logger ); this.resources = new RuleMigrationsDataResourcesClient( indexNameProviders.resources, username, - esClient, + esScopedClient.asInternalUser, logger ); this.integrations = new RuleMigrationsDataIntegrationsClient( indexNameProviders.integrations, username, - esClient, + esScopedClient.asInternalUser, logger ); this.prebuiltRules = new RuleMigrationsDataPrebuiltRulesClient( indexNameProviders.prebuiltrules, username, - esClient, + esScopedClient.asInternalUser, + logger + ); + this.lookups = new RuleMigrationsDataLookupsClient( + username, + esScopedClient.asCurrentUser, logger ); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts new file mode 100644 index 0000000000000..d19c67b594319 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sha256 } from 'js-sha256'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '@kbn/index-adapter'; + +export type LookupData = object[]; + +export class RuleMigrationsDataLookupsClient { + constructor( + protected username: string, + protected esClient: ElasticsearchClient, + protected logger: Logger + ) {} + + async create(lookupName: string, data: LookupData): Promise { + const indexName = `lookup-${lookupName}`; + try { + await this.executeEs(() => + this.esClient.indices.create({ + index: indexName, + settings: { index: { mode: 'lookup' } }, + mappings: { dynamic: 'runtime' }, + }) + ); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + this.logger.error(`Error creating lookup index ${indexName} - ${error.message}`); + throw error; + } + } + + if (data.length > 0) { + await this.indexData(indexName, data); + } + return indexName; + } + + async indexData(indexName: string, data: LookupData): Promise { + const body = data.flatMap((doc) => [ + { create: { _index: indexName, _id: this.generateDocumentHash(doc) } }, + doc, + ]); + + try { + await this.executeEs(() => this.esClient.bulk({ index: indexName, body })); + } catch (error) { + if (error?.statusCode !== 404) { + this.logger.error(`Error indexing data for lookup index ${indexName} - ${error.message}`); + throw error; + } + } + } + + private async executeEs(fn: () => Promise): Promise { + return retryTransientEsErrors(fn, { logger: this.logger }); + } + + private generateDocumentHash(document: object): string { + return sha256.create().update(JSON.stringify(document)).hex(); // document need to be created in a deterministic way + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts index 97e51e9bafdb0..c2bd1ed2e50f8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts @@ -16,10 +16,8 @@ import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client export type CreateRuleMigrationResourceInput = Pick< RuleMigrationResource, - 'migration_id' | 'type' | 'name' | 'metadata' -> & { - content?: string; -}; + 'migration_id' | 'type' | 'name' | 'content' | 'metadata' +>; export interface RuleMigrationResourceFilters { type?: RuleMigrationResourceType; names?: string[]; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts index e991ce2684f3e..461ed98fe058e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts @@ -9,7 +9,7 @@ import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { securityServiceMock } from '@kbn/core-security-server-mocks'; import type { InstallParams } from '@kbn/index-adapter'; -import { IndexPatternAdapter } from '@kbn/index-adapter'; +import { IndexPatternAdapter, IndexAdapter } from '@kbn/index-adapter'; import { loggerMock } from '@kbn/logging-mocks'; import { Subject } from 'rxjs'; import type { IndexNameProviders } from './rule_migrations_data_client'; @@ -28,6 +28,7 @@ jest.mock('./rule_migrations_data_client', () => ({ const MockedIndexPatternAdapter = IndexPatternAdapter as unknown as jest.MockedClass< typeof IndexPatternAdapter >; +const MockedIndexAdapter = IndexAdapter as unknown as jest.MockedClass; const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; @@ -42,95 +43,93 @@ describe('SiemRuleMigrationsDataService', () => { describe('constructor', () => { it('should create IndexPatternAdapters', () => { new RuleMigrationsDataService(logger, kibanaVersion); - expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(4); + expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(2); + expect(MockedIndexAdapter).toHaveBeenCalledTimes(2); }); it('should create component templates', () => { new RuleMigrationsDataService(logger, kibanaVersion); - const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + const [rulesAdapter, resourcesAdapter] = MockedIndexPatternAdapter.mock.instances; + const [integrationsAdapter, prebuiltRulesAdapter] = MockedIndexAdapter.mock.instances; + expect(rulesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-rules` }) ); - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect(resourcesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-resources` }) ); - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect(integrationsAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-integrations` }) ); - expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( - expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltrules` }) + expect(prebuiltRulesAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltRules` }) ); }); it('should create index templates', () => { new RuleMigrationsDataService(logger, kibanaVersion); - const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + const [rulesAdapter, resourcesAdapter] = MockedIndexPatternAdapter.mock.instances; + const [integrationsAdapter, prebuiltRulesAdapter] = MockedIndexAdapter.mock.instances; + expect(rulesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-rules` }) ); - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect(resourcesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-resources` }) ); - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect(integrationsAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: `${INDEX_PATTERN}-integrations` }) ); - expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( - expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltrules` }) + expect(prebuiltRulesAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltRules` }) ); }); }); describe('install', () => { it('should install index pattern', async () => { - const index = new RuleMigrationsDataService(logger, kibanaVersion); + const service = new RuleMigrationsDataService(logger, kibanaVersion); const params: Omit = { esClient, pluginStop$: new Subject(), }; - await index.install(params); + await service.install(params); const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; + const [indexAdapter] = MockedIndexAdapter.mock.instances; + expect(indexPatternAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); + expect(indexAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); }); }); describe('createClient', () => { const currentUser = securityServiceMock.createMockAuthenticatedUser(); - const createClientParams = { spaceId: 'space1', currentUser, esClient }; + const createClientParams = { + spaceId: 'space1', + currentUser, + esScopedClient: elasticsearchServiceMock.createStart().client.asScoped(), + }; it('should install space index pattern', async () => { - const index = new RuleMigrationsDataService(logger, kibanaVersion); + const service = new RuleMigrationsDataService(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), pluginStop$: new Subject(), }; - const [ - rulesIndexPatternAdapter, - resourcesIndexPatternAdapter, - integrationsIndexPatternAdapter, - prebuiltrulesIndexPatternAdapter, - ] = MockedIndexPatternAdapter.mock.instances; + const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter] = + MockedIndexPatternAdapter.mock.instances; (rulesIndexPatternAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); - await index.install(params); - index.createClient(createClientParams); + await service.install(params); + service.createClient(createClientParams); await mockIndexNameProviders.rules(); await mockIndexNameProviders.resources(); - await mockIndexNameProviders.integrations(); - await mockIndexNameProviders.prebuiltrules(); expect(rulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); expect(rulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); expect(resourcesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); expect(resourcesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); - - expect(integrationsIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); - expect(integrationsIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); - - expect(prebuiltrulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); - expect(prebuiltrulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts index 5799e5ab84c07..b59291c6a5a0e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts @@ -4,8 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; -import { IndexPatternAdapter, type FieldMap, type InstallParams } from '@kbn/index-adapter'; +import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server'; +import { + IndexAdapter, + IndexPatternAdapter, + type FieldMap, + type InstallParams, +} from '@kbn/index-adapter'; import type { IndexNameProvider, IndexNameProviders } from './rule_migrations_data_client'; import { RuleMigrationsDataClient } from './rule_migrations_data_client'; import { @@ -18,28 +23,55 @@ import { const TOTAL_FIELDS_LIMIT = 2500; export const INDEX_PATTERN = '.kibana-siem-rule-migrations'; -export type AdapterId = 'rules' | 'resources' | 'integrations' | 'prebuiltrules'; +export interface Adapters { + rules: IndexPatternAdapter; + resources: IndexPatternAdapter; + integrations: IndexAdapter; + prebuiltrules: IndexAdapter; +} + +export type AdapterId = keyof Adapters; interface CreateClientParams { spaceId: string; currentUser: AuthenticatedUser; - esClient: ElasticsearchClient; + esScopedClient: IScopedClusterClient; +} +interface CreateAdapterParams { + adapterId: AdapterId; + fieldMap: FieldMap; } export class RuleMigrationsDataService { - private readonly adapters: Record; + private readonly adapters: Adapters; constructor(private logger: Logger, private kibanaVersion: string) { this.adapters = { - rules: this.createAdapter({ id: 'rules', fieldMap: ruleMigrationsFieldMap }), - resources: this.createAdapter({ id: 'resources', fieldMap: ruleMigrationResourcesFieldMap }), - integrations: this.createAdapter({ id: 'integrations', fieldMap: integrationsFieldMap }), - prebuiltrules: this.createAdapter({ id: 'prebuiltrules', fieldMap: prebuiltRulesFieldMap }), + rules: this.createIndexPatternAdapter({ + adapterId: 'rules', + fieldMap: ruleMigrationsFieldMap, + }), + resources: this.createIndexPatternAdapter({ + adapterId: 'resources', + fieldMap: ruleMigrationResourcesFieldMap, + }), + integrations: this.createIndexAdapter({ + adapterId: 'integrations', + fieldMap: integrationsFieldMap, + }), + prebuiltrules: this.createIndexAdapter({ + adapterId: 'prebuiltrules', + fieldMap: prebuiltRulesFieldMap, + }), }; } - private createAdapter({ id, fieldMap }: { id: AdapterId; fieldMap: FieldMap }) { - const name = `${INDEX_PATTERN}-${id}`; + private getAdapterIndexName(adapterId: AdapterId) { + return `${INDEX_PATTERN}-${adapterId}`; + } + + private createIndexPatternAdapter({ adapterId, fieldMap }: CreateAdapterParams) { + const name = this.getAdapterIndexName(adapterId); const adapter = new IndexPatternAdapter(name, { kibanaVersion: this.kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, @@ -49,6 +81,17 @@ export class RuleMigrationsDataService { return adapter; } + private createIndexAdapter({ adapterId, fieldMap }: CreateAdapterParams) { + const name = this.getAdapterIndexName(adapterId); + const adapter = new IndexAdapter(name, { + kibanaVersion: this.kibanaVersion, + totalFieldsLimit: TOTAL_FIELDS_LIMIT, + }); + adapter.setComponentTemplate({ name, fieldMap }); + adapter.setIndexTemplate({ name, componentTemplateRefs: [name] }); + return adapter; + } + public async install(params: Omit): Promise { await Promise.all([ this.adapters.rules.install({ ...params, logger: this.logger }), @@ -58,26 +101,29 @@ export class RuleMigrationsDataService { ]); } - public createClient({ spaceId, currentUser, esClient }: CreateClientParams) { + public createClient({ spaceId, currentUser, esScopedClient }: CreateClientParams) { const indexNameProviders: IndexNameProviders = { - rules: this.createIndexNameProvider('rules', spaceId), - resources: this.createIndexNameProvider('resources', spaceId), - integrations: this.createIndexNameProvider('integrations', spaceId), - prebuiltrules: this.createIndexNameProvider('prebuiltrules', spaceId), + rules: this.createIndexNameProvider(this.adapters.rules, spaceId), + resources: this.createIndexNameProvider(this.adapters.resources, spaceId), + integrations: async () => this.getAdapterIndexName('integrations'), + prebuiltrules: async () => this.getAdapterIndexName('prebuiltrules'), }; return new RuleMigrationsDataClient( indexNameProviders, currentUser.username, - esClient, + esScopedClient, this.logger ); } - private createIndexNameProvider(adapter: AdapterId, spaceId: string): IndexNameProvider { + private createIndexNameProvider( + adapter: IndexPatternAdapter, + spaceId: string + ): IndexNameProvider { return async () => { - await this.adapters[adapter].createIndex(spaceId); // This will resolve instantly when the index is already created - return this.adapters[adapter].getIndexName(spaceId); + await adapter.createIndex(spaceId); // This will resolve instantly when the index is already created + return adapter.getIndexName(spaceId); }; } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index d9f4a1c5249cb..1fc276d7a6b81 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -65,8 +65,8 @@ export class SiemRuleMigrationsService { assert(currentUser, 'Current user must be authenticated'); assert(this.esClusterClient, 'ES client not available, please call setup first'); - const esClient = this.esClusterClient.asInternalUser; - const dataClient = this.dataService.createClient({ spaceId, currentUser, esClient }); + const esScopedClient = this.esClusterClient.asScoped(request); + const dataClient = this.dataService.createClient({ spaceId, currentUser, esScopedClient }); const taskClient = this.taskService.createClient({ currentUser, dataClient }); return { data: dataClient, task: taskClient }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index b1165ce982293..18d417a23e0dc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -6,10 +6,8 @@ */ import { END, START, StateGraph } from '@langchain/langgraph'; -import { RuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; import { getCreateSemanticQueryNode } from './nodes/create_semantic_query'; import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; -import { getProcessQueryNode } from './nodes/process_query'; import { migrateRuleState } from './state'; import { getTranslateRuleGraph } from './sub_graphs/translate_rule'; @@ -35,19 +33,19 @@ export function getRuleMigrationAgent({ logger, }); const createSemanticQueryNode = getCreateSemanticQueryNode({ model }); - const processQueryNode = getProcessQueryNode({ model, ruleMigrationsRetriever }); const siemMigrationAgentGraph = new StateGraph(migrateRuleState) // Nodes - .addNode('processQuery', processQueryNode) .addNode('createSemanticQuery', createSemanticQueryNode) .addNode('matchPrebuiltRule', matchPrebuiltRuleNode) .addNode('translationSubGraph', translationSubGraph) // Edges .addEdge(START, 'createSemanticQuery') .addEdge('createSemanticQuery', 'matchPrebuiltRule') - .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional, ['processQuery', END]) - .addEdge('processQuery', 'translationSubGraph') + .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional, [ + 'translationSubGraph', + END, + ]) .addEdge('translationSubGraph', END); const graph = siemMigrationAgentGraph.compile(); @@ -62,8 +60,5 @@ const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { if (state.elastic_rule?.prebuilt_rule_id) { return END; } - if (state.translation_result === RuleTranslationResult.UNTRANSLATABLE) { - return END; - } - return 'processQuery'; + return 'translationSubGraph'; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts index d7537fdc72dd0..2f3c29f25e0ce 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -72,16 +72,6 @@ export const getMatchPrebuiltRuleNode = ({ }; } } - const lookupTypes = ['inputlookup', 'outputlookup']; - if ( - state.original_rule?.query && - lookupTypes.some((type) => state.original_rule.query.includes(type)) - ) { - logger.debug( - `Rule: ${state.original_rule?.title} did not match any prebuilt rule, but contains inputlookup, dropping` - ); - return { translation_result: RuleTranslationResult.UNTRANSLATABLE }; - } return {}; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts deleted file mode 100644 index 68eaaeffd11b1..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/prompts.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ChatPromptTemplate } from '@langchain/core/prompts'; -import type { RuleMigrationResources } from '../../../retrievers/rule_resource_retriever'; - -interface ResourceContext { - macros: string; - lists: string; -} - -export const getResourcesContext = (resources: RuleMigrationResources): ResourceContext => { - const result: ResourceContext = { macros: '', lists: '' }; - - // Process macros - if (resources.macro?.length) { - const macrosMap = resources.macro.reduce((acc, macro) => { - acc[macro.name] = macro.content; - return acc; - }, {} as Record); - - result.macros = JSON.stringify(macrosMap, null, 2); - } - - // Process lists - if (resources.list?.length) { - const listsMap = resources.list.reduce((acc, list) => { - acc[list.name] = list.content; - return acc; - }, {} as Record); - - result.lists = JSON.stringify(listsMap, null, 2); - } - - return result; -}; - -export const REPLACE_QUERY_RESOURCE_PROMPT = ChatPromptTemplate.fromMessages([ - [ - 'system', - `You are an agent expert in Splunk SPL (Search Processing Language). -Your task is to inline a set of macros syntax using its values in a SPL query. -Here are some context for you to reference for your task, read it carefully as you will get questions about it later: - - - - -Always follow the below guidelines when replacing macros: -- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments. - -Having the following macros: - \`someSource\`: sourcetype="somesource" - \`searchTitle(1)\`: search title="$value$" - \`searchTitle\`: search title=* - \`searchType\`: search type=* -And the following SPL query: - \`\`\`spl - \`someSource\` \`someFilter\` - | \`searchTitle("sometitle")\` - | \`searchType("sometype")\` - | table * - \`\`\` -The correct replacement would be: - \`\`\`spl - sourcetype="somesource" \`someFilter\` - | search title="sometitle" - | \`searchType("sometype")\` - | table * - \`\`\` - -`, - ], - [ - 'human', - `Go through the SPL query and identify all the macros that are used. - -{macros} - - - -\`\`\`spl -{query} -\`\`\` - - -Divide the query up into separate section and go through each section one at a time to identify the macros used that need to be replaced using one of two scenarios: -- The macro is provided in the list of available macros: Replace it using its actual content. -- The macro is not in the list of available macros: Do not replace it, keep it in the query as it is. - - -- You will be provided with a SPL query and also the related macros used in the query. -- You have to replace the macros syntax in the SPL query and use their values inline, if provided. -- The original and modified queries must be equivalent. -- You must respond only with the modified query inside a \`\`\`spl code block, nothing else similar to the example response below. - - - -A: Please find the modified SPL query below: -\`\`\`spl -sourcetype="linux:audit" \`linux_auditd_normalized_proctitle_process\` -| rename host as dest -| where LIKE (process_exec, "%chown root%") -| stats count min(_time) as firstTime max(_time) as lastTime by process_exec proctitle normalized_proctitle_delimiter dest -| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(firstTime) -| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(lastTime) -| search * -\`\`\` - - -`, - ], - ['ai', 'Please find the modified SPL query below:'], -]); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts index 99d7d1439d63e..fc6ea5309ced5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts @@ -9,11 +9,12 @@ import { END, START, StateGraph } from '@langchain/langgraph'; import { isEmpty } from 'lodash/fp'; import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; import { getEcsMappingNode } from './nodes/ecs_mapping'; -import { getFilterIndexPatternsNode } from './nodes/filter_index_patterns'; +import { translationResultNode } from './nodes/translation_result'; import { getFixQueryErrorsNode } from './nodes/fix_query_errors'; import { getRetrieveIntegrationsNode } from './nodes/retrieve_integrations'; import { getTranslateRuleNode } from './nodes/translate_rule'; import { getValidationNode } from './nodes/validation'; +import { getInlineQueryNode } from './nodes/inline_query'; import { translateRuleState } from './state'; import type { TranslateRuleGraphParams, TranslateRuleState } from './types'; @@ -32,22 +33,24 @@ export function getTranslateRuleGraph({ connectorId, logger, }); + const inlineQueryNode = getInlineQueryNode({ model, ruleMigrationsRetriever }); const validationNode = getValidationNode({ logger }); const fixQueryErrorsNode = getFixQueryErrorsNode({ inferenceClient, connectorId, logger }); const retrieveIntegrationsNode = getRetrieveIntegrationsNode({ model, ruleMigrationsRetriever }); const ecsMappingNode = getEcsMappingNode({ inferenceClient, connectorId, logger }); - const filterIndexPatternsNode = getFilterIndexPatternsNode({ logger }); const translateRuleGraph = new StateGraph(translateRuleState) // Nodes + .addNode('inlineQuery', inlineQueryNode) + .addNode('retrieveIntegrations', retrieveIntegrationsNode) .addNode('translateRule', translateRuleNode) .addNode('validation', validationNode) .addNode('fixQueryErrors', fixQueryErrorsNode) - .addNode('retrieveIntegrations', retrieveIntegrationsNode) .addNode('ecsMapping', ecsMappingNode) - .addNode('filterIndexPatterns', filterIndexPatternsNode) + .addNode('translationResultNode', translationResultNode) // Edges - .addEdge(START, 'retrieveIntegrations') + .addEdge(START, 'inlineQuery') + .addEdge('inlineQuery', 'retrieveIntegrations') .addEdge('retrieveIntegrations', 'translateRule') .addEdge('translateRule', 'validation') .addEdge('fixQueryErrors', 'validation') @@ -55,9 +58,9 @@ export function getTranslateRuleGraph({ .addConditionalEdges('validation', validationRouter, [ 'fixQueryErrors', 'ecsMapping', - 'filterIndexPatterns', + 'translationResultNode', ]) - .addEdge('filterIndexPatterns', END); + .addEdge('translationResultNode', END); const graph = translateRuleGraph.compile(); graph.name = 'Translate Rule Graph'; @@ -76,5 +79,5 @@ const validationRouter = (state: TranslateRuleState) => { return 'ecsMapping'; } } - return 'filterIndexPatterns'; + return 'translationResultNode'; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts index 07753432e5dbc..08a6c5904819a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts @@ -59,7 +59,7 @@ export const getEcsMappingNode = ({ }; const getTranslationResult = (esqlQuery: string): RuleTranslationResult => { - if (esqlQuery.match(/\[(macro):[\s\S]*\]/)) { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { return RuleTranslationResult.PARTIAL; } return RuleTranslationResult.FULL; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts deleted file mode 100644 index b7cbcabff2ca2..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/filter_index_patterns.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; -import type { GraphNode } from '../../types'; - -interface GetFilterIndexPatternsNodeParams { - logger: Logger; -} - -/** - * When rule translation happens without any related integrations found we reuse the logs-* pattern to make validation easier. - * However we want to replace this with a value to notify the end user that it needs to be replaced. - */ -export const getFilterIndexPatternsNode = ({ - logger, -}: GetFilterIndexPatternsNodeParams): GraphNode => { - return async (state) => { - const query = state.elastic_rule?.query; - - if (query && query.includes('logs-*')) { - logger.debug('Replacing logs-* with a placeholder value'); - const newQuery = query.replace('logs-*', '[indexPattern:logs-*]'); - return { - elastic_rule: { - ...state.elastic_rule, - query: newQuery, - translation_result: RuleTranslationResult.PARTIAL, - }, - }; - } - - return {}; - }; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/index.ts similarity index 82% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/index.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/index.ts index 6feb852eba474..c466306e99074 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/index.ts @@ -4,4 +4,4 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -export { getProcessQueryNode } from './process_query'; +export { getInlineQueryNode } from './inline_query'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts similarity index 78% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts index e4b7e64e85b00..b88dd0aed0b59 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_query/process_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts @@ -7,20 +7,20 @@ import { StringOutputParser } from '@langchain/core/output_parsers'; import { isEmpty } from 'lodash/fp'; -import type { RuleMigrationsRetriever } from '../../../retrievers'; -import type { ChatModel } from '../../../util/actions_client_chat'; -import type { GraphNode } from '../../types'; +import type { RuleMigrationsRetriever } from '../../../../../retrievers'; +import type { ChatModel } from '../../../../../util/actions_client_chat'; +import type { GraphNode } from '../../../../types'; import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts'; -interface GetProcessQueryNodeParams { +interface GetInlineQueryNodeParams { model: ChatModel; ruleMigrationsRetriever: RuleMigrationsRetriever; } -export const getProcessQueryNode = ({ +export const getInlineQueryNode = ({ model, ruleMigrationsRetriever, -}: GetProcessQueryNodeParams): GraphNode => { +}: GetInlineQueryNodeParams): GraphNode => { return async (state) => { let query = state.original_rule.query; const resources = await ruleMigrationsRetriever.resources.getResources(state.original_rule); @@ -32,6 +32,7 @@ export const getProcessQueryNode = ({ const response = await replaceQueryResourcePrompt.invoke({ query: state.original_rule.query, macros: resourceContext.macros, + lookups: resourceContext.lookups, }); const splQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1] ?? ''; if (splQuery) { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts new file mode 100644 index 0000000000000..b624604cd1c2e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatPromptTemplate } from '@langchain/core/prompts'; +import type { RuleMigrationResources } from '../../../../../retrievers/rule_resource_retriever'; + +interface ResourceContext { + macros: string; + lookups: string; +} + +export const getResourcesContext = (resources: RuleMigrationResources): ResourceContext => { + const result: ResourceContext = { macros: '', lookups: '' }; + + // Process macros + if (resources.macro?.length) { + const macrosMap = resources.macro.reduce((acc, macro) => { + acc[macro.name] = macro.content; + return acc; + }, {} as Record); + + result.macros = JSON.stringify(macrosMap, null, 2); + } + + // Process lookups + if (resources.lookup?.length) { + const lookupsMap = resources.lookup.reduce((acc, lookup) => { + acc[lookup.name] = lookup.content; + return acc; + }, {} as Record); + + result.lookups = JSON.stringify(lookupsMap, null, 2); + } + + return result; +}; + +export const REPLACE_QUERY_RESOURCE_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an agent expert in Splunk SPL (Search Processing Language). +Your task is to replace macros and lookups syntax in a SPL query, using the actual content of the macros and lookup names provided to you. +Here are some context for you to reference for your task, read it carefully as you will get questions about it later: + + + + +You have to replace the macros syntax in the SPL query and use their value inline, if provided. + +Always follow the below guidelines when replacing macros: +- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments. +- Divide the query up into separate sections and go through each section one at a time to identify the macros used that need to be replaced, using one of two scenarios: + - The macro is provided in the list of available macros: Replace it using its actual content. + - The macro is not in the list of available macros: add a placeholder ("missing placeholder" from now on) in the query with the format [macro:(argumentCount)] including the [] keys, + Example: \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)] + +Having the following macros: + \`someSource\`: sourcetype="somesource" + \`searchTitle(1)\`: search title="$value$" + \`searchTitle\`: search title=* + \`searchType\`: search type=* + +And the following SPL query: + \`\`\`spl + \`someSource\` \`someFilter\` + | \`searchTitle("sometitle")\` + | \`searchType("sometype")\` + | \`anotherMacro("someParam","someOtherParam", 10)\` + | table * + \`\`\` + +The correct replacement would be: + \`\`\`spl + sourcetype="somesource" \`someFilter\` + | search title="sometitle" + | \`searchType("sometype")\` + | [macro:anotherMacro(3)] + | table * + \`\`\` + + + +You have to replace the lookup names in the SPL query with the correct name, if provided. + +Always follow the below guidelines when replacing lookups: +- Divide the query up into separate sections and go through each section one at a time to identify the lookups used that need to be replaced, using one of two scenarios: + - The lookup is provided in the list of available lookups: Replace the lookup name using its correct name provided. + - Remember the "_lookup" suffix in the lookup name in the query can be ignored when checking the list of available lookups + - If the inputlookup or outputlookup command is used, replace it with the correct lookup name. + - The lookup is not in the list of available lookups: add a placeholder ("missing placeholder" from now on) in the query with the format [lookup:] including the [] keys, + Example: "lookup users uid OUTPUTNEW username, department" -> "[lookup:users]" + - The lookup is in the list but has empty name: omit the lookup from the query entirely, as if it was empty. To do so you can use the EVAL command to set the fields to empty strings. + +Having the following lookups: + "some_table": "lookup-some_table" + "another": "lookup-another-2" + "lookupName3": "" + "someOutputLookup": "lookup-output" + +And the following SPL query: + \`\`\`spl + inputlookup some_table + | lookup another_lookup name OUTPUT description + | lookup yetAnotherLookup id OUTPUTNEW someField + | lookup lookupName3 uuid OUTPUTNEW group, name + | outputlookup append=T key_field=_key someOutputLookup + \`\`\` + +The correct replacement would be: + \`\`\`spl + inputlookup lookup-some_table + | lookup lookup-another-2 name OUTPUT description + | [lookup:yetAnotherLookup] + | EVAL group="", name="" + | outputlookup append=T key_field=_key lookup-output + \`\`\` + + + +- The original and modified queries must be equivalent, except for the "missing placeholders". +- You must respond only with the modified query inside a \`\`\`spl code block, nothing else similar to the example response below. + + +`, + ], + [ + 'human', + `Go through the SPL query and replace all the macros and lookups provided: + +{macros} + + + +{lookups} + + + +\`\`\`spl +{query} +\`\`\` + + + +A: Please find the modified SPL query below: +\`\`\`spl +sourcetype="linux:audit" \`linux_auditd_normalized_proctitle_process\` +| rename host as dest +| where LIKE (process_exec, "%chown root%") +| stats count min(_time) as firstTime max(_time) as lastTime by process_exec proctitle normalized_proctitle_delimiter dest +| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(firstTime) +| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(lastTime) +| search * +\`\`\` + + +`, + ], + ['ai', 'Please find the modified SPL query below:'], +]); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts index 74c9055bd7665..c30b99d0ba605 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts @@ -28,6 +28,9 @@ export const getRetrieveIntegrationsNode = ({ const query = state.semantic_query; const integrations = await ruleMigrationsRetriever.integrations.getIntegrations(query); + if (integrations.length === 0) { + return {}; + } const outputParser = new JsonOutputParser(); const mostRelevantIntegration = MATCH_INTEGRATION_PROMPT.pipe(model).pipe(outputParser); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts index 626251c3c8259..289a2f2600fd6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts @@ -9,20 +9,14 @@ import { ChatPromptTemplate } from '@langchain/core/prompts'; export const ESQL_SYNTAX_TRANSLATION_PROMPT = ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk SPL to Elasticsearch ES|QL. -Your goal is to translate the SPL query syntax into an equivalent Elastic Search Query Language (ES|QL) query without changing any of the field names except lookup lists and macros when relevant and focusing only on translating the syntax and structure. +Your goal is to translate the SPL query syntax into an equivalent Elastic Search Query Language (ES|QL) query without changing any of the field names, and focusing only on translating the syntax and structure. +You will also we asked to write a summary of the translation process followed at the end. -Here are some context for you to reference for your task, read it carefully as you will get questions about it later: +Here are some context for you to reference for your task, read it carefully: - -{splunk_rule} - - -If, in the SPL query, you find a macro call, mention it in the summary and add a placeholder in the query with the format [macro:(argumentCount)] including the [] keys, - Examples: - - \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)] - + -If in an SPL query you identify a looku list call, it should be translated the following way: +If in an SPL query you identify a "lookup OUTPUT/OUTPUTNEW" call, it should be translated the following way: \`\`\`spl ... | lookup users uid OUTPUTNEW username, department \`\`\` @@ -30,32 +24,62 @@ If in an SPL query you identify a looku list call, it should be translated the f In the above example it uses the following syntax: lookup 'index_name' 'field_to_match' OUTPUTNEW 'field1', 'field2' -However in the ES|QL query, some of the information is removed and should be used in the following way: +The ES|QL translation would be the following, using the LOOKUP JOIN operator and the ON clause: \`\`\`esql ... | LOOKUP JOIN 'index_name' ON 'field_to_match' \`\`\` -We do not define OUTPUTNEW or which fields is returned, only the index name and the field to match. +Note the fields to be returned are not defined in the ES|QL query, only the index name and the field to match. - + +If in an SPL query you identify a "inputlookup" call, it should be translated the following way. +- When the inputlookup appears as the source: +\`\`\`spl +| inputlookup users ... +\`\`\` -Go through each step and part of the splunk rule and query while following the below guide to produce the resulting ES|QL query: -- Analyze all the information about the related splunk rule and try to determine the intent of the rule, in order to translate into an equivalent ES|QL rule. -- Go through each part of the SPL query and determine the steps required to produce the same end results using ES|QL. Only focus on translating the structure without modifying any of the field names. -- Do NOT map any of the fields to the Elastic Common Schema (ECS), this will happen in a later step. -- Always remember to translate any lookup list using the lookup_syntax above -- Always remember to replace macro call with the appropriate placeholder as defined in the macro info. +It should be translated as a FROM clause in ES|QL. +\`\`\`esql +FROM users ... +\`\`\` +- When the inputlookup appears as a subquery with a condition, it should be translated using the LOOKUP JOIN operator in ES|QL. +\`\`\`esql +... | inputlookup users WHERE field="value" ... +\`\`\` + +It should be translated as a LOOKUP JOIN operator with the ON clause in ES|QL. +\`\`\`esql +... | LOOKUP JOIN users ON field="value" ... +\`\`\` + + -- Analyze the SPL query and identify the key components. -- Do NOT translate the field names of the SPL query. -- Always start the resulting ES|QL query by filtering using FROM and with these index patterns: {indexPatterns}. -- Always remember to translate any lookup list using the lookup_syntax above -- Always remember to replace macro call with the appropriate placeholder as defined in the macro info. +Go through each step and part of the splunk rule and query while following the below guide to produce the resulting ES|QL query: +- Analyze all the information about the related splunk rule and try to determine the intent of the rule, in order to translate into an equivalent ES|QL rule. +- Go through each part of the SPL query and determine the steps required to produce the same end results using ES|QL. +- Do NOT change the field names defined in the SPL query, keep them as they are in the ES|QL output. +- Always remember to translate any lookup operator using the lookups_guidelines above +- If you encounter any placeholders for missing macros or lookups in the SPL query, like [macro:] or [lookup:], keep them as they are in ES|QL output, even if they cause invalid syntax, and mention they are missing in the summary. +IMPORTANT: +The index pattern to use in the ES|QL query is "{indexPatterns}". +Always start the translated ES|QL query should start with: + +\`\`\`esql +FROM {indexPatterns} +| ... +\`\`\` + + + - First, the ES|QL query inside an \`\`\`esql code block. - At the end, the summary of the translation process followed in markdown, starting with "## Translation Summary". + + +{splunk_rule} + `); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts index 346df02714b67..e09c4e6961141 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts @@ -64,7 +64,7 @@ export const getTranslateRuleNode = ({ }; const getTranslationResult = (esqlQuery: string): RuleTranslationResult => { - if (esqlQuery.match(/\[(macro):[\s\S]*\]/)) { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { return RuleTranslationResult.PARTIAL; } return RuleTranslationResult.FULL; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts similarity index 78% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/index.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts index 6e7762633b7fd..919c3c4f18b17 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/filter_index_patterns/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts @@ -4,4 +4,4 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -export { getFilterIndexPatternsNode } from './filter_index_patterns'; +export { translationResultNode } from './translation_result'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts new file mode 100644 index 0000000000000..4bebc04d7ad26 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; +import type { GraphNode } from '../../types'; + +export const translationResultNode: GraphNode = async (state) => { + const query = state.elastic_rule?.query; + + if (query) { + /** + * When rule translation happens without any related integrations found we reuse the logs-* pattern to make validation easier. + * However we want to replace this with a value to notify the end user that it needs to be replaced. + */ + if (query.includes(' logs-*')) { + const newQuery = query.replace('logs-*', '[indexPattern]'); + return { + elastic_rule: { ...state.elastic_rule, query: newQuery }, + translation_result: RuleTranslationResult.PARTIAL, + }; + } + /** + * When rule translation misses macro or lookup a placeholder is added to the query + * to notify the user that it needs to be provided/replaced. + */ + if (query.match(/\[(macro|lookup):/)) { + return { translation_result: RuleTranslationResult.PARTIAL }; + } + } + + if (!state.translation_result) { + return { translation_result: RuleTranslationResult.UNTRANSLATABLE }; + } + + return {}; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts index 6f97678a04558..0cef5df6a260f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts @@ -25,13 +25,14 @@ export const getValidationNode = ({ logger }: GetValidationNodeParams): GraphNod // We want to prevent infinite loops, so we increment the iterations counter for each validation run. const currentIteration = ++state.validation_errors.iterations; let esqlErrors: string = ''; - if (!isEmpty(query)) { - const { errors, isEsqlQueryAggregating, hasMetadataOperator } = parseEsqlQuery(query); + const sanitizedQuery = query ? removePlaceHolders(query) : ''; + if (!isEmpty(sanitizedQuery)) { + const { errors, isEsqlQueryAggregating, hasMetadataOperator } = + parseEsqlQuery(sanitizedQuery); if (!isEmpty(errors)) { esqlErrors = JSON.stringify(errors); } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { - esqlErrors = - 'Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.'; + esqlErrors = `Queries that do't use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`; } } if (esqlErrors) { @@ -41,3 +42,10 @@ export const getValidationNode = ({ logger }: GetValidationNodeParams): GraphNod return { validation_errors: { iterations: currentIteration, esql_errors: esqlErrors } }; }; }; + +function removePlaceHolders(query: string): string { + return query + .replace(/\[indexPattern\]/g, 'logs-*') // Replace the indexPattern placeholder with logs-* + .replaceAll(/\[(macro|lookup):.*?\]/g, '') // Removes any macro or lookup placeholders + .replaceAll(/\n(\s*?\|\s*?\n)*/g, '\n'); // Removes any empty lines with | (pipe) alone after removing the placeholders +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts index 1e02eec2315e7..b0c50aa086ed3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts @@ -55,10 +55,10 @@ describe('RuleResourceRetriever', () => { expect(result).toEqual({}); }); - it('returns matching macro and list resources', async () => { + it('returns matching macro and lookup resources', async () => { const mockExistingResources = { macro: { macro1: { name: 'macro1', type: 'macro' } }, - list: { list1: { name: 'list1', type: 'list' } }, + lookup: { lookup1: { name: 'lookup1', type: 'lookup' } }, }; // Inject existing resources manually // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -66,7 +66,7 @@ describe('RuleResourceRetriever', () => { const mockResourcesIdentified = [ { name: 'macro1', type: 'macro' as const }, - { name: 'list1', type: 'list' as const }, + { name: 'lookup1', type: 'lookup' as const }, ]; MockResourceIdentifier.mockImplementation(() => ({ fromOriginalRule: jest.fn().mockReturnValue(mockResourcesIdentified), @@ -78,7 +78,7 @@ describe('RuleResourceRetriever', () => { const result = await retriever.getResources(originalRule); expect(result).toEqual({ macro: [{ name: 'macro1', type: 'macro' }], - list: [{ name: 'list1', type: 'list' }], + lookup: [{ name: 'lookup1', type: 'lookup' }], }); }); @@ -90,9 +90,9 @@ describe('RuleResourceRetriever', () => { macro1: { name: 'macro1', type: 'macro' }, macro2: { name: 'macro2', type: 'macro' }, }, - list: { - list1: { name: 'list1', type: 'list' }, - list2: { name: 'list2', type: 'list' }, + lookup: { + lookup1: { name: 'lookup1', type: 'lookup' }, + lookup2: { name: 'lookup2', type: 'lookup' }, }, }; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -100,12 +100,12 @@ describe('RuleResourceRetriever', () => { const mockResourcesIdentifiedFromRule = [ { name: 'macro1', type: 'macro' as const }, - { name: 'list1', type: 'list' as const }, + { name: 'lookup1', type: 'lookup' as const }, ]; const mockNestedResources = [ { name: 'macro2', type: 'macro' as const }, - { name: 'list2', type: 'list' as const }, + { name: 'lookup2', type: 'lookup' as const }, ]; MockResourceIdentifier.mockImplementation(() => ({ @@ -119,9 +119,9 @@ describe('RuleResourceRetriever', () => { { name: 'macro1', type: 'macro' }, { name: 'macro2', type: 'macro' }, ], - list: [ - { name: 'list1', type: 'list' }, - { name: 'list2', type: 'list' }, + lookup: [ + { name: 'lookup1', type: 'lookup' }, + { name: 'lookup2', type: 'lookup' }, ], }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts index b89939e199e5a..6fb9951c1985b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts @@ -21,7 +21,7 @@ export type RuleMigrationResources = Partial< >; interface ExistingResources { macro: Record; - list: Record; + lookup: Record; } export class RuleResourceRetriever { @@ -35,10 +35,10 @@ export class RuleResourceRetriever { public async initialize(): Promise { const batches = this.dataClient.resources.searchBatches( this.migrationId, - { filters: { hasContent: true } } + { filters: { hasContent: true } } // filters out missing (undefined) content resources, empty strings content will be included ); - const existingRuleResources: ExistingResources = { macro: {}, list: {} }; + const existingRuleResources: ExistingResources = { macro: {}, lookup: {} }; let resources; do { resources = await batches.next(); @@ -60,19 +60,19 @@ export class RuleResourceRetriever { const resourcesIdentifiedFromRule = resourceIdentifier.fromOriginalRule(originalRule); const macrosFound = new Map(); - const listsFound = new Map(); + const lookupsFound = new Map(); resourcesIdentifiedFromRule.forEach((resource) => { const existingResource = existingResources[resource.type][resource.name]; if (existingResource) { if (resource.type === 'macro') { macrosFound.set(resource.name, existingResource); - } else if (resource.type === 'list') { - listsFound.set(resource.name, existingResource); + } else if (resource.type === 'lookup') { + lookupsFound.set(resource.name, existingResource); } } }); - const resourcesFound = [...macrosFound.values(), ...listsFound.values()]; + const resourcesFound = [...macrosFound.values(), ...lookupsFound.values()]; if (!resourcesFound.length) { return {}; } @@ -88,8 +88,8 @@ export class RuleResourceRetriever { nestedResourcesFound.push(existingResource); if (resource.type === 'macro') { macrosFound.set(resource.name, existingResource); - } else if (resource.type === 'list') { - listsFound.set(resource.name, existingResource); + } else if (resource.type === 'lookup') { + lookupsFound.set(resource.name, existingResource); } } }); @@ -97,7 +97,7 @@ export class RuleResourceRetriever { return { ...(macrosFound.size > 0 ? { macro: Array.from(macrosFound.values()) } : {}), - ...(listsFound.size > 0 ? { list: Array.from(listsFound.values()) } : {}), + ...(lookupsFound.size > 0 ? { lookup: Array.from(lookupsFound.values()) } : {}), }; } } From 82b1ae7a02d20adff1ec46d98398fa1040e9378d Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 19 Dec 2024 20:58:55 +0100 Subject: [PATCH 2/9] agent changes --- .../splunk/splunk_identifier.test.ts | 14 ---- .../resources/splunk/splunk_identifier.ts | 12 ++-- .../rule_migrations_data_lookups_client.ts | 2 +- .../siem_migrations/rules/task/agent/graph.ts | 3 - .../match_prebuilt_rule.ts | 5 ++ .../nodes/match_prebuilt_rule/prompts.ts | 11 +-- .../agent/sub_graphs/translate_rule/graph.ts | 20 ++++-- .../nodes/inline_query/inline_query.ts | 18 +++++ .../nodes/inline_query/prompts.ts | 14 ++-- .../nodes/translate_rule/prompts.ts | 68 ++++++------------- .../nodes/translate_rule/translate_rule.ts | 13 +--- .../translation_result/translation_result.ts | 48 ++++++------- .../agent/sub_graphs/translate_rule/state.ts | 4 +- .../retrievers/rule_migrations_retriever.ts | 14 ++-- .../rules/task/rule_migrations_task_client.ts | 18 +++-- 15 files changed, 119 insertions(+), 145 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts index 58c48632a51c8..36b4adc2769e4 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts @@ -53,20 +53,6 @@ describe('splResourceIdentifier', () => { ]); }); - it('should extract lookup from inputlookup tables correctly', () => { - const query = 'inputlookup my_lookup_table WHERE field="value"'; - - const result = splResourceIdentifier(query); - expect(result).toEqual([{ type: 'lookup', name: 'my_lookup_table' }]); - }); - - it('should extract lookup from inputlookup with modifiers correctly', () => { - const query = 'inputlookup append=T start=10 my_lookup_table WHERE field="value"'; - - const result = splResourceIdentifier(query); - expect(result).toEqual([{ type: 'lookup', name: 'my_lookup_table' }]); - }); - it('should extract both macros and lookup tables correctly', () => { const query = '`macro_one` some search command | lookup my_lookup_table field AS alias OUTPUT new_field | lookup other_lookup_list | lookup third_lookup'; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts index 35373ba155460..90856896193dc 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts @@ -11,11 +11,10 @@ * Please make sure to test all regular expressions them before using them. * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker */ - import type { RuleMigrationResourceBase } from '../../../model/rule_migration.gen'; import type { ResourceIdentifier } from '../types'; -const lookupRegex = /\b(?:lookup|inputlookup)\s+([\w-]+)\b/g; // Captures only the lookup name +const lookupRegex = /\b(?:lookup)\s+([\w-]+)\b/g; // Captures only the lookup name const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments export const splResourceIdentifier: ResourceIdentifier = (input) => { @@ -34,7 +33,7 @@ export const splResourceIdentifier: ResourceIdentifier = (input) => { let lookupMatch; while ((lookupMatch = lookupRegex.exec(sanitizedInput)) !== null) { - resources.push({ type: 'lookup', name: lookupMatch[1].replace(/_lookup$/, '') }); // Remove _lookup suffix + resources.push({ type: 'lookup', name: lookupMatch[1].replace(/_lookup$/, '') }); } return resources; @@ -45,16 +44,13 @@ const commentRegex = /```.*?```/g; // Literal strings should be replaced with a placeholder to avoid matching macro and lookup names inside them const doubleQuoteStrRegex = /".*?"/g; const singleQuoteStrRegex = /'.*?'/g; -// inputlookup operator can have modifiers like appent=T before the lookup name, we need to remove them -const inputlookupModifiers = /\binputlookup\b(\s+\w+=\w+)*/gi; // lookup operator can have modifiers like local=true or update=false before the lookup name, we need to remove them -const lookupModifiers = /\blookup\b(\s+\w+=\w+)*/gi; +const lookupModifiers = /\blookup\b\s+((local|update)=\s*(?:true|false)\s*)+/gi; const sanitizeInput = (query: string) => { return query .replaceAll(commentRegex, '') .replaceAll(doubleQuoteStrRegex, '"literal"') .replaceAll(singleQuoteStrRegex, "'literal'") - .replaceAll(lookupModifiers, 'lookup ') - .replaceAll(inputlookupModifiers, 'inputlookup '); + .replaceAll(lookupModifiers, 'lookup '); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts index d19c67b594319..93763d6508cf0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_lookups_client.ts @@ -19,7 +19,7 @@ export class RuleMigrationsDataLookupsClient { ) {} async create(lookupName: string, data: LookupData): Promise { - const indexName = `lookup-${lookupName}`; + const indexName = `lookup_${lookupName}`; try { await this.executeEs(() => this.esClient.indices.create({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index 18d417a23e0dc..988e919943ba1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -53,9 +53,6 @@ export function getRuleMigrationAgent({ return graph; } -/* - * If the original splunk rule has no prebuilt rule match, we will start processing the query, unless it is related to input/outputlookups. - */ const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { if (state.elastic_rule?.prebuilt_rule_id) { return END; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts index 2f3c29f25e0ce..011170da08242 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -21,6 +21,7 @@ interface GetMatchPrebuiltRuleNodeParams { interface GetMatchedRuleResponse { match: string; + summary: string; } export const getMatchPrebuiltRuleNode = ({ @@ -35,6 +36,9 @@ export const getMatchPrebuiltRuleNode = ({ query, techniqueIds.join(',') ); + if (prebuiltRules.length === 0) { + return {}; + } const outputParser = new JsonOutputParser(); const mostRelevantRule = MATCH_PREBUILT_RULE_PROMPT.pipe(model).pipe(outputParser); @@ -62,6 +66,7 @@ export const getMatchPrebuiltRuleNode = ({ const matchedRule = prebuiltRules.find((r) => r.name === response.match); if (matchedRule) { return { + comments: [response.summary], elastic_rule: { title: matchedRule.name, description: matchedRule.description, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts index 12fb7ec70febf..9163ac7ed5b6c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts @@ -28,10 +28,11 @@ Here are some context for you to reference for your task, read it carefully as y -- Always reply with a JSON object with the key "match" and the value being the most relevant matched elastic detection rule name. Do not reply with anything else. +- Always reply with a JSON object with the key "match" and the value being the most relevant matched elastic detection rule name, and a "summary" entry with the reasons behind the match. Do not reply with anything else. - Only reply with exact matches, if you are unsure or do not find a very confident match, always reply with an empty string value in the match key, do not guess or reply with anything else. -- If there is one Elastic rule in the list that covers the same usecase, set the name of the matching rule as a value of the match key. Do not reply with anything else. -- If there are multiple rules in the list that cover the same usecase, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". +- If there is one Elastic rule in the list that covers the same use case, set the name of the matching rule as a value of the match key. Do not reply with anything else. +- If there are multiple rules in the list that cover the same use case, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". +- Finally, write a "summary" in markdown format with the reasons behind the rule matching, or otherwise, why none of the rules suggested matched. Starting with "## Matching Summary". @@ -41,8 +42,10 @@ Description: The following analytic detects the suspicious add user account type A: Please find the match JSON object below: \`\`\`json -{{"match": "Linux User Account Creation"}} +{{"match": "Linux User Account Creation":}} \`\`\` +## Matching Summary +The Splunk rule "Linux Auditd Add User Account Type" is matched with the Elastic rule "Linux User Account Creation" because both rules cover the same use case of detecting user account creation on Linux systems. `, ], diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts index fc6ea5309ced5..b18f60c006549 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts @@ -47,10 +47,13 @@ export function getTranslateRuleGraph({ .addNode('validation', validationNode) .addNode('fixQueryErrors', fixQueryErrorsNode) .addNode('ecsMapping', ecsMappingNode) - .addNode('translationResultNode', translationResultNode) + .addNode('translationResult', translationResultNode) // Edges .addEdge(START, 'inlineQuery') - .addEdge('inlineQuery', 'retrieveIntegrations') + .addConditionalEdges('inlineQuery', translatableRouter, [ + 'retrieveIntegrations', + 'translationResult', + ]) .addEdge('retrieveIntegrations', 'translateRule') .addEdge('translateRule', 'validation') .addEdge('fixQueryErrors', 'validation') @@ -58,15 +61,22 @@ export function getTranslateRuleGraph({ .addConditionalEdges('validation', validationRouter, [ 'fixQueryErrors', 'ecsMapping', - 'translationResultNode', + 'translationResult', ]) - .addEdge('translationResultNode', END); + .addEdge('translationResult', END); const graph = translateRuleGraph.compile(); graph.name = 'Translate Rule Graph'; return graph; } +const translatableRouter = (state: TranslateRuleState) => { + if (!state.inline_query) { + return 'translationResult'; + } + return 'retrieveIntegrations'; +}; + const validationRouter = (state: TranslateRuleState) => { if ( state.validation_errors.iterations <= MAX_VALIDATION_ITERATIONS && @@ -79,5 +89,5 @@ const validationRouter = (state: TranslateRuleState) => { return 'ecsMapping'; } } - return 'translationResultNode'; + return 'translationResult'; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts index b88dd0aed0b59..02d0f4125a56b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts @@ -23,6 +23,12 @@ export const getInlineQueryNode = ({ }: GetInlineQueryNodeParams): GraphNode => { return async (state) => { let query = state.original_rule.query; + + // Check before to avoid unnecessary LLM calls + if (!isSupported(query)) { + return {}; + } + const resources = await ruleMigrationsRetriever.resources.getResources(state.original_rule); if (!isEmpty(resources)) { const replaceQueryParser = new StringOutputParser(); @@ -38,7 +44,19 @@ export const getInlineQueryNode = ({ if (splQuery) { query = splQuery; } + + // Check after replacing in case the replacements made it untranslatable + if (!isSupported(query)) { + return {}; + } } return { inline_query: query }; }; }; + +const isSupported = (query: string) => { + if (query.includes(' inputlookup ')) { + return false; + } + return true; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts index b624604cd1c2e..de436679ad44f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts @@ -90,33 +90,29 @@ Always follow the below guidelines when replacing lookups: - Divide the query up into separate sections and go through each section one at a time to identify the lookups used that need to be replaced, using one of two scenarios: - The lookup is provided in the list of available lookups: Replace the lookup name using its correct name provided. - Remember the "_lookup" suffix in the lookup name in the query can be ignored when checking the list of available lookups - - If the inputlookup or outputlookup command is used, replace it with the correct lookup name. - The lookup is not in the list of available lookups: add a placeholder ("missing placeholder" from now on) in the query with the format [lookup:] including the [] keys, Example: "lookup users uid OUTPUTNEW username, department" -> "[lookup:users]" - The lookup is in the list but has empty name: omit the lookup from the query entirely, as if it was empty. To do so you can use the EVAL command to set the fields to empty strings. Having the following lookups: - "some_table": "lookup-some_table" - "another": "lookup-another-2" + "some_list": "lookup_some_list" + "another": "lookup_another-2" "lookupName3": "" - "someOutputLookup": "lookup-output" And the following SPL query: \`\`\`spl - inputlookup some_table + | lookup some_list name OUTPUT title | lookup another_lookup name OUTPUT description | lookup yetAnotherLookup id OUTPUTNEW someField | lookup lookupName3 uuid OUTPUTNEW group, name - | outputlookup append=T key_field=_key someOutputLookup \`\`\` The correct replacement would be: \`\`\`spl - inputlookup lookup-some_table - | lookup lookup-another-2 name OUTPUT description + | lookup lookup_some_list name OUTPUT title + | lookup lookup_another-2 name OUTPUTNEW description | [lookup:yetAnotherLookup] | EVAL group="", name="" - | outputlookup append=T key_field=_key lookup-output \`\`\` diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts index 289a2f2600fd6..d9ca5fff7889e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts @@ -9,14 +9,15 @@ import { ChatPromptTemplate } from '@langchain/core/prompts'; export const ESQL_SYNTAX_TRANSLATION_PROMPT = ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk SPL to Elasticsearch ES|QL. -Your goal is to translate the SPL query syntax into an equivalent Elastic Search Query Language (ES|QL) query without changing any of the field names, and focusing only on translating the syntax and structure. -You will also we asked to write a summary of the translation process followed at the end. +Your goal is to translate the SPL query syntax into an equivalent Elastic Search Query Language (ES|QL) query without changing any of the field names except lookup lists and macros when relevant and focusing only on translating the syntax and structure. -Here are some context for you to reference for your task, read it carefully: +Here are some context for you to reference for your task, read it carefully as you will get questions about it later: - + +{splunk_rule} + -If in an SPL query you identify a "lookup OUTPUT/OUTPUTNEW" call, it should be translated the following way: +If in an SPL query you identify a looku list call, it should be translated the following way: \`\`\`spl ... | lookup users uid OUTPUTNEW username, department \`\`\` @@ -24,62 +25,31 @@ If in an SPL query you identify a "lookup OUTPUT/OUTPUTNEW" call, it should be t In the above example it uses the following syntax: lookup 'index_name' 'field_to_match' OUTPUTNEW 'field1', 'field2' -The ES|QL translation would be the following, using the LOOKUP JOIN operator and the ON clause: +However in the ES|QL query, some of the information is removed and should be used in the following way: \`\`\`esql ... | LOOKUP JOIN 'index_name' ON 'field_to_match' \`\`\` -Note the fields to be returned are not defined in the ES|QL query, only the index name and the field to match. +We do not define OUTPUTNEW or which fields is returned, only the index name and the field to match. - -If in an SPL query you identify a "inputlookup" call, it should be translated the following way. -- When the inputlookup appears as the source: -\`\`\`spl -| inputlookup users ... -\`\`\` - -It should be translated as a FROM clause in ES|QL. -\`\`\`esql -FROM users ... -\`\`\` - -- When the inputlookup appears as a subquery with a condition, it should be translated using the LOOKUP JOIN operator in ES|QL. -\`\`\`esql -... | inputlookup users WHERE field="value" ... -\`\`\` - -It should be translated as a LOOKUP JOIN operator with the ON clause in ES|QL. -\`\`\`esql -... | LOOKUP JOIN users ON field="value" ... -\`\`\` - - + - Go through each step and part of the splunk rule and query while following the below guide to produce the resulting ES|QL query: - Analyze all the information about the related splunk rule and try to determine the intent of the rule, in order to translate into an equivalent ES|QL rule. -- Go through each part of the SPL query and determine the steps required to produce the same end results using ES|QL. -- Do NOT change the field names defined in the SPL query, keep them as they are in the ES|QL output. -- Always remember to translate any lookup operator using the lookups_guidelines above -- If you encounter any placeholders for missing macros or lookups in the SPL query, like [macro:] or [lookup:], keep them as they are in ES|QL output, even if they cause invalid syntax, and mention they are missing in the summary. - +- Go through each part of the SPL query and determine the steps required to produce the same end results using ES|QL. Only focus on translating the structure without modifying any of the field names. +- Do NOT map any of the fields to the Elastic Common Schema (ECS), this will happen in a later step. +- Always remember to translate any lookup list using the lookup_syntax above -IMPORTANT: -The index pattern to use in the ES|QL query is "{indexPatterns}". -Always start the translated ES|QL query should start with: -\`\`\`esql -FROM {indexPatterns} -| ... -\`\`\` - - + +- Analyze the SPL query and identify the key components. +- Do NOT translate the field names of the SPL query. +- Always start the resulting ES|QL query by filtering using FROM and with these index pattern: {indexPatterns}. +- Always remember to translate any lookup list using the lookup_syntax above +- Always remember to replace macro call with the appropriate placeholder as defined in the macro info. + - First, the ES|QL query inside an \`\`\`esql code block. - At the end, the summary of the translation process followed in markdown, starting with "## Translation Summary". - - -{splunk_rule} - `); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts index e09c4e6961141..051341c770e3b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts @@ -7,7 +7,6 @@ import type { Logger } from '@kbn/core/server'; import type { InferenceClient } from '@kbn/inference-plugin/server'; -import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_caller'; import type { GraphNode } from '../../types'; import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts'; @@ -42,15 +41,12 @@ export const getTranslateRuleNode = ({ }); const response = await esqlKnowledgeBaseCaller(prompt); - const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; const translationSummary = response.match(/## Translation Summary[\s\S]*$/)?.[0] ?? ''; - const translationResult = getTranslationResult(esqlQuery); - return { response, comments: [translationSummary], - translation_result: translationResult, elastic_rule: { title: state.original_rule.title, integration_id: integrationId, @@ -62,10 +58,3 @@ export const getTranslateRuleNode = ({ }; }; }; - -const getTranslationResult = (esqlQuery: string): RuleTranslationResult => { - if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { - return RuleTranslationResult.PARTIAL; - } - return RuleTranslationResult.FULL; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts index 4bebc04d7ad26..90e4eeb54908a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts @@ -9,32 +9,32 @@ import { RuleTranslationResult } from '../../../../../../../../../../common/siem import type { GraphNode } from '../../types'; export const translationResultNode: GraphNode = async (state) => { - const query = state.elastic_rule?.query; + // Set defaults + const elasticRule = { + title: state.original_rule.title, + description: state.original_rule.description || state.original_rule.title, + severity: 'low', + ...state.elastic_rule, + }; - if (query) { - /** - * When rule translation happens without any related integrations found we reuse the logs-* pattern to make validation easier. - * However we want to replace this with a value to notify the end user that it needs to be replaced. - */ - if (query.includes(' logs-*')) { - const newQuery = query.replace('logs-*', '[indexPattern]'); - return { - elastic_rule: { ...state.elastic_rule, query: newQuery }, - translation_result: RuleTranslationResult.PARTIAL, - }; - } - /** - * When rule translation misses macro or lookup a placeholder is added to the query - * to notify the user that it needs to be provided/replaced. - */ - if (query.match(/\[(macro|lookup):/)) { - return { translation_result: RuleTranslationResult.PARTIAL }; - } - } + const query = elasticRule.query; + let translationResult; - if (!state.translation_result) { - return { translation_result: RuleTranslationResult.UNTRANSLATABLE }; + if (!query) { + translationResult = RuleTranslationResult.UNTRANSLATABLE; + } else { + if (state.validation_errors?.esql_errors) { + translationResult = RuleTranslationResult.PARTIAL; + } else if (query.startsWith('FROM logs-*')) { + elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]'); + translationResult = RuleTranslationResult.PARTIAL; + } else if (query.match(/\[(macro|lookup):.*?\]/)) { + translationResult = RuleTranslationResult.PARTIAL; + } } - return {}; + return { + elastic_rule: elasticRule, + translation_result: translationResult, + }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts index 873f1880d2252..e16589f85d2fe 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts @@ -38,9 +38,9 @@ export const translateRuleState = Annotation.Root({ reducer: (current, value) => value ?? current, default: () => '', }), - elastic_rule: Annotation({ + elastic_rule: Annotation>({ reducer: (state, action) => ({ ...state, ...action }), - default: () => ({} as ElasticRule), + default: () => ({}), }), validation_errors: Annotation({ reducer: (current, value) => value ?? current, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 29852558cda48..196281c150db9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -32,13 +32,13 @@ export class RuleMigrationsRetriever { public async initialize() { await Promise.all([ this.resources.initialize(), - // Populates the indices used for RAG searches on prebuilt rules and integrations. - this.clients.data.prebuiltRules.create({ - rulesClient: this.clients.rules, - soClient: this.clients.savedObjects, - }), - // Will use Fleet API client for integration retrieval as an argument once feature is available - this.clients.data.integrations.create(), + // // Populates the indices used for RAG searches on prebuilt rules and integrations. + // this.clients.data.prebuiltRules.create({ + // rulesClient: this.clients.rules, + // soClient: this.clients.savedObjects, + // }), + // // Will use Fleet API client for integration retrieval as an argument once feature is available + // this.clients.data.integrations.create(), ]).catch((error) => { throw new Error(`Failed to initialize RuleMigrationsRetriever: ${error}`); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index 1edd1b449070c..beb614b0ec748 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -108,15 +108,19 @@ export class RuleMigrationsTaskClient { await Promise.all( ruleMigrations.map(async (ruleMigration) => { this.logger.debug(`Starting migration of rule "${ruleMigration.original_rule.title}"`); + if (ruleMigration.elastic_rule?.id) { + await this.data.rules.saveCompleted(ruleMigration); + return; // skip already installed rules + } try { const start = Date.now(); - const invocation = agent.invoke( - { original_rule: ruleMigration.original_rule }, - config - ); + const invocationData = { original_rule: ruleMigration.original_rule }; + // using withAbortRace is a workaround for the issue with the langGraph signal not working properly - const migrationResult = await withAbortRace(invocation); + const migrationResult = await withAbortRace( + agent.invoke(invocationData, config) + ); const duration = (Date.now() - start) / 1000; this.logger.debug( @@ -190,16 +194,16 @@ export class RuleMigrationsTaskClient { rules: rulesClient, savedObjects: soClient, }); + await ruleMigrationsRetriever.initialize(); - const agent = getRuleMigrationAgent({ + return getRuleMigrationAgent({ connectorId, model, inferenceClient, ruleMigrationsRetriever, logger: this.logger, }); - return agent; } /** Updates all the rules in a migration to be re-executed */ From 9cf201988ef62e1f0c2fb162e08f2a07d1eb0644 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 19 Dec 2024 21:01:34 +0100 Subject: [PATCH 3/9] replace index pattern first --- .../nodes/translation_result/translation_result.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts index 90e4eeb54908a..374011a90fc1d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts @@ -23,11 +23,11 @@ export const translationResultNode: GraphNode = async (state) => { if (!query) { translationResult = RuleTranslationResult.UNTRANSLATABLE; } else { - if (state.validation_errors?.esql_errors) { - translationResult = RuleTranslationResult.PARTIAL; - } else if (query.startsWith('FROM logs-*')) { + if (query.startsWith('FROM logs-*')) { elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]'); translationResult = RuleTranslationResult.PARTIAL; + } else if (state.validation_errors?.esql_errors) { + translationResult = RuleTranslationResult.PARTIAL; } else if (query.match(/\[(macro|lookup):.*?\]/)) { translationResult = RuleTranslationResult.PARTIAL; } From 85221a54558078f0e4bf8ef9d35c29e25d0ead8c Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Fri, 20 Dec 2024 10:24:30 +0100 Subject: [PATCH 4/9] uncomment code --- .../task/agent/sub_graphs/translate_rule/state.ts | 4 ++-- .../task/retrievers/rule_migrations_retriever.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts index e16589f85d2fe..16d668a698488 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts @@ -9,7 +9,7 @@ import type { BaseMessage } from '@langchain/core/messages'; import { Annotation, messagesStateReducer } from '@langchain/langgraph'; import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; import type { - ElasticRule, + ElasticRulePartial, OriginalRule, RuleMigration, } from '../../../../../../../../common/siem_migrations/model/rule_migration.gen'; @@ -38,7 +38,7 @@ export const translateRuleState = Annotation.Root({ reducer: (current, value) => value ?? current, default: () => '', }), - elastic_rule: Annotation>({ + elastic_rule: Annotation({ reducer: (state, action) => ({ ...state, ...action }), default: () => ({}), }), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 196281c150db9..29852558cda48 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -32,13 +32,13 @@ export class RuleMigrationsRetriever { public async initialize() { await Promise.all([ this.resources.initialize(), - // // Populates the indices used for RAG searches on prebuilt rules and integrations. - // this.clients.data.prebuiltRules.create({ - // rulesClient: this.clients.rules, - // soClient: this.clients.savedObjects, - // }), - // // Will use Fleet API client for integration retrieval as an argument once feature is available - // this.clients.data.integrations.create(), + // Populates the indices used for RAG searches on prebuilt rules and integrations. + this.clients.data.prebuiltRules.create({ + rulesClient: this.clients.rules, + soClient: this.clients.savedObjects, + }), + // Will use Fleet API client for integration retrieval as an argument once feature is available + this.clients.data.integrations.create(), ]).catch((error) => { throw new Error(`Failed to initialize RuleMigrationsRetriever: ${error}`); }); From f33fb4550628d0ebdfb085e189473e48fe56ccff Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Fri, 20 Dec 2024 10:54:34 +0100 Subject: [PATCH 5/9] type fixes --- .../migration_status_panels/migration_ready_panel.tsx | 4 ++-- .../migration_status_panels/upload_missing_panel.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx index 3c230cba4c34f..d10ac6f6eb551 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx @@ -8,7 +8,7 @@ import React, { useCallback, useEffect } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiButton, EuiPanel } from '@elastic/eui'; import { CenteredLoadingSpinner } from '../../../../common/components/centered_loading_spinner'; -import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { PanelText } from '../../../../common/components/panel_text'; import { useStartMigration } from '../../service/hooks/use_start_migration'; import type { RuleMigrationStats } from '../../types'; @@ -21,7 +21,7 @@ export interface MigrationReadyPanelProps { } export const MigrationReadyPanel = React.memo(({ migrationStats }) => { const { openFlyout } = useRuleMigrationDataInputContext(); - const [missingResources, setMissingResources] = React.useState([]); + const [missingResources, setMissingResources] = React.useState([]); const { getMissingResources, isLoading } = useGetMissingResources(setMissingResources); useEffect(() => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx index f1c6bdd71613a..569c726c9a87e 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/upload_missing_panel.tsx @@ -16,7 +16,7 @@ import { } from '@elastic/eui'; import { AssistantIcon } from '@kbn/ai-assistant-icon'; import type { SpacerSize } from '@elastic/eui/src/components/spacer/spacer'; -import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationResourceBase } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { PanelText } from '../../../../common/components/panel_text'; import { useGetMissingResources } from '../../service/hooks/use_get_missing_resources'; import * as i18n from './translations'; @@ -31,7 +31,7 @@ export const RuleMigrationsUploadMissingPanel = React.memo { const { euiTheme } = useEuiTheme(); const { openFlyout } = useRuleMigrationDataInputContext(); - const [missingResources, setMissingResources] = React.useState([]); + const [missingResources, setMissingResources] = React.useState([]); const { getMissingResources, isLoading } = useGetMissingResources(setMissingResources); useEffect(() => { From eea54873a755469d1b34b1a7b34284f05e98aeb9 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Fri, 20 Dec 2024 17:56:14 +0100 Subject: [PATCH 6/9] fix test --- .../rules/data/rule_migrations_data_service.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts index 461ed98fe058e..cb6c94ebae40f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts @@ -61,7 +61,7 @@ describe('SiemRuleMigrationsDataService', () => { expect.objectContaining({ name: `${INDEX_PATTERN}-integrations` }) ); expect(prebuiltRulesAdapter.setComponentTemplate).toHaveBeenCalledWith( - expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltRules` }) + expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltrules` }) ); }); @@ -79,7 +79,7 @@ describe('SiemRuleMigrationsDataService', () => { expect.objectContaining({ name: `${INDEX_PATTERN}-integrations` }) ); expect(prebuiltRulesAdapter.setIndexTemplate).toHaveBeenCalledWith( - expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltRules` }) + expect.objectContaining({ name: `${INDEX_PATTERN}-prebuiltrules` }) ); }); }); From f758e7df57cd4e3dfaa9f0da218a102b6ef07409 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Sat, 21 Dec 2024 17:23:22 +0100 Subject: [PATCH 7/9] test fixed --- .../siem_migrations/rules/siem_rule_migrations_service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 2e701f0fc7b3e..a25fa7d576114 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -95,7 +95,7 @@ describe('SiemRuleMigrationsService', () => { expect(mockDataCreateClient).toHaveBeenCalledWith({ spaceId: createClientParams.spaceId, currentUser: createClientParams.currentUser, - esClient: esClusterClient.asInternalUser, + esScopedClient: esClusterClient.asScoped(), }); }); From cc221eb5d80f4a8e646241dcc805b0bca2cda141 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Mon, 23 Dec 2024 17:22:35 +0100 Subject: [PATCH 8/9] translation agent improvements --- .../index-adapter/src/field_maps/types.ts | 13 +++--- ...ule_migrations_data_integrations_client.ts | 24 +++-------- ...e_migrations_data_prebuilt_rules_client.ts | 2 +- .../rules/data/rule_migrations_field_maps.ts | 11 ++--- .../match_prebuilt_rule.ts | 12 ++++-- .../nodes/match_prebuilt_rule/prompts.ts | 28 +++++++++---- .../siem_migrations/rules/task/agent/state.ts | 4 +- .../nodes/ecs_mapping/ecs_mapping.ts | 3 +- .../nodes/inline_query/inline_query.ts | 24 +++++++---- .../nodes/inline_query/prompts.ts | 41 +++++++++---------- .../nodes/retrieve_integrations/prompts.ts | 30 ++++++++++---- .../retrieve_integrations.ts | 24 ++++++----- .../nodes/translate_rule/prompts.ts | 5 ++- .../nodes/translate_rule/translate_rule.ts | 6 +-- .../translation_result/translation_result.ts | 2 + .../nodes/validation/validation.ts | 30 ++++++++------ .../agent/sub_graphs/translate_rule/state.ts | 6 +-- .../task/retrievers/integration_retriever.ts | 19 ++++----- .../retrievers/prebuilt_rules_retriever.ts | 27 ++++++------ .../retrievers/rule_migrations_retriever.ts | 19 ++++----- .../rules/task/rule_migrations_task_client.ts | 1 + .../rules/task/util/comments.ts | 11 +++++ .../server/lib/siem_migrations/rules/types.ts | 6 +-- 23 files changed, 196 insertions(+), 152 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts diff --git a/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts b/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts index 9c96fa2d796f6..1906f899e9131 100644 --- a/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts +++ b/x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts @@ -61,11 +61,14 @@ export type FieldMap = Record< // This utility type flattens all the keys of a schema object and its nested objects as a union type. // Its purpose is to ensure that the FieldMap keys are always in sync with the schema object. // It assumes all optional fields of the schema are required in the field map, they can always be omitted from the resulting type. -export type SchemaFieldMapKeys< - T extends Record, - Key = keyof T -> = Key extends string - ? NonNullable extends Record +// We need to use any to avoid TS errors since interfaces do not satisfy Record, but they do satisfy Record. +/* eslint-disable @typescript-eslint/no-explicit-any */ +export type SchemaFieldMapKeys, Key = keyof T> = Key extends string + ? NonNullable extends any[] + ? NonNullable extends Array> + ? `${Key}` | `${Key}.${SchemaFieldMapKeys[number]>}` + : `${Key}` + : NonNullable extends Record ? `${Key}` | `${Key}.${SchemaFieldMapKeys>}` : `${Key}` : never; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts index fdb063836f9e4..2308a37252a93 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { Integration } from '../types'; +import type { RuleMigrationIntegration } from '../types'; import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; /* This will be removed once the package registry changes is performed */ @@ -17,7 +17,7 @@ const MIN_SCORE = 40 as const; const RETURNED_INTEGRATIONS = 5 as const; /* This is a temp implementation to allow further development until https://github.com/elastic/package-registry/issues/1252 */ -const INTEGRATIONS = integrationsFile as Integration[]; +const INTEGRATIONS = integrationsFile as RuleMigrationIntegration[]; /* BULK_MAX_SIZE defines the number to break down the bulk operations by. * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ @@ -52,30 +52,18 @@ export class RuleMigrationsDataIntegrationsClient extends RuleMigrationsDataBase } /** Based on a LLM generated semantic string, returns the 5 best results with a score above 40 */ - async retrieveIntegrations(semanticString: string): Promise { + async retrieveIntegrations(semanticString: string): Promise { const index = await this.getIndexName(); const query = { bool: { should: [ - { - semantic: { - query: semanticString, - field: 'elser_embedding', - boost: 1.5, - }, - }, - { - multi_match: { - query: semanticString, - fields: ['title^2', 'description'], - boost: 3, - }, - }, + { semantic: { query: semanticString, field: 'elser_embedding', boost: 1.5 } }, + { multi_match: { query: semanticString, fields: ['title^2', 'description'], boost: 3 } }, ], }, }; const results = await this.esClient - .search({ + .search({ index, query, size: RETURNED_INTEGRATIONS, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts index ccd158c347c77..ccc0b31b31dc3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts @@ -50,7 +50,7 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas filteredRules.push({ rule_id: rule.rule_id, name: rule.name, - installedRuleId: ruleVersions.current?.id, + installed_rule_id: ruleVersions.current?.id, description: rule.description, elser_embedding: `${rule.name} - ${rule.description}`, ...(mitreAttackIds?.length && { mitre_attack_ids: mitreAttackIds }), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts index c03b7ef52e784..9369b3b5e1bc7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts @@ -10,6 +10,7 @@ import type { RuleMigration, RuleMigrationResource, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationIntegration, RuleMigrationPrebuiltRule } from '../types'; export const ruleMigrationsFieldMap: FieldMap>> = { '@timestamp': { type: 'date', required: false }, @@ -52,22 +53,22 @@ export const ruleMigrationResourcesFieldMap: FieldMap< updated_by: { type: 'keyword', required: false }, }; -export const integrationsFieldMap: FieldMap = { - '@timestamp': { type: 'date', required: true }, +export const integrationsFieldMap: FieldMap> = { + id: { type: 'keyword', required: true }, title: { type: 'text', required: true }, description: { type: 'text', required: true }, data_streams: { type: 'nested', array: true, required: true }, 'data_streams.dataset': { type: 'keyword', required: true }, 'data_streams.title': { type: 'text', required: true }, 'data_streams.index_pattern': { type: 'keyword', required: true }, - elser_embeddings: { type: 'semantic_text', required: true }, + elser_embedding: { type: 'semantic_text', required: true }, }; -export const prebuiltRulesFieldMap: FieldMap = { - '@timestamp': { type: 'date', required: true }, +export const prebuiltRulesFieldMap: FieldMap> = { name: { type: 'text', required: true }, description: { type: 'text', required: true }, elser_embedding: { type: 'semantic_text', required: true }, rule_id: { type: 'keyword', required: true }, + installed_rule_id: { type: 'keyword', required: true }, mitre_attack_ids: { type: 'keyword', array: true, required: false }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts index 011170da08242..fd3dfc983443e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -12,6 +12,7 @@ import type { RuleMigrationsRetriever } from '../../../retrievers'; import type { ChatModel } from '../../../util/actions_client_chat'; import type { GraphNode } from '../../types'; import { MATCH_PREBUILT_RULE_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../util/comments'; interface GetMatchPrebuiltRuleNodeParams { model: ChatModel; @@ -37,7 +38,7 @@ export const getMatchPrebuiltRuleNode = ({ techniqueIds.join(',') ); if (prebuiltRules.length === 0) { - return {}; + return { comments: ['## Prebuilt Rule Matching Summary\nNo related prebuilt rule found.'] }; } const outputParser = new JsonOutputParser(); @@ -62,21 +63,24 @@ export const getMatchPrebuiltRuleNode = ({ rules: JSON.stringify(elasticSecurityRules, null, 2), splunk_rule: JSON.stringify(splunkRule, null, 2), })) as GetMatchedRuleResponse; + + const comments = response.summary ? [cleanMarkdown(response.summary)] : undefined; + if (response.match) { const matchedRule = prebuiltRules.find((r) => r.name === response.match); if (matchedRule) { return { - comments: [response.summary], + comments, elastic_rule: { title: matchedRule.name, description: matchedRule.description, - id: matchedRule.installedRuleId, + id: matchedRule.installed_rule_id, prebuilt_rule_id: matchedRule.rule_id, }, translation_result: RuleTranslationResult.FULL, }; } } - return {}; + return { comments }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts index 9163ac7ed5b6c..c109a8dc3dd8d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts @@ -10,7 +10,8 @@ export const MATCH_PREBUILT_RULE_PROMPT = ChatPromptTemplate.fromMessages([ [ 'system', `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. -You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Detection Rule that covers the same threat, if any. +You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Prebuilt Rule for the same purpose, if any. + Here are some context for you to reference for your task, read it carefully as you will get questions about it later: @@ -22,18 +23,26 @@ Here are some context for you to reference for your task, read it carefully as y ], [ 'human', - `See the below description of the relevant splunk rule and try to match it with any of the elastic detection rules with similar names. + `See the below description of the splunk rule, try to find a Elastic Prebuilt Rule with similar purpose. {splunk_rule} +- Carefully analyze the Splunk Detection Rule data provided by the user. +- Match the Splunk rule to the most relevant Elastic Prebuilt Rules from the list provided above. +- If no related Elastic Prebuilt Rule is found, reply with an empty string. +- Provide a concise reasoning summary for your decision, explaining why the selected Prebuilt Rule is the best fit, or why no suitable match was found. + + + - Always reply with a JSON object with the key "match" and the value being the most relevant matched elastic detection rule name, and a "summary" entry with the reasons behind the match. Do not reply with anything else. - Only reply with exact matches, if you are unsure or do not find a very confident match, always reply with an empty string value in the match key, do not guess or reply with anything else. -- If there is one Elastic rule in the list that covers the same use case, set the name of the matching rule as a value of the match key. Do not reply with anything else. -- If there are multiple rules in the list that cover the same use case, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". -- Finally, write a "summary" in markdown format with the reasons behind the rule matching, or otherwise, why none of the rules suggested matched. Starting with "## Matching Summary". - +- If there is only one match, answer with the name of the rule in the "match" key. Do not reply with anything else. +- If there are multiple matches, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". +- Finally, write a "summary" in markdown format with the reasoning behind the decision. Starting with "## Prebuilt Rule Matching Summary\n". +- Make sure the JSON object is formatted correctly and the values properly escaped. + U: @@ -42,10 +51,11 @@ Description: The following analytic detects the suspicious add user account type A: Please find the match JSON object below: \`\`\`json -{{"match": "Linux User Account Creation":}} +{{ + "match": "Linux User Account Creation", + "summary": "## Prebuilt Rule Matching Summary\\\nThe Splunk rule \"Linux Auditd Add User Account Type\" is matched with the Elastic rule \"Linux User Account Creation\" because both rules cover the same use case of detecting user account creation on Linux systems." +}} \`\`\` -## Matching Summary -The Splunk rule "Linux Auditd Add User Account Type" is matched with the Elastic rule "Linux User Account Creation" because both rules cover the same use case of detecting user account creation on Linux systems. `, ], diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts index 66b3c0c8e7a71..cf07e676715a7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -7,6 +7,7 @@ import type { BaseMessage } from '@langchain/core/messages'; import { Annotation, messagesStateReducer } from '@langchain/langgraph'; +import { uniq } from 'lodash/fp'; import type { RuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; import type { ElasticRule, @@ -33,7 +34,8 @@ export const migrateRuleState = Annotation.Root({ }), translation_result: Annotation(), comments: Annotation({ - reducer: (current, value) => (value ? (current ?? []).concat(value) : current), + // Translation subgraph causes the original main graph comments to be concatenated again, we need to deduplicate them. + reducer: (current, value) => uniq(value ? (current ?? []).concat(value) : current), default: () => [], }), response: Annotation(), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts index 08a6c5904819a..cddf85dea2d31 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts @@ -12,6 +12,7 @@ import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_ca import type { GraphNode } from '../../types'; import { SIEM_RULE_MIGRATION_CIM_ECS_MAP } from './cim_ecs_map'; import { ESQL_TRANSLATE_ECS_MAPPING_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetEcsMappingNodeParams { inferenceClient: InferenceClient; @@ -47,7 +48,7 @@ export const getEcsMappingNode = ({ return { response, - comments: [ecsSummary], + comments: [cleanMarkdown(ecsSummary)], translation_finalized: true, translation_result: translationResult, elastic_rule: { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts index 02d0f4125a56b..e5d16f59658c3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts @@ -11,6 +11,7 @@ import type { RuleMigrationsRetriever } from '../../../../../retrievers'; import type { ChatModel } from '../../../../../util/actions_client_chat'; import type { GraphNode } from '../../../../types'; import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetInlineQueryNodeParams { model: ChatModel; @@ -25,8 +26,9 @@ export const getInlineQueryNode = ({ let query = state.original_rule.query; // Check before to avoid unnecessary LLM calls - if (!isSupported(query)) { - return {}; + let unsupportedComment = getUnsupportedComment(query); + if (unsupportedComment) { + return { comments: [unsupportedComment] }; } const resources = await ruleMigrationsRetriever.resources.getResources(state.original_rule); @@ -40,23 +42,27 @@ export const getInlineQueryNode = ({ macros: resourceContext.macros, lookups: resourceContext.lookups, }); - const splQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1] ?? ''; + const splQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; + const inliningSummary = response.match(/## Inlining Summary[\s\S]*$/)?.[0] ?? ''; if (splQuery) { query = splQuery; } // Check after replacing in case the replacements made it untranslatable - if (!isSupported(query)) { - return {}; + unsupportedComment = getUnsupportedComment(query); + if (unsupportedComment) { + return { comments: [unsupportedComment] }; } + + return { inline_query: query, comments: [cleanMarkdown(inliningSummary)] }; } return { inline_query: query }; }; }; -const isSupported = (query: string) => { - if (query.includes(' inputlookup ')) { - return false; +const getUnsupportedComment = (query: string): string | undefined => { + const unsupportedText = '## Translation Summary\nCan not create custom translation.\n'; + if (query.includes(' inputlookup')) { + return `${unsupportedText}Reason: \`inputlookup\` command is not supported.`; } - return true; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts index de436679ad44f..49f2b2b57f479 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts @@ -43,7 +43,7 @@ export const REPLACE_QUERY_RESOURCE_PROMPT = ChatPromptTemplate.fromMessages([ [ 'system', `You are an agent expert in Splunk SPL (Search Processing Language). -Your task is to replace macros and lookups syntax in a SPL query, using the actual content of the macros and lookup names provided to you. +Your task is to replace macros and lookups syntax in a SPL query, using the actual content of the macros and lookup names provided to you. And write a summary of the replacement steps made. Here are some context for you to reference for your task, read it carefully as you will get questions about it later: @@ -84,15 +84,15 @@ The correct replacement would be: -You have to replace the lookup names in the SPL query with the correct name, if provided. +You have to replace the lookup names in the SPL query with the Elastic name, if provided. Always follow the below guidelines when replacing lookups: - Divide the query up into separate sections and go through each section one at a time to identify the lookups used that need to be replaced, using one of two scenarios: - - The lookup is provided in the list of available lookups: Replace the lookup name using its correct name provided. - - Remember the "_lookup" suffix in the lookup name in the query can be ignored when checking the list of available lookups + - The lookup is provided in the list of available lookups: Replace the lookup name using its Elastic name provided. + - Remember the "_lookup" suffix in the lookup name in the query should be ignored when checking the list of available lookups - The lookup is not in the list of available lookups: add a placeholder ("missing placeholder" from now on) in the query with the format [lookup:] including the [] keys, Example: "lookup users uid OUTPUTNEW username, department" -> "[lookup:users]" - - The lookup is in the list but has empty name: omit the lookup from the query entirely, as if it was empty. To do so you can use the EVAL command to set the fields to empty strings. + - The lookup is in the list but has empty name: omit the lookup from the query entirely, as if it was empty. To do so you can use the "eval" command to set the fields to empty strings. Having the following lookups: "some_list": "lookup_some_list" @@ -103,7 +103,7 @@ And the following SPL query: \`\`\`spl | lookup some_list name OUTPUT title | lookup another_lookup name OUTPUT description - | lookup yetAnotherLookup id OUTPUTNEW someField + | lookup yet_another_lookup id OUTPUTNEW someField | lookup lookupName3 uuid OUTPUTNEW group, name \`\`\` @@ -111,14 +111,14 @@ The correct replacement would be: \`\`\`spl | lookup lookup_some_list name OUTPUT title | lookup lookup_another-2 name OUTPUTNEW description - | [lookup:yetAnotherLookup] - | EVAL group="", name="" + | [lookup:yet_another] + | eval group="", name="" \`\`\` - The original and modified queries must be equivalent, except for the "missing placeholders". -- You must respond only with the modified query inside a \`\`\`spl code block, nothing else similar to the example response below. +- You must respond with the modified query inside a \`\`\`spl code block, followed by a summary of the replacement steps made in markdown format, starting with "## Inlining Summary". `, @@ -140,19 +140,16 @@ The correct replacement would be: \`\`\` - -A: Please find the modified SPL query below: -\`\`\`spl -sourcetype="linux:audit" \`linux_auditd_normalized_proctitle_process\` -| rename host as dest -| where LIKE (process_exec, "%chown root%") -| stats count min(_time) as firstTime max(_time) as lastTime by process_exec proctitle normalized_proctitle_delimiter dest -| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(firstTime) -| convert timeformat="%Y-%m-%dT%H:%M:%S" ctime(lastTime) -| search * -\`\`\` - - + +- First, the modified SPL query inside an \`\`\`spl code block. +- At the end, a step by step explanation of the replacements made in markdown format: + - Start with "## Inlining Summary" title + - Followed by information about the original query + - Then, the steps taken to replace the macros and lookups + - Finally, the modified SPL query + - Inside SPL language code blocks, Please add a line break before each pipe (|) character in the query. + - Make sure the Markdown is formatted correctly and the values properly escaped. + `, ], ['ai', 'Please find the modified SPL query below:'], diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts index d562bb31b1628..786f737990661 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/prompts.ts @@ -9,10 +9,13 @@ import { ChatPromptTemplate } from '@langchain/core/prompts'; export const MATCH_INTEGRATION_PROMPT = ChatPromptTemplate.fromMessages([ [ 'system', - `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. -You will be provided with a Splunk Detection Rule name by the user, your goal is to try to find the most relevant Elastic Integration from the integration list below if any, and return either the most relevant. If none seems relevant you should always return empty. -Here are some context for you to reference for your task, read it carefully as you will get questions about it later: + `You are a Cybersecurity expert specializing in SIEM solutions. Your task is to assist in migrating detection rules from Splunk Security to Elastic Security by identifying the most appropriate Elastic Integration for a given Splunk rule. +Elastic Integrations are pre-built packages that ingest data from various sources into Elastic Security. +They enable Elastic Security detection rules to monitor environments, detect threats, and ensure a strong security posture. +Your goal is to identify the Elastic Integration that aligns best with the data source referenced in the provided Splunk rule. + +Here is the Elastic integrations context for you to reference for your task, read it carefully as you will get questions about it later: {integrations} @@ -28,19 +31,30 @@ Here are some context for you to reference for your task, read it carefully as y -- Always reply with a JSON object with the key "match" and the value being the most relevant matched integration title. Do not reply with anything else. -- Only reply with exact matches, if you are unsure or do not find a very confident match, always reply with an empty string value in the match key, do not guess or reply with anything else. -- If there is one elastic integration in the list that covers the relevant usecase, set the title of the matching integration as a value of the match key. Do not reply with anything else. -- If there are multiple elastic integrations in the list that cover the same usecase, answer with the most specific of them, for example if the rule is related to "Sysmon" then the Sysmon integration is more specific than Windows. +- Carefully analyze the Splunk Detection Rule data provided by the user. +- Match the data source in the Splunk rule to the most relevant Elastic Integration from the list provided above. +- If no related integration is found, reply with an empty string. +- Provide a concise reasoning summary for your decision, explaining why the selected integration is the best fit or why no suitable match was found. + +- Always reply with a JSON object with the key "match" and the value being the most relevant matched integration title, and a "summary" entry with the reasons behind the match. Do not reply with anything else. +- Only reply with exact matches or an empty string inside the "match" value, do not guess or reply with anything else. +- If there are multiple elastic integrations in the list that match, answer the most specific of them, for example if the rule is related to "Sysmon" then the Sysmon integration is more specific than Windows. +- Finally, write a "summary" in markdown format with the reasoning behind the integration matching, or otherwise, why none of the integrations suggested matched. Starting with "## Integration Matching Summary\n". +- Make sure the JSON object is formatted correctly and the values properly escaped. + + U: Linux Auditd Add User Account Type A: Please find the match JSON object below: \`\`\`json -{{"match": "auditd_manager"}} +{{ + "match": "auditd_manager", + "summary": "## Integration Matching Summary\\\nThe Splunk rule \"Linux Auditd Add User Account Type\" is matched with the \"auditd_manager\" integration because it ingests data from auditd logs which is the right data to detect user account creation on Linux systems." +}} \`\`\` `, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts index c30b99d0ba605..11c9069fc77ac 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts @@ -10,6 +10,7 @@ import type { RuleMigrationsRetriever } from '../../../../../retrievers'; import type { ChatModel } from '../../../../../util/actions_client_chat'; import type { GraphNode } from '../../types'; import { MATCH_INTEGRATION_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetRetrieveIntegrationsNodeParams { model: ChatModel; @@ -18,6 +19,7 @@ interface GetRetrieveIntegrationsNodeParams { interface GetMatchedIntegrationResponse { match: string; + summary: string; } export const getRetrieveIntegrationsNode = ({ @@ -29,18 +31,16 @@ export const getRetrieveIntegrationsNode = ({ const integrations = await ruleMigrationsRetriever.integrations.getIntegrations(query); if (integrations.length === 0) { - return {}; + return { comments: ['## Integration Matching Summary\nNo related integration found.'] }; } const outputParser = new JsonOutputParser(); const mostRelevantIntegration = MATCH_INTEGRATION_PROMPT.pipe(model).pipe(outputParser); - const elasticSecurityIntegrations = integrations.map((integration) => { - return { - title: integration.title, - description: integration.description, - }; - }); + const integrationsInfo = integrations.map((integration) => ({ + title: integration.title, + description: integration.description, + })); const splunkRule = { title: state.original_rule.title, description: state.original_rule.description, @@ -49,16 +49,20 @@ export const getRetrieveIntegrationsNode = ({ /* * Takes the most relevant integration from the array of integration(s) returned by the semantic query, returns either the most relevant or none. */ + const integrationsJson = JSON.stringify(integrationsInfo, null, 2); const response = (await mostRelevantIntegration.invoke({ - integrations: JSON.stringify(elasticSecurityIntegrations, null, 2), + integrations: integrationsJson, splunk_rule: JSON.stringify(splunkRule, null, 2), })) as GetMatchedIntegrationResponse; + + const comments = response.summary ? [cleanMarkdown(response.summary)] : undefined; + if (response.match) { const matchedIntegration = integrations.find((r) => r.title === response.match); if (matchedIntegration) { - return { integration: matchedIntegration }; + return { integration: matchedIntegration, comments }; } } - return {}; + return { comments }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts index d9ca5fff7889e..23a6829e6235d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts @@ -17,7 +17,7 @@ Here are some context for you to reference for your task, read it carefully as y {splunk_rule} -If in an SPL query you identify a looku list call, it should be translated the following way: +If in an SPL query you identify a lookup call, it should be translated the following way: \`\`\`spl ... | lookup users uid OUTPUTNEW username, department \`\`\` @@ -39,7 +39,6 @@ Go through each step and part of the splunk rule and query while following the b - Do NOT map any of the fields to the Elastic Common Schema (ECS), this will happen in a later step. - Always remember to translate any lookup list using the lookup_syntax above - - Analyze the SPL query and identify the key components. - Do NOT translate the field names of the SPL query. @@ -51,5 +50,7 @@ Go through each step and part of the splunk rule and query while following the b - First, the ES|QL query inside an \`\`\`esql code block. - At the end, the summary of the translation process followed in markdown, starting with "## Translation Summary". + - Inside SPL language code blocks, Please add a line break before each pipe (|) character in the query. + - Make sure the Markdown is formatted correctly and the values properly escaped. `); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts index 051341c770e3b..d360b6a1c246d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts @@ -10,6 +10,7 @@ import type { InferenceClient } from '@kbn/inference-plugin/server'; import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_caller'; import type { GraphNode } from '../../types'; import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts'; +import { cleanMarkdown } from '../../../../../util/comments'; interface GetTranslateRuleNodeParams { inferenceClient: InferenceClient; @@ -46,12 +47,9 @@ export const getTranslateRuleNode = ({ return { response, - comments: [translationSummary], + comments: [cleanMarkdown(translationSummary)], elastic_rule: { - title: state.original_rule.title, integration_id: integrationId, - description: state.original_rule.description, - severity: 'low', query: esqlQuery, query_language: 'esql', }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts index 374011a90fc1d..c70b4d3445ec7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts @@ -30,6 +30,8 @@ export const translationResultNode: GraphNode = async (state) => { translationResult = RuleTranslationResult.PARTIAL; } else if (query.match(/\[(macro|lookup):.*?\]/)) { translationResult = RuleTranslationResult.PARTIAL; + } else { + translationResult = RuleTranslationResult.FULL; } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts index 0cef5df6a260f..7d5e2104a3ae4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts @@ -23,22 +23,26 @@ export const getValidationNode = ({ logger }: GetValidationNodeParams): GraphNod const query = state.elastic_rule.query; // We want to prevent infinite loops, so we increment the iterations counter for each validation run. - const currentIteration = ++state.validation_errors.iterations; + const currentIteration = state.validation_errors.iterations + 1; let esqlErrors: string = ''; - const sanitizedQuery = query ? removePlaceHolders(query) : ''; - if (!isEmpty(sanitizedQuery)) { - const { errors, isEsqlQueryAggregating, hasMetadataOperator } = - parseEsqlQuery(sanitizedQuery); - if (!isEmpty(errors)) { - esqlErrors = JSON.stringify(errors); - } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { - esqlErrors = `Queries that do't use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`; + try { + const sanitizedQuery = query ? removePlaceHolders(query) : ''; + if (!isEmpty(sanitizedQuery)) { + const { errors, isEsqlQueryAggregating, hasMetadataOperator } = + parseEsqlQuery(sanitizedQuery); + if (!isEmpty(errors)) { + esqlErrors = JSON.stringify(errors); + } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { + esqlErrors = `Queries that do't use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`; + } } + if (esqlErrors) { + logger.debug(`ESQL query validation failed: ${esqlErrors}`); + } + } catch (error) { + esqlErrors = error.message; + logger.info(`Error parsing ESQL query: ${error.message}`); } - if (esqlErrors) { - logger.debug(`ESQL query validation failed: ${esqlErrors}`); - } - return { validation_errors: { iterations: currentIteration, esql_errors: esqlErrors } }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts index 16d668a698488..9282163ab0002 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts @@ -13,7 +13,7 @@ import type { OriginalRule, RuleMigration, } from '../../../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { Integration } from '../../../../types'; +import type { RuleMigrationIntegration } from '../../../../types'; import type { TranslateRuleValidationErrors } from './types'; export const translateRuleState = Annotation.Root({ @@ -22,9 +22,9 @@ export const translateRuleState = Annotation.Root({ default: () => [], }), original_rule: Annotation(), - integration: Annotation({ + integration: Annotation({ reducer: (current, value) => value ?? current, - default: () => ({} as Integration), + default: () => ({} as RuleMigrationIntegration), }), translation_finalized: Annotation({ reducer: (current, value) => value ?? current, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts index 7913e2c438081..725a949528cdc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts @@ -5,19 +5,18 @@ * 2.0. */ -import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; -import type { Integration } from '../../types'; +import type { RuleMigrationIntegration } from '../../types'; +import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever'; export class IntegrationRetriever { - constructor(private readonly dataClient: RuleMigrationsDataClient) {} + constructor(private readonly clients: RuleMigrationsRetrieverClients) {} - public async getIntegrations(semanticString: string): Promise { - return this.integrationRetriever(semanticString); + public async index() { + // TODO: use Fleet API client for integration retrieval as an argument once feature is available + return this.clients.data.integrations.create(); } - private integrationRetriever = async (semanticString: string): Promise => { - const integrations = await this.dataClient.integrations.retrieveIntegrations(semanticString); - - return integrations; - }; + public async getIntegrations(semanticString: string): Promise { + return this.clients.data.integrations.retrieveIntegrations(semanticString); + } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts index 0350bb43fe5ab..3a06526102f2d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts @@ -5,25 +5,28 @@ * 2.0. */ -import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; import type { RuleMigrationPrebuiltRule } from '../../types'; +import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever'; export class PrebuiltRulesRetriever { - constructor(private readonly dataClient: RuleMigrationsDataClient) {} + constructor(private readonly clients: RuleMigrationsRetrieverClients) {} + + // TODO: + // 1. Implement the `initialize` method to retrieve prebuilt rules and keep them in memory. + // 2. Improve the `retrieveRules` method to return the real prebuilt rules instead of the ELSER index doc. + + // Populates the indices used for RAG searches on prebuilt rules and integrations. + public async index() { + return this.clients.data.prebuiltRules.create({ + rulesClient: this.clients.rules, + soClient: this.clients.savedObjects, + }); + } public async getRules( semanticString: string, techniqueIds: string ): Promise { - return this.prebuiltRulesRetriever(semanticString, techniqueIds); + return this.clients.data.prebuiltRules.retrieveRules(semanticString, techniqueIds); } - - private prebuiltRulesRetriever = async ( - semanticString: string, - techniqueIds: string - ): Promise => { - const rules = await this.dataClient.prebuiltRules.retrieveRules(semanticString, techniqueIds); - - return rules; - }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 29852558cda48..07bc6d05d574b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -12,7 +12,7 @@ import { IntegrationRetriever } from './integration_retriever'; import { PrebuiltRulesRetriever } from './prebuilt_rules_retriever'; import { RuleResourceRetriever } from './rule_resource_retriever'; -interface RuleMigrationsRetrieverDeps { +export interface RuleMigrationsRetrieverClients { data: RuleMigrationsDataClient; rules: RulesClient; savedObjects: SavedObjectsClientContract; @@ -23,22 +23,17 @@ export class RuleMigrationsRetriever { public readonly integrations: IntegrationRetriever; public readonly prebuiltRules: PrebuiltRulesRetriever; - constructor(migrationId: string, private readonly clients: RuleMigrationsRetrieverDeps) { - this.resources = new RuleResourceRetriever(migrationId, this.clients.data); - this.integrations = new IntegrationRetriever(this.clients.data); - this.prebuiltRules = new PrebuiltRulesRetriever(this.clients.data); + constructor(migrationId: string, clients: RuleMigrationsRetrieverClients) { + this.resources = new RuleResourceRetriever(migrationId, clients.data); + this.integrations = new IntegrationRetriever(clients); + this.prebuiltRules = new PrebuiltRulesRetriever(clients); } public async initialize() { await Promise.all([ this.resources.initialize(), - // Populates the indices used for RAG searches on prebuilt rules and integrations. - this.clients.data.prebuiltRules.create({ - rulesClient: this.clients.rules, - soClient: this.clients.savedObjects, - }), - // Will use Fleet API client for integration retrieval as an argument once feature is available - this.clients.data.integrations.create(), + this.prebuiltRules.index(), + this.integrations.index(), ]).catch((error) => { throw new Error(`Failed to initialize RuleMigrationsRetriever: ${error}`); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index beb614b0ec748..3fc825e57f78a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -215,6 +215,7 @@ export class RuleMigrationsTaskClient { await this.data.rules.updateStatus(migrationId, undefined, SiemMigrationStatus.PENDING, { refresh: true, }); + // await this.data.rules.updateRetry(migrationId); return { updated: true }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts new file mode 100644 index 0000000000000..4dad4235ad4e0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const cleanMarkdown = (markdown: string): string => { + // Use languages known by the code block plugin + return markdown.replaceAll('```esql', '```sql').replaceAll('```spl', '```splunk-spl'); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index f13a407ee2500..d1df32ebbcb61 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -15,9 +15,9 @@ export type Stored = T & { id: string }; export type StoredRuleMigration = Stored; export type StoredRuleMigrationResource = Stored; -export interface Integration { - title: string; +export interface RuleMigrationIntegration { id: string; + title: string; description: string; data_streams: Array<{ dataset: string; title: string; index_pattern: string }>; elser_embedding: string; @@ -25,7 +25,7 @@ export interface Integration { export interface RuleMigrationPrebuiltRule { rule_id: string; - installedRuleId?: string; + installed_rule_id?: string; name: string; description: string; elser_embedding: string; From ef8707525156f5280205acfdbc5f8c1466e5e108 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 2 Jan 2025 12:28:14 +0100 Subject: [PATCH 9/9] fix test --- .../rules/task/retrievers/integration_retriever.test.ts | 5 ++++- .../rules/task/retrievers/integration_retriever.ts | 2 +- .../rules/task/retrievers/prebuilt_rules_retriever.ts | 3 +-- .../rules/task/retrievers/rule_migrations_retriever.ts | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts index 2aa01a9c9c41d..2adb942e0c419 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.test.ts @@ -7,6 +7,7 @@ import { MockRuleMigrationsDataClient } from '../../data/__mocks__/mocks'; import { IntegrationRetriever } from './integration_retriever'; +import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever'; describe('IntegrationRetriever', () => { let integrationRetriever: IntegrationRetriever; @@ -19,7 +20,9 @@ describe('IntegrationRetriever', () => { elser_embedding: 'elser_embedding', }; beforeEach(() => { - integrationRetriever = new IntegrationRetriever(mockRuleMigrationsDataClient); + integrationRetriever = new IntegrationRetriever({ + data: mockRuleMigrationsDataClient, + } as RuleMigrationsRetrieverClients); mockRuleMigrationsDataClient.integrations.retrieveIntegrations.mockImplementation( async (_: string) => { return mockIntegrationItem; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts index 725a949528cdc..18ddb51a5b83b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/integration_retriever.ts @@ -11,7 +11,7 @@ import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever export class IntegrationRetriever { constructor(private readonly clients: RuleMigrationsRetrieverClients) {} - public async index() { + public async populateIndex() { // TODO: use Fleet API client for integration retrieval as an argument once feature is available return this.clients.data.integrations.create(); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts index 3a06526102f2d..03fd24d4e6b4d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts @@ -15,8 +15,7 @@ export class PrebuiltRulesRetriever { // 1. Implement the `initialize` method to retrieve prebuilt rules and keep them in memory. // 2. Improve the `retrieveRules` method to return the real prebuilt rules instead of the ELSER index doc. - // Populates the indices used for RAG searches on prebuilt rules and integrations. - public async index() { + public async populateIndex() { return this.clients.data.prebuiltRules.create({ rulesClient: this.clients.rules, soClient: this.clients.savedObjects, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 07bc6d05d574b..5616bfd4fb26b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -32,8 +32,8 @@ export class RuleMigrationsRetriever { public async initialize() { await Promise.all([ this.resources.initialize(), - this.prebuiltRules.index(), - this.integrations.index(), + this.prebuiltRules.populateIndex(), + this.integrations.populateIndex(), ]).catch((error) => { throw new Error(`Failed to initialize RuleMigrationsRetriever: ${error}`); });