From 6f38076236cea90470a8c174a04993c3b90c6959 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 6 Dec 2024 18:57:11 +0000 Subject: [PATCH 01/28] [Security Solution][Detection Engine] adds preview logged requests for the rest of rules --- .../components/rule_preview/index.test.tsx | 3 +- .../components/rule_preview/index.tsx | 2 +- .../threshold/find_threshold_signals.ts | 26 ++++++++++++-- .../rule_types/threshold/threshold.ts | 34 +++++++++++-------- .../rule_types/utils/logged_requests/index.ts | 1 + .../logged_requests/log_search_request.ts | 27 +++++++++++++++ .../rule_types/utils/single_search_after.ts | 17 ++++++++++ .../trial_license_complete_tier/threshold.ts | 33 ++++++++++++++++++ 8 files changed, 124 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx index 25d5b90d5408a..1618e301a0670 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx @@ -41,14 +41,13 @@ jest.mock('../../../../common/hooks/use_experimental_features', () => ({ // rule types that do not support logged requests const doNotSupportLoggedRequests: Type[] = [ - 'threshold', 'threat_match', 'machine_learning', 'query', 'new_terms', ]; -const supportLoggedRequests: Type[] = ['esql', 'eql']; +const supportLoggedRequests: Type[] = ['esql', 'eql', 'threshold']; const getMockIndexPattern = (): DataViewBase => ({ fields, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx index 7d51009dccda3..e841dfd92f496 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx @@ -43,7 +43,7 @@ import { usePreviewInvocationCount } from './use_preview_invocation_count'; export const REASONABLE_INVOCATION_COUNT = 200; -const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = ['esql', 'eql']; +const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = ['esql', 'eql', 'threshold']; const timeRanges = [ { start: 'now/d', end: 'now', label: 'Today' }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts index ce9bec5f32249..f1895a680b766 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts @@ -32,6 +32,7 @@ import type { import { shouldFilterByCardinality, searchResultHasAggs } from './utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { getMaxSignalsWarning } from '../utils/utils'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; interface FindThresholdSignalsParams { from: string; @@ -46,6 +47,7 @@ interface FindThresholdSignalsParams { primaryTimestamp: TimestampOverride; secondaryTimestamp: TimestampOverride | undefined; aggregatableTimestampField: string; + isLoggedRequestsEnabled?: boolean; } const hasThresholdFields = (threshold: ThresholdNormalized) => !!threshold.field.length; @@ -55,6 +57,7 @@ interface SearchAfterResults { searchErrors: string[]; } +// eslint-disable-next-line complexity export const findThresholdSignals = async ({ from, to, @@ -68,11 +71,13 @@ export const findThresholdSignals = async ({ primaryTimestamp, secondaryTimestamp, aggregatableTimestampField, + isLoggedRequestsEnabled, }: FindThresholdSignalsParams): Promise<{ buckets: ThresholdBucket[]; searchDurations: string[]; searchErrors: string[]; warnings: string[]; + loggedRequests?: RulePreviewLoggedRequest[]; }> => { // Leaf aggregations used below const buckets: ThresholdBucket[] = []; @@ -81,13 +86,19 @@ export const findThresholdSignals = async ({ searchErrors: [], }; const warnings: string[] = []; + const loggedRequests: RulePreviewLoggedRequest[] = []; const includeCardinalityFilter = shouldFilterByCardinality(threshold); if (hasThresholdFields(threshold)) { let sortKeys: Record | undefined; do { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: thresholdLoggedRequests, + } = await singleSearchAfter({ aggregations: buildThresholdMultiBucketAggregation({ threshold, aggregatableTimestampField, @@ -105,9 +116,12 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, + isLoggedRequestsEnabled, }); searchAfterResults.searchDurations.push(searchDuration); + loggedRequests.push(...(thresholdLoggedRequests ?? [])); + if (!isEmpty(searchErrors)) { searchAfterResults.searchErrors.push(...searchErrors); sortKeys = undefined; // this will eject us out of the loop @@ -125,7 +139,12 @@ export const findThresholdSignals = async ({ } } while (sortKeys && buckets.length <= maxSignals); } else { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: thresholdLoggedRequests, + } = await singleSearchAfter({ aggregations: buildThresholdSingleBucketAggregation({ threshold, aggregatableTimestampField, @@ -143,10 +162,12 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, + isLoggedRequestsEnabled, }); searchAfterResults.searchDurations.push(searchDuration); searchAfterResults.searchErrors.push(...searchErrors); + loggedRequests.push(...(thresholdLoggedRequests ?? [])); if ( !searchResultHasAggs(searchResult) && @@ -182,5 +203,6 @@ export const findThresholdSignals = async ({ buckets: buckets.slice(0, maxSignals), ...searchAfterResults, warnings, + ...(isLoggedRequestsEnabled ? { loggedRequests } : {}), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts index d56e164438509..318f5e0d49bca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts @@ -46,6 +46,7 @@ import { buildThresholdSignalHistory } from './build_signal_history'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { getSignalHistory, transformBulkCreatedItemsToHits } from './utils'; import type { ExperimentalFeatures } from '../../../../../common'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; export const thresholdExecutor = async ({ inputIndex, @@ -96,6 +97,8 @@ export const thresholdExecutor = async ({ }): Promise => { const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; + const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled); + // const loggedRequests: RulePreviewLoggedRequest[] = []; return withSecuritySpan('thresholdExecutor', async () => { const exceptionsWarning = getUnprocessedExceptionsWarnings(unprocessedExceptions); @@ -140,20 +143,22 @@ export const thresholdExecutor = async ({ }); // Look for new events over threshold - const { buckets, searchErrors, searchDurations, warnings } = await findThresholdSignals({ - inputIndexPattern: inputIndex, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - maxSignals: tuple.maxSignals, - services, - ruleExecutionLogger, - filter: esFilter, - threshold: ruleParams.threshold, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, - aggregatableTimestampField, - }); + const { buckets, searchErrors, searchDurations, warnings, loggedRequests } = + await findThresholdSignals({ + inputIndexPattern: inputIndex, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + maxSignals: tuple.maxSignals, + services, + ruleExecutionLogger, + filter: esFilter, + threshold: ruleParams.threshold, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + aggregatableTimestampField, + isLoggedRequestsEnabled, + }); const alertSuppression = completeRule.ruleParams.alertSuppression; @@ -227,6 +232,7 @@ export const thresholdExecutor = async ({ ...newSignalHistory, }, }, + ...(isLoggedRequestsEnabled ? { loggedRequests } : {}), }; }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/index.ts index 7cea6e0d75fa6..ce74795bd6bf1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/index.ts @@ -8,3 +8,4 @@ export * from './log_esql'; export * from './log_eql'; export * from './log_query'; +export * from './log_search_request'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts new file mode 100644 index 0000000000000..70a7b76c75bec --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export const logSearchRequest = (searchRequest: estypes.SearchRequest): string => { + // handle deprecated body property which still used widely across Detection Engine + if (searchRequest.body) { + const { body, runtime_mappings: _, index, ...params } = searchRequest; + const urlParams = Object.entries(params) + .reduce((acc, [key, value]) => { + if (value) { + acc.push(`${key}=${value}`); + } + + return acc; + }, []) + .join('&'); + return `POST /${index}/_search?${urlParams}\n${JSON.stringify(searchRequest.body, null, 2)}`; + } + + return '???'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index c10403848474f..55761dbafdb74 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -17,6 +17,8 @@ import { createErrorsFromShard, makeFloatString } from './utils'; import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; +import { logSearchRequest } from './logged_requests'; export interface SingleSearchAfterParams { aggregations?: Record; @@ -35,6 +37,7 @@ export interface SingleSearchAfterParams { runtimeMappings: estypes.MappingRuntimeFields | undefined; additionalFilters?: estypes.QueryDslQueryContainer[]; overrideBody?: OverrideBodyQuery; + isLoggedRequestsEnabled?: boolean; } // utilize search_after for paging results into bulk. @@ -57,12 +60,16 @@ export const singleSearchAfter = async < trackTotalHits, additionalFilters, overrideBody, + isLoggedRequestsEnabled, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; searchErrors: string[]; + loggedRequests?: RulePreviewLoggedRequest[]; }> => { return withSecuritySpan('singleSearchAfter', async () => { + const loggedRequests: RulePreviewLoggedRequest[] = []; + try { const searchAfterQuery = buildEventsSearchQuery({ aggregations, @@ -98,10 +105,19 @@ export const singleSearchAfter = async < errors: nextSearchAfterResult._shards.failures ?? [], }); + if (isLoggedRequestsEnabled) { + loggedRequests.push({ + request: logSearchRequest(searchAfterQuery as estypes.SearchRequest), + description: 'Search documents', + duration: end - start, + }); + } + return { searchResult: nextSearchAfterResult, searchDuration: makeFloatString(end - start), searchErrors, + loggedRequests, }; } catch (exc) { ruleExecutionLogger.error(`Searching events operation failed: ${exc}`); @@ -129,6 +145,7 @@ export const singleSearchAfter = async < searchResult: searchRes, searchDuration: '-1.0', searchErrors: exc.message, + loggedRequests, }; } diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts index 2f7086664fbcb..4f33f08821902 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts @@ -688,5 +688,38 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe.only('preview logged requests', () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['auditbeat-*']), + threshold: { + field: 'host.id', + value: 1, // This value generates 7 alerts with the current esArchive + }, + max_signals: 5, + }; + + it('should not return requests property when not enabled', async () => { + const { logs } = await previewRule({ + supertest, + rule, + }); + + expect(logs[0].requests).toEqual(undefined); + }); + it('should return requests property when enable_logged_requests set to true', async () => { + const { logs } = await previewRule({ + supertest, + rule, + enableLoggedRequests: true, + }); + + const requests = logs[0].requests; + + expect(requests).toHaveLength(1); + expect(requests![0].description).toBe('Threshold request to find all matches'); + expect(requests![0].request).toContain('POST /auditbeat-*/_search?allow_no_indices=true'); + }); + }); }); }; From c7eb832bfa47925ceea6258c894de94a13e9a813 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:03:03 +0000 Subject: [PATCH 02/28] [Security Solution][Detection Engine] refactoring and type fixing --- .../threat_mapping/get_event_count.ts | 2 +- .../threshold/find_threshold_signals.ts | 12 +++-- .../rule_types/threshold/threshold.ts | 1 - .../rule_types/threshold/utils.ts | 14 +++++ .../rule_types/translations.ts | 16 ++++++ .../lib/detection_engine/rule_types/types.ts | 2 +- .../utils/build_events_query.test.ts | 52 ++++++++++--------- .../rule_types/utils/build_events_query.ts | 9 ++-- .../logged_requests/log_search_request.ts | 11 ++-- .../rule_types/utils/single_search_after.ts | 14 ++--- 10 files changed, 87 insertions(+), 46 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts index c74424f65d514..0f631542679bd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts @@ -100,7 +100,7 @@ export const getEventCount = async ({ secondaryTimestamp, searchAfterSortIds: undefined, runtimeMappings: undefined, - }).body.query; + }).body?.query; const response = await esClient.count({ body: { query: eventSearchQueryBodyQuery }, ignore_unavailable: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts index f1895a680b766..6beced85dd28f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts @@ -29,10 +29,11 @@ import type { ThresholdBucket, ThresholdSingleBucketAggregationResult, } from './types'; -import { shouldFilterByCardinality, searchResultHasAggs } from './utils'; +import { shouldFilterByCardinality, searchResultHasAggs, stringifyAfterKey } from './utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { getMaxSignalsWarning } from '../utils/utils'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; +import * as i18n from '../translations'; interface FindThresholdSignalsParams { from: string; @@ -116,7 +117,9 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, - isLoggedRequestsEnabled, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(stringifyAfterKey(sortKeys)) + : undefined, }); searchAfterResults.searchDurations.push(searchDuration); @@ -130,7 +133,6 @@ export const findThresholdSignals = async ({ } else if (searchResultHasAggs(searchResult)) { const thresholdTerms = searchResult.aggregations?.thresholdTerms; sortKeys = thresholdTerms?.after_key; - buckets.push( ...((searchResult.aggregations?.thresholdTerms.buckets as ThresholdBucket[]) ?? []) ); @@ -162,7 +164,9 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, - isLoggedRequestsEnabled, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION() + : undefined, }); searchAfterResults.searchDurations.push(searchDuration); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts index 318f5e0d49bca..96fbff088e7a5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts @@ -46,7 +46,6 @@ import { buildThresholdSignalHistory } from './build_signal_history'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { getSignalHistory, transformBulkCreatedItemsToHits } from './utils'; import type { ExperimentalFeatures } from '../../../../../common'; -import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; export const thresholdExecutor = async ({ inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts index c73becbdd8f57..2e5e0faa29eda 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts @@ -116,3 +116,17 @@ export const transformBulkCreatedItemsToHits = ( }, }; }); + +/** + * converts ES after_key object into string + * for example: { "agent.name": "test" } would become `agent.name: test` + */ +export const stringifyAfterKey = (afterKey: Record | undefined) => { + if (!afterKey) { + return; + } + + return Object.entries(afterKey) + .map((entry) => entry.join(': ')) + .join(', '); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index 88445b0d41dc3..2bc35c13c8591 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -27,3 +27,19 @@ export const EQL_SEARCH_REQUEST_DESCRIPTION = i18n.translate( defaultMessage: 'EQL request to find all matches', } ); + +export const FIND_THRESHOLD_BUCKETS_DESCRIPTION = (afterBucket?: string) => + afterBucket + ? i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlRuleType.findThresholdRuleBucketsDescription', + { + defaultMessage: 'Find all terms that exceeds threshold value after {afterBucket}', + values: { afterBucket }, + } + ) + : i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlRuleType.findThresholdRuleBucketsDescription', + { + defaultMessage: 'Find all terms that exceeds threshold value', + } + ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 94aa55e6b0dd5..e4ba927500184 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -410,5 +410,5 @@ export interface SearchAfterAndBulkCreateReturnType { // the new fields can be added later if needed export interface OverrideBodyQuery { _source?: estypes.SearchSourceConfig; - fields?: estypes.Fields; + fields?: Array; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts index 03ee439c2c969..074931be1988d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.test.ts @@ -23,9 +23,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -79,9 +79,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -176,9 +176,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -233,9 +233,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -290,9 +290,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -346,9 +346,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -409,9 +409,9 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ @@ -470,7 +470,7 @@ describe('create_signals', () => { trackTotalHits: false, runtimeMappings: undefined, }); - expect(query.track_total_hits).toEqual(false); + expect(query.body?.track_total_hits).toEqual(false); }); test('if sortOrder is provided it should be included', () => { @@ -487,12 +487,14 @@ describe('create_signals', () => { trackTotalHits: false, runtimeMappings: undefined, }); - expect(query.body.sort[0]).toEqual({ - '@timestamp': { - order: 'desc', - unmapped_type: 'date', + expect(query?.body?.sort).toEqual([ + { + '@timestamp': { + order: 'desc', + unmapped_type: 'date', + }, }, - }); + ]); }); test('it respects sort order for timestampOverride', () => { @@ -508,18 +510,20 @@ describe('create_signals', () => { sortOrder: 'desc', runtimeMappings: undefined, }); - expect(query.body.sort[0]).toEqual({ - 'event.ingested': { - order: 'desc', - unmapped_type: 'date', + expect(query?.body?.sort).toEqual([ + { + 'event.ingested': { + order: 'desc', + unmapped_type: 'date', + }, }, - }); - expect(query.body.sort[1]).toEqual({ - '@timestamp': { - order: 'desc', - unmapped_type: 'date', + { + '@timestamp': { + order: 'desc', + unmapped_type: 'date', + }, }, - }); + ]); }); test('it respects overriderBody params', () => { @@ -541,11 +545,11 @@ describe('create_signals', () => { expect(query).toEqual({ allow_no_indices: true, index: ['auditbeat-*'], - size: 100, runtime_mappings: undefined, track_total_hits: undefined, ignore_unavailable: true, body: { + size: 100, query: { bool: { filter: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts index c8998a83e11da..67fb75eef50c4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/build_events_query.ts @@ -114,7 +114,7 @@ export const buildEventsSearchQuery = ({ trackTotalHits, additionalFilters, overrideBody, -}: BuildEventsSearchQuery) => { +}: BuildEventsSearchQuery): estypes.SearchRequest => { const timestamps = secondaryTimestamp ? [primaryTimestamp, secondaryTimestamp] : [primaryTimestamp]; @@ -152,14 +152,13 @@ export const buildEventsSearchQuery = ({ }); } - const searchQuery = { + const searchQuery: estypes.SearchRequest = { allow_no_indices: true, - runtime_mappings: runtimeMappings, index, - size, ignore_unavailable: true, - track_total_hits: trackTotalHits, body: { + track_total_hits: trackTotalHits, + size, query: { bool: { filter: filterWithTime, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts index 70a7b76c75bec..671477b3b122b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts @@ -10,18 +10,23 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export const logSearchRequest = (searchRequest: estypes.SearchRequest): string => { // handle deprecated body property which still used widely across Detection Engine if (searchRequest.body) { - const { body, runtime_mappings: _, index, ...params } = searchRequest; + const { body, index, ...params } = searchRequest; const urlParams = Object.entries(params) .reduce((acc, [key, value]) => { - if (value) { + if (value != null) { acc.push(`${key}=${value}`); } return acc; }, []) .join('&'); - return `POST /${index}/_search?${urlParams}\n${JSON.stringify(searchRequest.body, null, 2)}`; + return `POST /${index}/_search?${urlParams}\n${JSON.stringify( + { ...searchRequest.body }, + null, + 2 + )}`; } + // TODO: fix me return '???'; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index 55761dbafdb74..90c63f124a686 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -37,7 +37,7 @@ export interface SingleSearchAfterParams { runtimeMappings: estypes.MappingRuntimeFields | undefined; additionalFilters?: estypes.QueryDslQueryContainer[]; overrideBody?: OverrideBodyQuery; - isLoggedRequestsEnabled?: boolean; + loggedRequestDescription?: string; } // utilize search_after for paging results into bulk. @@ -60,7 +60,7 @@ export const singleSearchAfter = async < trackTotalHits, additionalFilters, overrideBody, - isLoggedRequestsEnabled, + loggedRequestDescription, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; @@ -95,7 +95,7 @@ export const singleSearchAfter = async < const start = performance.now(); const { body: nextSearchAfterResult } = await services.scopedClusterClient.asCurrentUser.search( - searchAfterQuery as estypes.SearchRequest, + searchAfterQuery, { meta: true } ); @@ -105,11 +105,11 @@ export const singleSearchAfter = async < errors: nextSearchAfterResult._shards.failures ?? [], }); - if (isLoggedRequestsEnabled) { + if (loggedRequestDescription) { loggedRequests.push({ - request: logSearchRequest(searchAfterQuery as estypes.SearchRequest), - description: 'Search documents', - duration: end - start, + request: logSearchRequest(searchAfterQuery), + description: loggedRequestDescription, + duration: Math.round(end - start), }); } From 328987e5f5b76c6583ceda0b059abeefbbb07218 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:08:34 +0000 Subject: [PATCH 03/28] [Security Solution][Detection Engine] fix FTR test --- .../trial_license_complete_tier/threshold.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts index 4f33f08821902..0f37077da10af 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts @@ -689,14 +689,13 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe.only('preview logged requests', () => { + describe('preview logged requests', () => { const rule: ThresholdRuleCreateProps = { ...getThresholdRuleForAlertTesting(['auditbeat-*']), threshold: { field: 'host.id', - value: 1, // This value generates 7 alerts with the current esArchive + value: 100, }, - max_signals: 5, }; it('should not return requests property when not enabled', async () => { @@ -716,9 +715,12 @@ export default ({ getService }: FtrProviderContext) => { const requests = logs[0].requests; - expect(requests).toHaveLength(1); - expect(requests![0].description).toBe('Threshold request to find all matches'); + expect(requests).toHaveLength(2); + expect(requests![0].description).toBe('Find all terms that exceeds threshold value'); expect(requests![0].request).toContain('POST /auditbeat-*/_search?allow_no_indices=true'); + expect(requests![1].description).toBe( + 'Find all terms that exceeds threshold value after host.id: f9c7ca2d33f548a8b37667f6fffc59ce' + ); }); }); }); From 3eabaeddb903ca431c31ed0fc54feb44cafb9470 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:32:01 +0000 Subject: [PATCH 04/28] [Security Solution][Detection Engine] add unit tests --- .../rule_types/threshold/utils.test.ts | 14 ++ .../log_search_request.test.ts | 181 ++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts index 2a54c3b0e156f..8b1e0dc9d6377 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts @@ -13,9 +13,23 @@ import { getSignalHistory, getThresholdTermsHash, transformBulkCreatedItemsToHits, + stringifyAfterKey, } from './utils'; describe('threshold utils', () => { + describe('stringifyAfterKey', () => { + it('should stringify after_key object with single key value', () => { + expect(stringifyAfterKey({ 'agent.name': 'test' })).toBe('agent.name: test'); + }); + it('should stringify after_key object with multiple key values', () => { + expect(stringifyAfterKey({ 'agent.name': 'test', 'destination.ip': '127.0.0.1' })).toBe( + 'agent.name: test, destination.ip: 127.0.0.1' + ); + }); + it('should return undefined if after_key is undefined', () => { + expect(stringifyAfterKey(undefined)).toBeUndefined(); + }); + }); describe('calcualteThresholdSignalUuid', () => { it('should generate a uuid without key', () => { const startedAt = new Date('2020-12-17T16:27:00Z'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.test.ts new file mode 100644 index 0000000000000..9321b1f5bab7c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.test.ts @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { logSearchRequest } from './log_search_request'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +describe('logSearchRequest', () => { + it('should match inline snapshot when deprecated search request used', () => { + const searchRequest = { + allow_no_indices: true, + index: ['close_alerts*'], + ignore_unavailable: true, + body: { + size: 0, + query: { + bool: { + filter: [ + { + bool: { + must: [], + filter: [{ query_string: { query: '*' } }, { bool: { must_not: [] } }], + should: [], + must_not: [], + }, + }, + { + range: { + '@timestamp': { + lte: '2024-12-09T17:26:48.786Z', + gte: '2013-07-14T00:26:48.786Z', + format: 'strict_date_optional_time', + }, + }, + }, + ], + }, + }, + fields: [ + { field: '*', include_unmapped: true }, + { field: '@timestamp', format: 'strict_date_optional_time' }, + ], + aggregations: { + thresholdTerms: { + composite: { + sources: [ + { 'agent.name': { terms: { field: 'agent.name' } } }, + { 'destination.ip': { terms: { field: 'destination.ip' } } }, + ], + after: { 'agent.name': 'test-6', 'destination.ip': '127.0.0.1' }, + size: 10000, + }, + aggs: { + max_timestamp: { max: { field: '@timestamp' } }, + min_timestamp: { min: { field: '@timestamp' } }, + count_check: { + bucket_selector: { + buckets_path: { docCount: '_count' }, + script: 'params.docCount >= 1', + }, + }, + }, + }, + }, + runtime_mappings: {}, + sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'date' } }], + }, + }; + + expect(logSearchRequest(searchRequest as unknown as estypes.SearchRequest)) + .toMatchInlineSnapshot(` + "POST /close_alerts*/_search?allow_no_indices=true&ignore_unavailable=true + { + \\"size\\": 0, + \\"query\\": { + \\"bool\\": { + \\"filter\\": [ + { + \\"bool\\": { + \\"must\\": [], + \\"filter\\": [ + { + \\"query_string\\": { + \\"query\\": \\"*\\" + } + }, + { + \\"bool\\": { + \\"must_not\\": [] + } + } + ], + \\"should\\": [], + \\"must_not\\": [] + } + }, + { + \\"range\\": { + \\"@timestamp\\": { + \\"lte\\": \\"2024-12-09T17:26:48.786Z\\", + \\"gte\\": \\"2013-07-14T00:26:48.786Z\\", + \\"format\\": \\"strict_date_optional_time\\" + } + } + } + ] + } + }, + \\"fields\\": [ + { + \\"field\\": \\"*\\", + \\"include_unmapped\\": true + }, + { + \\"field\\": \\"@timestamp\\", + \\"format\\": \\"strict_date_optional_time\\" + } + ], + \\"aggregations\\": { + \\"thresholdTerms\\": { + \\"composite\\": { + \\"sources\\": [ + { + \\"agent.name\\": { + \\"terms\\": { + \\"field\\": \\"agent.name\\" + } + } + }, + { + \\"destination.ip\\": { + \\"terms\\": { + \\"field\\": \\"destination.ip\\" + } + } + } + ], + \\"after\\": { + \\"agent.name\\": \\"test-6\\", + \\"destination.ip\\": \\"127.0.0.1\\" + }, + \\"size\\": 10000 + }, + \\"aggs\\": { + \\"max_timestamp\\": { + \\"max\\": { + \\"field\\": \\"@timestamp\\" + } + }, + \\"min_timestamp\\": { + \\"min\\": { + \\"field\\": \\"@timestamp\\" + } + }, + \\"count_check\\": { + \\"bucket_selector\\": { + \\"buckets_path\\": { + \\"docCount\\": \\"_count\\" + }, + \\"script\\": \\"params.docCount >= 1\\" + } + } + } + } + }, + \\"runtime_mappings\\": {}, + \\"sort\\": [ + { + \\"@timestamp\\": { + \\"order\\": \\"desc\\", + \\"unmapped_type\\": \\"date\\" + } + } + ] + }" + `); + }); +}); From 2f6651fda16e5425c9822a00c640a97e93cf485b Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:34:17 +0000 Subject: [PATCH 05/28] [Security Solution][Detection Engine] fix i18n --- .../server/lib/detection_engine/rule_types/translations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index 2bc35c13c8591..1819c63d15e28 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -31,7 +31,7 @@ export const EQL_SEARCH_REQUEST_DESCRIPTION = i18n.translate( export const FIND_THRESHOLD_BUCKETS_DESCRIPTION = (afterBucket?: string) => afterBucket ? i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlRuleType.findThresholdRuleBucketsDescription', + 'xpack.securitySolution.detectionEngine.esqlRuleType.findThresholdRuleBucketsAfterDescription', { defaultMessage: 'Find all terms that exceeds threshold value after {afterBucket}', values: { afterBucket }, From 97c506b4a33abc63eaa8ebe810b453c1171c0dfc Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:22:51 +0000 Subject: [PATCH 06/28] [Security Solution][Detection Engine] add ML rule type --- .../components/rule_preview/index.test.tsx | 11 +--- .../components/rule_preview/index.tsx | 7 ++- .../rule_types/ml/create_ml_alert_type.ts | 13 +++- .../rule_types/ml/find_ml_signals.ts | 25 +++++++- .../lib/detection_engine/rule_types/ml/ml.ts | 28 ++++++--- .../rule_types/threshold/threshold.ts | 1 - .../rule_types/translations.ts | 7 +++ .../logged_requests/log_search_request.ts | 2 +- .../server/lib/machine_learning/index.ts | 60 ++++++++++--------- .../machine_learning.ts | 24 ++++++++ 10 files changed, 123 insertions(+), 55 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx index 1618e301a0670..3a19fdf48a633 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx @@ -40,14 +40,9 @@ jest.mock('../../../../common/hooks/use_experimental_features', () => ({ })); // rule types that do not support logged requests -const doNotSupportLoggedRequests: Type[] = [ - 'threat_match', - 'machine_learning', - 'query', - 'new_terms', -]; - -const supportLoggedRequests: Type[] = ['esql', 'eql', 'threshold']; +const doNotSupportLoggedRequests: Type[] = ['threat_match', 'query', 'new_terms']; + +const supportLoggedRequests: Type[] = ['esql', 'eql', 'threshold', 'machine_learning']; const getMockIndexPattern = (): DataViewBase => ({ fields, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx index e841dfd92f496..4c947ec91cd1f 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx @@ -43,7 +43,12 @@ import { usePreviewInvocationCount } from './use_preview_invocation_count'; export const REASONABLE_INVOCATION_COUNT = 200; -const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = ['esql', 'eql', 'threshold']; +const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = [ + 'esql', + 'eql', + 'threshold', + 'machine_learning', +]; const timeRanges = [ { start: 'now/d', end: 'now', label: 'Today' }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts index c5d918becefdf..84e8652eaf654 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts @@ -18,7 +18,12 @@ import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; export const createMlAlertType = ( createOptions: CreateRuleOptions -): SecurityAlertType => { +): SecurityAlertType< + MachineLearningRuleParams, + { isLoggedRequestsEnabled?: boolean }, + {}, + 'default' +> => { const { experimentalFeatures, ml, licensing, scheduleNotificationResponseActionsService } = createOptions; return { @@ -76,6 +81,7 @@ export const createMlAlertType = ( alertSuppression: completeRule.ruleParams.alertSuppression, licensing, }); + const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled); const wrapSuppressedHits: WrapSuppressedHits = (events, buildReasonMessage) => wrapSuppressedAlerts({ @@ -93,7 +99,7 @@ export const createMlAlertType = ( intendedTimestamp, }); - const result = await mlExecutor({ + const { result, loggedRequests } = await mlExecutor({ completeRule, tuple, ml, @@ -110,8 +116,9 @@ export const createMlAlertType = ( isAlertSuppressionActive, experimentalFeatures, scheduleNotificationResponseActionsService, + isLoggedRequestsEnabled, }); - return { ...result, state }; + return { ...result, state, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) }; }, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts index 38a21961bda55..b5eea99e93071 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts @@ -10,7 +10,10 @@ import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server import type { MlPluginSetup } from '@kbn/ml-plugin/server'; import type { Filter } from '@kbn/es-query'; import type { AnomalyResults } from '../../../machine_learning'; -import { getAnomalies } from '../../../machine_learning'; +import { getAnomalies, buildAnomalyQuery } from '../../../machine_learning'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; +import { logSearchRequest } from '../utils/logged_requests'; +import * as i18n from '../translations'; export const findMlSignals = async ({ ml, @@ -22,6 +25,7 @@ export const findMlSignals = async ({ to, maxSignals, exceptionFilter, + isLoggedRequestsEnabled, }: { ml: MlPluginSetup; request: KibanaRequest; @@ -32,7 +36,10 @@ export const findMlSignals = async ({ to: string; maxSignals: number; exceptionFilter: Filter | undefined; -}): Promise => { + isLoggedRequestsEnabled: boolean; +}): Promise<{ anomalyResults: AnomalyResults; loggedRequests?: RulePreviewLoggedRequest[] }> => { + const loggedRequests: RulePreviewLoggedRequest[] = []; + const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient); const params = { jobIds, @@ -42,5 +49,17 @@ export const findMlSignals = async ({ maxRecords: maxSignals, exceptionFilter, }; - return getAnomalies(params, mlAnomalySearch); + + const anomalyResults = await getAnomalies(params, mlAnomalySearch); + + if (isLoggedRequestsEnabled) { + const searchQuery = buildAnomalyQuery(params); + searchQuery.index = '.ml-anomalies-*'; + loggedRequests.push({ + request: logSearchRequest(searchQuery), + description: i18n.ML_SEARCH_ANOMALIES_DESCRIPTION, + duration: anomalyResults.took, + }); + } + return { anomalyResults, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts index 1da14640c5a51..74a02534ca3dc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts @@ -17,6 +17,7 @@ import type { } from '@kbn/alerting-plugin/server'; import type { ListClient } from '@kbn/lists-plugin/server'; import type { Filter } from '@kbn/es-query'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { isJobStarted } from '../../../../../common/machine_learning/helpers'; import type { ExperimentalFeatures } from '../../../../../common/experimental_features'; import type { CompleteRule, MachineLearningRuleParams } from '../../rule_schema'; @@ -61,6 +62,7 @@ interface MachineLearningRuleExecutorParams { isAlertSuppressionActive: boolean; experimentalFeatures: ExperimentalFeatures; scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService']; + isLoggedRequestsEnabled: boolean; } export const mlExecutor = async ({ @@ -80,9 +82,11 @@ export const mlExecutor = async ({ alertWithSuppression, experimentalFeatures, scheduleNotificationResponseActionsService, + isLoggedRequestsEnabled, }: MachineLearningRuleExecutorParams) => { const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; + const loggedRequests: RulePreviewLoggedRequest[] = []; return withSecuritySpan('mlExecutor', async () => { if (ml == null) { @@ -122,7 +126,7 @@ export const mlExecutor = async ({ let anomalyResults: AnomalyResults; try { - anomalyResults = await findMlSignals({ + const searchResults = await findMlSignals({ ml, // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is // currently unused by the mlAnomalySearch function. @@ -134,14 +138,17 @@ export const mlExecutor = async ({ to: tuple.to.toISOString(), maxSignals: tuple.maxSignals, exceptionFilter, + isLoggedRequestsEnabled, }); + anomalyResults = searchResults.anomalyResults; + loggedRequests.push(...(searchResults.loggedRequests ?? [])); } catch (error) { if (typeof error.message === 'string' && (error.message as string).endsWith('missing')) { result.userError = true; } result.errors.push(error.message); result.success = false; - return result; + return { result }; } // TODO we add the max_signals warning _before_ filtering the anomalies against the exceptions list. Is that correct? @@ -204,12 +211,15 @@ export const mlExecutor = async ({ signalsCount: result.createdSignalsCount, responseActions: completeRule.ruleParams.responseActions, }); - return mergeReturns([ - result, - createSearchAfterReturnType({ - success: anomalyResults._shards.failed === 0, - errors: searchErrors, - }), - ]); + return { + result: mergeReturns([ + result, + createSearchAfterReturnType({ + success: anomalyResults._shards.failed === 0, + errors: searchErrors, + }), + ]), + ...(isLoggedRequestsEnabled ? { loggedRequests } : {}), + }; }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts index 96fbff088e7a5..f3640b0ea84da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts @@ -97,7 +97,6 @@ export const thresholdExecutor = async ({ const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled); - // const loggedRequests: RulePreviewLoggedRequest[] = []; return withSecuritySpan('thresholdExecutor', async () => { const exceptionsWarning = getUnprocessedExceptionsWarnings(unprocessedExceptions); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index 1819c63d15e28..ee44c85c01732 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -43,3 +43,10 @@ export const FIND_THRESHOLD_BUCKETS_DESCRIPTION = (afterBucket?: string) => defaultMessage: 'Find all terms that exceeds threshold value', } ); + +export const ML_SEARCH_ANOMALIES_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlRuleType.mlSearchAnomaliesRequestDescription', + { + defaultMessage: 'Find all anomalies', + } +); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts index 671477b3b122b..d6497d29a04b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts @@ -20,7 +20,7 @@ export const logSearchRequest = (searchRequest: estypes.SearchRequest): string = return acc; }, []) .join('&'); - return `POST /${index}/_search?${urlParams}\n${JSON.stringify( + return `POST /${index}/_search${urlParams ? `?${urlParams}` : ''}\n${JSON.stringify( { ...searchRequest.body }, null, 2 diff --git a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts index e308619ed0f47..e8eb0f2c19ee6 100644 --- a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts +++ b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts @@ -30,41 +30,43 @@ export const getAnomalies = async ( params: AnomaliesSearchParams, mlAnomalySearch: MlAnomalySearch ): Promise => { + const queryRequest = buildAnomalyQuery(params); + return mlAnomalySearch(queryRequest, params.jobIds); +}; + +export const buildAnomalyQuery = (params: AnomaliesSearchParams): estypes.SearchRequest => { const boolCriteria = buildCriteria(params); - return mlAnomalySearch( - { - body: { - size: params.maxRecords || 100, - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:record', - analyze_wildcard: false, - }, + return { + body: { + size: params.maxRecords || 100, + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false, }, - { term: { is_interim: false } }, - { - bool: { - must: boolCriteria, - }, + }, + { term: { is_interim: false } }, + { + bool: { + must: boolCriteria, }, - ], - must_not: params.exceptionFilter?.query, - }, + }, + ], + must_not: params.exceptionFilter?.query, }, - fields: [ - { - field: '*', - include_unmapped: true, - }, - ], - sort: [{ record_score: { order: 'desc' as const } }], }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + sort: [{ record_score: { order: 'desc' as const } }], }, - params.jobIds - ); + }; }; const buildCriteria = (params: AnomaliesSearchParams): object[] => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning.ts index b27846745b011..101260235f454 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning.ts @@ -345,5 +345,29 @@ export default ({ getService }: FtrProviderContext) => { ); }); }); + + describe('preview logged requests', () => { + it('should not return requests property when not enabled', async () => { + const { logs } = await previewRule({ + supertest, + rule, + }); + + expect(logs[0].requests).toEqual(undefined); + }); + it('should return requests property when enable_logged_requests set to true', async () => { + const { logs } = await previewRule({ + supertest, + rule, + enableLoggedRequests: true, + }); + + const requests = logs[0].requests; + + expect(requests).toHaveLength(1); + expect(requests![0].description).toBe('Find all anomalies'); + expect(requests![0].request).toContain('POST /.ml-anomalies-*/_search'); + }); + }); }); }; From bb321433332db72d427a4fa6c962dde4209ef177 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:48:05 +0000 Subject: [PATCH 07/28] [Security Solution][Detection Engine] fix types --- .../lib/detection_engine/rule_types/ml/ml.test.ts | 12 ++++++------ .../server/lib/detection_engine/rule_types/ml/ml.ts | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts index 2a3fa8360e3f8..9374c729d0d0a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts @@ -97,7 +97,7 @@ describe('ml_executor', () => { it('should record a partial failure if Machine learning job summary was null', async () => { jobsSummaryMock.mockResolvedValue([]); - const response = await mlExecutor({ + const { result } = await mlExecutor({ completeRule: mlCompleteRule, tuple, ml: mlMock, @@ -119,7 +119,7 @@ describe('ml_executor', () => { expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain( 'Machine learning job(s) are not started' ); - expect(response.warningMessages.length).toEqual(1); + expect(result.warningMessages.length).toEqual(1); }); it('should record a partial failure if Machine learning job was not started', async () => { @@ -131,7 +131,7 @@ describe('ml_executor', () => { }, ]); - const response = await mlExecutor({ + const { result } = await mlExecutor({ completeRule: mlCompleteRule, tuple, ml: mlMock, @@ -153,7 +153,7 @@ describe('ml_executor', () => { expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain( 'Machine learning job(s) are not started' ); - expect(response.warningMessages.length).toEqual(1); + expect(result.warningMessages.length).toEqual(1); }); it('should report job missing errors as user errors', async () => { @@ -161,7 +161,7 @@ describe('ml_executor', () => { message: 'my_test_job_name missing', }); - const result = await mlExecutor({ + const { result } = await mlExecutor({ completeRule: mlCompleteRule, tuple, ml: mlMock, @@ -220,7 +220,7 @@ describe('ml_executor', () => { ); }); it('should call scheduleNotificationResponseActionsService', async () => { - const result = await mlExecutor({ + const { result } = await mlExecutor({ completeRule: mlCompleteRule, tuple, ml: mlMock, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts index 74a02534ca3dc..3dd14b29cb4f3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts @@ -62,7 +62,7 @@ interface MachineLearningRuleExecutorParams { isAlertSuppressionActive: boolean; experimentalFeatures: ExperimentalFeatures; scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService']; - isLoggedRequestsEnabled: boolean; + isLoggedRequestsEnabled?: boolean; } export const mlExecutor = async ({ @@ -82,7 +82,7 @@ export const mlExecutor = async ({ alertWithSuppression, experimentalFeatures, scheduleNotificationResponseActionsService, - isLoggedRequestsEnabled, + isLoggedRequestsEnabled = false, }: MachineLearningRuleExecutorParams) => { const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; From 8ae45c04e58f08d2ecfea267b914a967fdbbc193 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:32:58 +0000 Subject: [PATCH 08/28] [Security Solution][Detection Engine] fix tests --- .../lib/detection_engine/rule_types/ml/ml.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts index 9374c729d0d0a..250e78cdd20ce 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts @@ -57,9 +57,11 @@ describe('ml_executor', () => { ruleType: mlCompleteRule.ruleConfig.ruleTypeId, }); (findMlSignals as jest.Mock).mockResolvedValue({ - _shards: {}, - hits: { - hits: [], + anomalyResults: { + _shards: {}, + hits: { + hits: [], + }, }, }); (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ @@ -194,7 +196,7 @@ describe('ml_executor', () => { })) ); - const result = await mlExecutor({ + const { result } = await mlExecutor({ completeRule: mlCompleteRule, tuple, ml: mlMock, From d395a29ca1442dd8e3484762b374514bee7e62db Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 15 Jan 2025 11:04:56 +0000 Subject: [PATCH 09/28] [Security Solution][Detection Engine] ass query --- .../components/rule_preview/index.test.tsx | 13 +++- .../components/rule_preview/index.tsx | 2 + .../group_and_bulk_create.ts | 11 ++- .../query/create_query_alert_type.ts | 1 + .../rule_types/query/query.ts | 4 ++ .../lib/detection_engine/rule_types/types.ts | 2 + .../utils/search_after_bulk_create_factory.ts | 20 +++++- .../custom_query.ts | 69 +++++++++++++++++++ 8 files changed, 113 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx index 3a19fdf48a633..bebed21103294 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx @@ -40,9 +40,16 @@ jest.mock('../../../../common/hooks/use_experimental_features', () => ({ })); // rule types that do not support logged requests -const doNotSupportLoggedRequests: Type[] = ['threat_match', 'query', 'new_terms']; - -const supportLoggedRequests: Type[] = ['esql', 'eql', 'threshold', 'machine_learning']; +const doNotSupportLoggedRequests: Type[] = ['threat_match', 'new_terms']; + +const supportLoggedRequests: Type[] = [ + 'esql', + 'eql', + 'threshold', + 'machine_learning', + 'query', + 'saved_query', +]; const getMockIndexPattern = (): DataViewBase => ({ fields, diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx index 4c947ec91cd1f..9028a5a0ef15d 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx @@ -48,6 +48,8 @@ const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = [ 'eql', 'threshold', 'machine_learning', + 'query', + 'saved_query', ]; const timeRanges = [ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index 97640be9bbe0a..12d15fffb05b9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -49,6 +49,7 @@ export interface GroupAndBulkCreateParams { groupByFields: string[]; eventsTelemetry: ITelemetryEventsSender | undefined; experimentalFeatures: ExperimentalFeatures; + isLoggedRequestsEnabled: boolean; } export interface GroupAndBulkCreateReturnType extends SearchAfterAndBulkCreateReturnType { @@ -128,6 +129,7 @@ export const groupAndBulkCreate = async ({ groupByFields, eventsTelemetry, experimentalFeatures, + isLoggedRequestsEnabled, }: GroupAndBulkCreateParams): Promise => { return withSecuritySpan('groupAndBulkCreate', async () => { const tuple = runOpts.tuple; @@ -197,11 +199,14 @@ export const groupAndBulkCreate = async ({ secondaryTimestamp: runOpts.secondaryTimestamp, runtimeMappings: runOpts.runtimeMappings, additionalFilters: bucketHistoryFilter, + loggedRequestDescription: isLoggedRequestsEnabled ? 'Find events' : undefined, }; - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter( - eventsSearchParams - ); + const { searchResult, searchDuration, searchErrors, loggedRequests } = + await singleSearchAfter(eventsSearchParams); + if (isLoggedRequestsEnabled) { + toReturn.loggedRequests = loggedRequests; + } toReturn.searchAfterTimes.push(searchDuration); toReturn.errors.push(...searchErrors); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts index 284b844773b68..f9e9e39409d4e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts @@ -81,6 +81,7 @@ export const createQueryAlertType = ( bucketHistory: state.suppressionGroupHistory, licensing, scheduleNotificationResponseActionsService, + isLoggedRequestsEnabled: Boolean(state?.isLoggedRequestsEnabled), }); }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts index 106a7589a2826..df8c5dd736777 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts @@ -34,6 +34,7 @@ export const queryExecutor = async ({ bucketHistory, scheduleNotificationResponseActionsService, licensing, + isLoggedRequestsEnabled, }: { runOpts: RunOpts; experimentalFeatures: ExperimentalFeatures; @@ -44,6 +45,7 @@ export const queryExecutor = async ({ bucketHistory?: BucketHistory[]; scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService']; licensing: LicensingPluginSetup; + isLoggedRequestsEnabled: boolean; }) => { const completeRule = runOpts.completeRule; const ruleParams = completeRule.ruleParams; @@ -77,6 +79,7 @@ export const queryExecutor = async ({ groupByFields: ruleParams.alertSuppression.groupBy, eventsTelemetry, experimentalFeatures, + isLoggedRequestsEnabled, }) : { ...(await searchAfterAndBulkCreate({ @@ -95,6 +98,7 @@ export const queryExecutor = async ({ runtimeMappings: runOpts.runtimeMappings, primaryTimestamp: runOpts.primaryTimestamp, secondaryTimestamp: runOpts.secondaryTimestamp, + isLoggedRequestsEnabled, })), state: {}, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index e4ba927500184..2ea02761adb82 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -390,6 +390,7 @@ export interface SearchAfterAndBulkCreateParams { primaryTimestamp: string; secondaryTimestamp?: string; additionalFilters?: estypes.QueryDslQueryContainer[]; + isLoggedRequestsEnabled?: boolean; } export interface SearchAfterAndBulkCreateReturnType { @@ -405,6 +406,7 @@ export interface SearchAfterAndBulkCreateReturnType { userError?: boolean; warningMessages: string[]; suppressedAlertsCount?: number; + loggedRequests?: RulePreviewLoggedRequest[]; } // the new fields can be added later if needed diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index 49e03da44bb5b..ddd06b4cba00b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -26,6 +26,7 @@ import type { } from '../types'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { GenericBulkCreateResponse } from '../factories'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; @@ -56,10 +57,13 @@ export const searchAfterAndBulkCreateFactory = async ({ additionalFilters, bulkCreateExecutor, getWarningMessage, + isLoggedRequestsEnabled, }: SearchAfterAndBulkCreateFactoryParams): Promise => { + // eslint-disable-next-line complexity return withSecuritySpan('searchAfterAndBulkCreate', async () => { let toReturn = createSearchAfterReturnType(); let searchingIteration = 0; + const loggedRequests: RulePreviewLoggedRequest[] = []; // sortId tells us where to start our next consecutive search_after query let sortIds: estypes.SortResults | undefined; @@ -88,7 +92,12 @@ export const searchAfterAndBulkCreateFactory = async ({ ); if (hasSortId) { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: singleSearchLoggedRequests = [], + } = await singleSearchAfter({ searchAfterSortIds: sortIds, index: inputIndexPattern, runtimeMappings, @@ -103,6 +112,11 @@ export const searchAfterAndBulkCreateFactory = async ({ trackTotalHits, sortOrder, additionalFilters, + loggedRequestDescription: isLoggedRequestsEnabled + ? sortIds + ? `Find events after cursor ${sortIds}` + : 'Find events' + : undefined, }); mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); toReturn = mergeReturns([ @@ -116,7 +130,7 @@ export const searchAfterAndBulkCreateFactory = async ({ errors: searchErrors, }), ]); - + loggedRequests.push(...singleSearchLoggedRequests); // determine if there are any candidate signals to be processed const totalHits = getTotalHitsValue(mergedSearchResults.hits.total); const lastSortIds = getSafeSortIds( @@ -211,6 +225,6 @@ export const searchAfterAndBulkCreateFactory = async ({ } } ruleExecutionLogger.debug(`Completed bulk indexing of ${toReturn.createdSignalsCount} alert`); - return toReturn; + return { ...toReturn, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) }; }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query.ts index 09887716041d0..13ea5e62b03c5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/query/trial_license_complete_tier/custom_query.ts @@ -2777,5 +2777,74 @@ export default ({ getService }: FtrProviderContext) => { expect(get(previewAlerts[0]?._source, 'event.dataset')).toEqual('network_traffic.tls'); }); }); + + describe('preview logged requests', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/ecs_compliant' + ); + }); + + const rule: QueryRuleCreateProps = getRuleForAlertTesting(['ecs_compliant']); + const ruleWithSuppression: QueryRuleCreateProps = { + ...rule, + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 500, + unit: 'm', + }, + }, + }; + + it('should not return requests property when not enabled', async () => { + const { logs } = await previewRule({ + supertest, + rule, + }); + + expect(logs[0].requests).toEqual(undefined); + }); + it('should return requests property when enable_logged_requests set to true', async () => { + const { logs } = await previewRule({ + supertest, + rule, + enableLoggedRequests: true, + }); + + const requests = logs[0].requests; + + expect(requests).toHaveLength(1); + expect(requests![0].description).toBe('Find events'); + expect(requests![0].request).toContain('POST /ecs_compliant/_search?allow_no_indices=true'); + }); + + it('should not return requests property when not enabled and suppression configured', async () => { + const { logs } = await previewRule({ + supertest, + rule: ruleWithSuppression, + }); + + expect(logs[0].requests).toEqual(undefined); + }); + + it('should return requests property when enable_logged_requests set to true and suppression configured', async () => { + const { logs } = await previewRule({ + supertest, + rule: ruleWithSuppression, + enableLoggedRequests: true, + }); + + const requests = logs[0].requests; + + expect(requests).toHaveLength(1); + expect(requests![0].description).toBe('Find events'); + expect(requests![0].request).toContain('POST /ecs_compliant/_search?allow_no_indices=true'); + }); + }); }); }; From 677b6a57d15ca3113c3e8262dee39aaf656fe8ee Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 15 Jan 2025 11:40:06 +0000 Subject: [PATCH 10/28] [Security Solution][Detection Engine] add translations --- .../rule_types/translations.ts | 16 ++++++++++ .../utils/search_after_bulk_create_factory.ts | 29 +++++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index ee44c85c01732..83a5746530595 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -50,3 +50,19 @@ export const ML_SEARCH_ANOMALIES_DESCRIPTION = i18n.translate( defaultMessage: 'Find all anomalies', } ); + +export const FIND_EVENTS_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.queryRuleType.findEventsDescription', + { + defaultMessage: 'Find events', + } +); + +export const FIND_EVENTS_AFTER_CURSOR_DESCRIPTION = (cursor?: string) => + i18n.translate( + 'xpack.securitySolution.detectionEngine.queryRuleType.findEventsAfterCursorDescription', + { + defaultMessage: 'Find events after cursor {cursor}', + values: { cursor }, + } + ); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index ddd06b4cba00b..db589034769d0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -29,6 +29,19 @@ import type { GenericBulkCreateResponse } from '../factories'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; +import * as i18n from '../translations'; + +const createLoggedRequestsDescription = ( + isLoggedRequestsEnabled: boolean | undefined, + sortIds: estypes.SortResults | undefined +): string | undefined => { + if (!isLoggedRequestsEnabled) { + return undefined; + } + return sortIds + ? i18n.FIND_EVENTS_AFTER_CURSOR_DESCRIPTION(JSON.stringify(sortIds)) + : i18n.FIND_EVENTS_DESCRIPTION; +}; export interface SearchAfterAndBulkCreateFactoryParams extends SearchAfterAndBulkCreateParams { bulkCreateExecutor: (params: { @@ -112,11 +125,10 @@ export const searchAfterAndBulkCreateFactory = async ({ trackTotalHits, sortOrder, additionalFilters, - loggedRequestDescription: isLoggedRequestsEnabled - ? sortIds - ? `Find events after cursor ${sortIds}` - : 'Find events' - : undefined, + loggedRequestDescription: createLoggedRequestsDescription( + isLoggedRequestsEnabled, + sortIds + ), }); mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); toReturn = mergeReturns([ @@ -225,6 +237,11 @@ export const searchAfterAndBulkCreateFactory = async ({ } } ruleExecutionLogger.debug(`Completed bulk indexing of ${toReturn.createdSignalsCount} alert`); - return { ...toReturn, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) }; + + if (isLoggedRequestsEnabled) { + toReturn.loggedRequests = loggedRequests; + } + + return toReturn; }); }; From e0a2bba4174cd9fff66a5a64e7d4636801e4b901 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 15 Jan 2025 11:47:30 +0000 Subject: [PATCH 11/28] [Security Solution][Detection Engine] fixes logger --- .../logged_requests/log_search_request.ts | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts index d6497d29a04b4..56cd49cf574a5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts @@ -7,26 +7,23 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -export const logSearchRequest = (searchRequest: estypes.SearchRequest): string => { - // handle deprecated body property which still used widely across Detection Engine - if (searchRequest.body) { - const { body, index, ...params } = searchRequest; - const urlParams = Object.entries(params) - .reduce((acc, [key, value]) => { - if (value != null) { - acc.push(`${key}=${value}`); - } +export const logSearchRequest = (searchRequest: estypes.SearchRequest): string | undefined => { + const { body, index, ...params } = searchRequest; + const urlParams = Object.entries(params) + .reduce((acc, [key, value]) => { + if (value != null) { + acc.push(`${key}=${value}`); + } - return acc; - }, []) - .join('&'); - return `POST /${index}/_search${urlParams ? `?${urlParams}` : ''}\n${JSON.stringify( - { ...searchRequest.body }, - null, - 2 - )}`; + return acc; + }, []) + .join('&'); + + const url = `/${index}/_search${urlParams ? `?${urlParams}` : ''}`; + + if (body) { + return `POST ${url}\n${JSON.stringify({ ...body }, null, 2)}`; } - // TODO: fix me - return '???'; + return `GET ${url}`; }; From 084a47ec135bf2c518f98c96e45522dbd8c74355 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 15 Jan 2025 12:49:03 +0000 Subject: [PATCH 12/28] [Security Solution][Detection Engine] refactoring --- .../rule_types/query/alert_suppression/group_and_bulk_create.ts | 2 ++ .../server/lib/detection_engine/rule_types/query/query.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index 12d15fffb05b9..ec95b486e22d7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -55,6 +55,7 @@ export interface GroupAndBulkCreateParams { export interface GroupAndBulkCreateReturnType extends SearchAfterAndBulkCreateReturnType { state: { suppressionGroupHistory: BucketHistory[]; + isLoggedRequestsEnabled?: boolean; }; } @@ -151,6 +152,7 @@ export const groupAndBulkCreate = async ({ errors: [], warningMessages: [], state: { + isLoggedRequestsEnabled, suppressionGroupHistory: filteredBucketHistory, }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts index df8c5dd736777..3a197ab5178a2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts @@ -100,7 +100,7 @@ export const queryExecutor = async ({ secondaryTimestamp: runOpts.secondaryTimestamp, isLoggedRequestsEnabled, })), - state: {}, + state: { isLoggedRequestsEnabled }, }; scheduleNotificationResponseActionsService({ From 395f2dc3d8bfad2eb5ed3f5225bcabde893d5ee4 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:42:38 +0000 Subject: [PATCH 13/28] [Security Solution][Detection Engine] fix type error --- .../rule_types/utils/logged_requests/log_search_request.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts index 56cd49cf574a5..1b0d3d9dde80b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_search_request.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -export const logSearchRequest = (searchRequest: estypes.SearchRequest): string | undefined => { +export const logSearchRequest = (searchRequest: estypes.SearchRequest): string => { const { body, index, ...params } = searchRequest; const urlParams = Object.entries(params) .reduce((acc, [key, value]) => { From 68995fb9bf2c08066986670b49f4696502fccccd Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 20 Jan 2025 18:09:20 +0000 Subject: [PATCH 14/28] [Security Solution][Detection Engine] refactoring + new terms POC --- .../rule_preview/rule_preview.gen.ts | 3 +- .../rule_preview/rule_preview.schema.yaml | 4 +- .../components/rule_preview/index.test.tsx | 3 +- .../components/rule_preview/index.tsx | 2 + .../rule_preview/logged_requests.tsx | 15 +-- .../rule_preview/logged_requests_item.tsx | 50 ++++------ .../rule_preview/logged_requests_pages.tsx | 79 +++++++++++++++ .../rule_preview/logged_requests_query.tsx | 44 +++++++++ .../components/rule_preview/preview_logs.tsx | 10 +- .../components/rule_preview/translations.ts | 6 ++ .../rule_preview/use_accordion_styling.ts | 3 +- .../new_terms/create_new_terms_alert_type.ts | 33 ++++++- .../new_terms/multi_terms_composite.ts | 28 +++++- .../threshold/find_threshold_signals.ts | 4 +- .../rule_types/threshold/utils.test.ts | 14 --- .../rule_types/threshold/utils.ts | 14 --- .../rule_types/translations.ts | 48 +++++++++ .../lib/detection_engine/rule_types/types.ts | 6 ++ .../utils/search_after_bulk_create_factory.ts | 21 ++-- .../rule_types/utils/single_search_after.ts | 14 ++- .../rule_types/utils/utils.test.ts | 15 +++ .../rule_types/utils/utils.ts | 14 +++ .../trial_license_complete_tier/new_terms.ts | 98 +++++++++++++++++++ 23 files changed, 434 insertions(+), 94 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx create mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.tsx diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.gen.ts index ad9b6d9ea12c2..ac49b2de2afe5 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.gen.ts @@ -37,9 +37,10 @@ export const RulePreviewParams = z.object({ export type RulePreviewLoggedRequest = z.infer; export const RulePreviewLoggedRequest = z.object({ - request: NonEmptyString, + request: NonEmptyString.optional(), description: NonEmptyString.optional(), duration: z.number().int().optional(), + request_type: NonEmptyString.optional(), }); export type RulePreviewLogs = z.infer; diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.schema.yaml index 400b84e533a02..c680fc6b89459 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.schema.yaml @@ -110,8 +110,8 @@ components: $ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString' duration: type: integer - required: - - request + request_type: + $ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString' RulePreviewLogs: type: object diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx index bebed21103294..f2315d519b2e2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx @@ -40,7 +40,7 @@ jest.mock('../../../../common/hooks/use_experimental_features', () => ({ })); // rule types that do not support logged requests -const doNotSupportLoggedRequests: Type[] = ['threat_match', 'new_terms']; +const doNotSupportLoggedRequests: Type[] = ['threat_match']; const supportLoggedRequests: Type[] = [ 'esql', @@ -49,6 +49,7 @@ const supportLoggedRequests: Type[] = [ 'machine_learning', 'query', 'saved_query', + 'new_terms', ]; const getMockIndexPattern = (): DataViewBase => ({ diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx index 9028a5a0ef15d..fd480022a7a33 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx @@ -50,6 +50,7 @@ const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = [ 'machine_learning', 'query', 'saved_query', + 'new_terms', ]; const timeRanges = [ @@ -323,6 +324,7 @@ const RulePreviewComponent: React.FC = ({ hasNoiseWarning={hasNoiseWarning} isAborted={isAborted} showElasticsearchRequests={showElasticsearchRequests && isLoggedRequestsSupported} + ruleType={ruleType} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx index d7b62c6f08c69..2e99eca3449c6 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx @@ -8,7 +8,7 @@ import type { FC } from 'react'; import React, { useMemo } from 'react'; import { EuiSpacer } from '@elastic/eui'; -import { css } from '@emotion/css'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { RulePreviewLogs } from '../../../../../common/api/detection_engine'; import * as i18n from './translations'; @@ -16,7 +16,10 @@ import { OptimizedAccordion } from './optimized_accordion'; import { LoggedRequestsItem } from './logged_requests_item'; import { useAccordionStyling } from './use_accordion_styling'; -const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[] }> = ({ logs }) => { +const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[]; ruleType: Type }> = ({ + logs, + ruleType, +}) => { const cssStyles = useAccordionStyling(); const AccordionContent = useMemo( @@ -25,12 +28,12 @@ const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[] }> = ({ logs }) => { {logs.map((log) => ( - + ))} ), - [logs] + [logs, ruleType] ); if (logs.length === 0) { @@ -44,9 +47,7 @@ const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[] }> = ({ logs }) => { data-test-subj="preview-logged-requests-accordion" buttonContent={i18n.LOGGED_REQUESTS_ACCORDION_BUTTON} borders="horizontal" - css={css` - ${cssStyles} - `} + css={cssStyles} > {AccordionContent} diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx index 2f2e7d74bf826..769d413cd3762 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx @@ -7,23 +7,29 @@ import type { FC, PropsWithChildren } from 'react'; import React from 'react'; -import { css } from '@emotion/css'; -import { EuiSpacer, EuiCodeBlock, useEuiPaddingSize, EuiFlexItem } from '@elastic/eui'; +import { useEuiPaddingSize } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { RulePreviewLogs } from '../../../../../common/api/detection_engine'; import * as i18n from './translations'; import { PreferenceFormattedDate } from '../../../../common/components/formatted_date'; import { OptimizedAccordion } from './optimized_accordion'; +import { LoggedRequestsQuery } from './logged_requests_query'; import { useAccordionStyling } from './use_accordion_styling'; +import { LoggedRequestsPages } from './logged_requests_pages'; -const LoggedRequestsItemComponent: FC> = ({ +const PAGE_VIEW_RULE_TYPES: Type[] = ['query', 'saved_query']; + +const LoggedRequestsItemComponent: FC> = ({ startedAt, duration, - requests, + requests = [], + ruleType, }) => { const paddingLarge = useEuiPaddingSize('l'); const cssStyles = useAccordionStyling(); + const isPageViewSupported = PAGE_VIEW_RULE_TYPES.includes(ruleType); return ( > = ({ } id={`ruleExecution-${startedAt}`} - css={css` - margin-left: ${paddingLarge}; - ${cssStyles} - `} + css={{ + 'margin-left': paddingLarge, + ...cssStyles, + }} > - {(requests ?? []).map((request, key) => ( - - - - {request?.description ?? null} {request?.duration ? `[${request.duration}ms]` : null} - - - - {request.request} - - - ))} + {isPageViewSupported ? ( + + ) : ( + requests.map((request, key) => ) + )} ); }; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx new file mode 100644 index 0000000000000..67f3c1a552f05 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FC } from 'react'; +import React from 'react'; + +import { useEuiPaddingSize } from '@elastic/eui'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine'; +import { OptimizedAccordion } from './optimized_accordion'; +import { useAccordionStyling } from './use_accordion_styling'; +import { LoggedRequestsQuery } from './logged_requests_query'; + +const ruleRequestsTypesMap: Record> = { + query: { + findDocuments: 'pageDelimiter', + }, + saved_query: { + findDocuments: 'pageDelimiter', + }, +}; + +const transformRequestsToPages = ( + requests: RulePreviewLoggedRequest[], + ruleType: Type +): RulePreviewLoggedRequest[][] => { + const pages: RulePreviewLoggedRequest[][] = []; + requests.forEach((request) => { + if (pages.length === 0) { + pages.push([request]); + } else if ( + request.request_type && + ruleRequestsTypesMap[ruleType][request.request_type] === 'pageDelimiter' + ) { + pages.push([request]); + } else { + pages.at(-1)?.push(request); + } + }); + + return pages; +}; + +const LoggedRequestsPagesComponent: FC<{ + requests: RulePreviewLoggedRequest[]; + ruleType: Type; +}> = ({ requests, ruleType }) => { + const cssStyles = useAccordionStyling(); + const paddingLarge = useEuiPaddingSize('l'); + const pages = transformRequestsToPages(requests, ruleType); + + return ( + <> + {pages.map((pageRequests, key) => ( + + {pageRequests.map((request, requestKey) => ( + + ))} + + ))} + + ); +}; + +export const LoggedRequestsPages = React.memo(LoggedRequestsPagesComponent); +LoggedRequestsPages.displayName = 'LoggedRequestsPages'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.tsx new file mode 100644 index 0000000000000..71dc154e45ac2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FC } from 'react'; +import React from 'react'; + +import { EuiSpacer, EuiCodeBlock, useEuiPaddingSize, EuiFlexItem } from '@elastic/eui'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine'; + +export const LoggedRequestsQueryComponent: FC = ({ + description, + duration, + request, +}) => { + const paddingLarge = useEuiPaddingSize('l'); + + return ( + + + + {description ?? null} {duration ? `[${duration}ms]` : null} + + + {request ? ( + + {request} + + ) : null} + + ); +}; + +export const LoggedRequestsQuery = React.memo(LoggedRequestsQueryComponent); +LoggedRequestsQuery.displayName = 'LoggedRequestsQuery'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx index 196408cbc1371..a204d89e1c641 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx @@ -7,8 +7,8 @@ import type { FC, PropsWithChildren } from 'react'; import React, { Fragment, useMemo } from 'react'; -import { css } from '@emotion/css'; import { EuiCallOut, EuiText, EuiSpacer, EuiAccordion } from '@elastic/eui'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { RulePreviewLogs } from '../../../../../common/api/detection_engine'; import * as i18n from './translations'; @@ -20,6 +20,7 @@ interface PreviewLogsProps { hasNoiseWarning: boolean; isAborted: boolean; showElasticsearchRequests: boolean; + ruleType: Type; } interface SortedLogs { @@ -53,6 +54,7 @@ const PreviewLogsComponent: React.FC = ({ hasNoiseWarning, isAborted, showElasticsearchRequests, + ruleType, }) => { const sortedLogs = useMemo( () => @@ -76,7 +78,7 @@ const PreviewLogsComponent: React.FC = ({ {isAborted ? : null} - {showElasticsearchRequests ? : null} + {showElasticsearchRequests ? : null} ); }; @@ -110,9 +112,7 @@ const LogAccordion: FC> = ({ logs, isError, isError ? i18n.QUERY_PREVIEW_SEE_ALL_ERRORS : i18n.QUERY_PREVIEW_SEE_ALL_WARNINGS } borders="horizontal" - css={css` - ${cssStyles} - `} + css={cssStyles} > {restOfLogs.map((log, key) => ( { const paddingLarge = useEuiPaddingSize('l'); const paddingSmall = useEuiPaddingSize('s'); - return `padding-bottom: ${paddingLarge}; - padding-top: ${paddingSmall};`; + return { 'padding-bottom': paddingLarge, 'padding-top': paddingSmall }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 99a7bfc0c0b4d..1d08f96146a7e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -39,15 +39,18 @@ import { getUnprocessedExceptionsWarnings, getMaxSignalsWarning, getSuppressionMaxSignalsWarning, + stringifyAfterKey, } from '../utils/utils'; import { createEnrichEventsFunction } from '../utils/enrichments'; import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; import { multiTermsComposite } from './multi_terms_composite'; import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; +import * as i18n from '../translations'; export const createNewTermsAlertType = ( createOptions: CreateRuleOptions -): SecurityAlertType => { +): SecurityAlertType => { const { logger, licensing, experimentalFeatures, scheduleNotificationResponseActionsService } = createOptions; return { @@ -119,6 +122,9 @@ export const createNewTermsAlertType = ( state, } = execOptions; + const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled); + const loggedRequests: RulePreviewLoggedRequest[] = []; + // Validate the history window size compared to `from` at runtime as well as in the `validate` // function because rule preview does not use the `validate` function defined on the rule type validateHistoryWindowStart({ @@ -168,7 +174,12 @@ export const createNewTermsAlertType = ( // PHASE 1: Fetch a page of terms using a composite aggregation. This will collect a page from // all of the terms seen over the last rule interval. In the next phase we'll determine which // ones are new. - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + const { + searchResult, + searchDuration, + searchErrors, + loggedRequests: firstPhaseLoggedRequests = [], + } = await singleSearchAfter({ aggregations: buildRecentTermsAgg({ fields: params.newTermsFields, after: afterKey, @@ -185,7 +196,11 @@ export const createNewTermsAlertType = ( primaryTimestamp, secondaryTimestamp, runtimeMappings, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION(stringifyAfterKey(afterKey)) + : undefined, }); + loggedRequests.push(...firstPhaseLoggedRequests); const searchResultWithAggs = searchResult as RecentTermsAggResult; if (!searchResultWithAggs.aggregations) { throw new Error('Aggregations were missing on recent terms search result'); @@ -316,7 +331,9 @@ export const createNewTermsAlertType = ( afterKey, createAlertsHook, isAlertSuppressionActive, + isLoggedRequestsEnabled, }); + loggedRequests.push(...(bulkCreateResult?.loggedRequests ?? [])); if (bulkCreateResult?.alertsWereTruncated) { break; @@ -330,6 +347,7 @@ export const createNewTermsAlertType = ( searchResult: pageSearchResult, searchDuration: pageSearchDuration, searchErrors: pageSearchErrors, + loggedRequests: pageSearchLoggedRequests = [], } = await singleSearchAfter({ aggregations: buildNewTermsAgg({ newValueWindowStart: tuple.from, @@ -350,9 +368,13 @@ export const createNewTermsAlertType = ( pageSize: 0, primaryTimestamp, secondaryTimestamp, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(afterKey)) + : undefined, }); result.searchAfterTimes.push(pageSearchDuration); result.errors.push(...pageSearchErrors); + loggedRequests.push(...pageSearchLoggedRequests); logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`); @@ -374,6 +396,7 @@ export const createNewTermsAlertType = ( searchResult: docFetchSearchResult, searchDuration: docFetchSearchDuration, searchErrors: docFetchSearchErrors, + loggedRequests: docFetchLoggedRequests = [], } = await singleSearchAfter({ aggregations: buildDocFetchAgg({ timestampField: aggregatableTimestampField, @@ -392,9 +415,13 @@ export const createNewTermsAlertType = ( pageSize: 0, primaryTimestamp, secondaryTimestamp, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION(stringifyAfterKey(afterKey)) + : undefined, }); result.searchAfterTimes.push(docFetchSearchDuration); result.errors.push(...docFetchSearchErrors); + loggedRequests.push(...docFetchLoggedRequests); const docFetchResultWithAggs = docFetchSearchResult as DocFetchAggResult; @@ -424,7 +451,7 @@ export const createNewTermsAlertType = ( responseActions: completeRule.ruleParams.responseActions, }); - return { ...result, state }; + return { ...result, state, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) }; }, }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts index 572443566f8d8..8f32f6c53adfb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -22,10 +22,16 @@ import type { CreateAlertsHook, } from './build_new_terms_aggregation'; import type { NewTermsFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; -import { getMaxSignalsWarning, getSuppressionMaxSignalsWarning } from '../utils/utils'; +import { + getMaxSignalsWarning, + getSuppressionMaxSignalsWarning, + stringifyAfterKey, +} from '../utils/utils'; import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression'; import type { RuleServices, SearchAfterAndBulkCreateReturnType, RunOpts } from '../types'; +import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; +import * as i18n from '../translations'; /** * composite aggregation page batch size set to 500 as it shows th best performance(refer https://github.com/elastic/kibana/pull/157413) and @@ -49,6 +55,7 @@ interface MultiTermsCompositeArgsBase { afterKey: Record | undefined; createAlertsHook: CreateAlertsHook; isAlertSuppressionActive: boolean; + isLoggedRequestsEnabled: boolean; } interface MultiTermsCompositeArgs extends MultiTermsCompositeArgsBase { @@ -75,6 +82,7 @@ const multiTermsCompositeNonRetryable = async ({ createAlertsHook, batchSize, isAlertSuppressionActive, + isLoggedRequestsEnabled, }: MultiTermsCompositeArgs): Promise< Omit, 'suppressedItemsCount'> | undefined > => { @@ -87,6 +95,8 @@ const multiTermsCompositeNonRetryable = async ({ secondaryTimestamp, } = runOpts; + const loggedRequests: RulePreviewLoggedRequest[] = []; + let internalAfterKey = afterKey ?? undefined; let i = 0; @@ -115,6 +125,7 @@ const multiTermsCompositeNonRetryable = async ({ searchResult: pageSearchResult, searchDuration: pageSearchDuration, searchErrors: pageSearchErrors, + loggedRequests: pageSearchLoggedRequests = [], } = await singleSearchAfter({ aggregations: buildCompositeNewTermsAgg({ newValueWindowStart: tuple.from, @@ -136,10 +147,14 @@ const multiTermsCompositeNonRetryable = async ({ pageSize: 0, primaryTimestamp, secondaryTimestamp, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(internalAfterKey)) + : undefined, }); result.searchAfterTimes.push(pageSearchDuration); result.errors.push(...pageSearchErrors); + loggedRequests.push(...pageSearchLoggedRequests); logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`); const pageSearchResultWithAggs = pageSearchResult as CompositeNewTermsAggResult; @@ -156,6 +171,7 @@ const multiTermsCompositeNonRetryable = async ({ searchResult: docFetchSearchResult, searchDuration: docFetchSearchDuration, searchErrors: docFetchSearchErrors, + loggedRequests: docFetchLoggedRequests = [], } = await singleSearchAfter({ aggregations: buildCompositeDocFetchAgg({ newValueWindowStart: tuple.from, @@ -175,9 +191,13 @@ const multiTermsCompositeNonRetryable = async ({ pageSize: 0, primaryTimestamp, secondaryTimestamp, + loggedRequestDescription: isLoggedRequestsEnabled + ? i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION(stringifyAfterKey(internalAfterKey)) + : undefined, }); result.searchAfterTimes.push(docFetchSearchDuration); result.errors.push(...docFetchSearchErrors); + loggedRequests.push(...docFetchLoggedRequests); const docFetchResultWithAggs = docFetchSearchResult as CompositeDocFetchAggResult; @@ -186,6 +206,10 @@ const multiTermsCompositeNonRetryable = async ({ } const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs); + // console.log('bulkCreateResult.loggedRequests ', bulkCreateResult.loggedRequests.length); + if (isLoggedRequestsEnabled) { + bulkCreateResult.loggedRequests = loggedRequests; + } if (bulkCreateResult.alertsWereTruncated) { result.warningMessages.push( @@ -197,6 +221,8 @@ const multiTermsCompositeNonRetryable = async ({ internalAfterKey = batch[batch.length - 1]?.key; } + + return { loggedRequests }; }; /** diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts index 6beced85dd28f..1f4e817efbd43 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts @@ -29,9 +29,9 @@ import type { ThresholdBucket, ThresholdSingleBucketAggregationResult, } from './types'; -import { shouldFilterByCardinality, searchResultHasAggs, stringifyAfterKey } from './utils'; +import { shouldFilterByCardinality, searchResultHasAggs } from './utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { getMaxSignalsWarning } from '../utils/utils'; +import { getMaxSignalsWarning, stringifyAfterKey } from '../utils/utils'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import * as i18n from '../translations'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts index 8b1e0dc9d6377..2a54c3b0e156f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts @@ -13,23 +13,9 @@ import { getSignalHistory, getThresholdTermsHash, transformBulkCreatedItemsToHits, - stringifyAfterKey, } from './utils'; describe('threshold utils', () => { - describe('stringifyAfterKey', () => { - it('should stringify after_key object with single key value', () => { - expect(stringifyAfterKey({ 'agent.name': 'test' })).toBe('agent.name: test'); - }); - it('should stringify after_key object with multiple key values', () => { - expect(stringifyAfterKey({ 'agent.name': 'test', 'destination.ip': '127.0.0.1' })).toBe( - 'agent.name: test, destination.ip: 127.0.0.1' - ); - }); - it('should return undefined if after_key is undefined', () => { - expect(stringifyAfterKey(undefined)).toBeUndefined(); - }); - }); describe('calcualteThresholdSignalUuid', () => { it('should generate a uuid without key', () => { const startedAt = new Date('2020-12-17T16:27:00Z'); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts index 2e5e0faa29eda..c73becbdd8f57 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts @@ -116,17 +116,3 @@ export const transformBulkCreatedItemsToHits = ( }, }; }); - -/** - * converts ES after_key object into string - * for example: { "agent.name": "test" } would become `agent.name: test` - */ -export const stringifyAfterKey = (afterKey: Record | undefined) => { - if (!afterKey) { - return; - } - - return Object.entries(afterKey) - .map((entry) => entry.join(': ')) - .join(', '); -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index 83a5746530595..f473f59190f5b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -66,3 +66,51 @@ export const FIND_EVENTS_AFTER_CURSOR_DESCRIPTION = (cursor?: string) => values: { cursor }, } ); + +export const FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION = (afterKey?: string) => + afterKey + ? i18n.translate( + 'xpack.securitySolution.detectionEngine.newTermsRuleType.findAllNewTermsFieldsAfterDescription', + { + defaultMessage: 'Find all values in history window after {afterKey}', + values: { afterKey }, + } + ) + : i18n.translate( + 'xpack.securitySolution.detectionEngine.newTermsRuleType.findAllNewTermsFieldsDescription', + { + defaultMessage: 'Find all values in history window', + } + ); + +export const FIND_NEW_TERMS_VALUES_DESCRIPTION = (afterKey?: string) => + afterKey + ? i18n.translate( + 'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsValuesAfterDescription', + { + defaultMessage: 'Find new values after {afterKey}', + values: { afterKey }, + } + ) + : i18n.translate( + 'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsValuesDescription', + { + defaultMessage: 'Find new values', + } + ); + +export const FIND_NEW_TERMS_EVENTS_DESCRIPTION = (afterKey?: string) => + afterKey + ? i18n.translate( + 'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsEventsAfterDescription', + { + defaultMessage: 'Find documents associated with new values after {afterKey}', + values: { afterKey }, + } + ) + : i18n.translate( + 'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsEventsDescription', + { + defaultMessage: 'Find documents associated with new values', + } + ); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 2ea02761adb82..6224ae13352d2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -409,6 +409,12 @@ export interface SearchAfterAndBulkCreateReturnType { loggedRequests?: RulePreviewLoggedRequest[]; } +export interface LoggedRequestsEnabled { + type: string; + description: string; + skipRequestQuery?: boolean; +} + // the new fields can be added later if needed export interface OverrideBodyQuery { _source?: estypes.SearchSourceConfig; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index db589034769d0..15a741e243230 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -23,6 +23,7 @@ import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType, SignalSourceHit, + LoggedRequestsEnabled, } from '../types'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { GenericBulkCreateResponse } from '../factories'; @@ -31,16 +32,23 @@ import type { RulePreviewLoggedRequest } from '../../../../../common/api/detecti import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; import * as i18n from '../translations'; -const createLoggedRequestsDescription = ( +const createLoggedRequestsConfig = ( isLoggedRequestsEnabled: boolean | undefined, - sortIds: estypes.SortResults | undefined -): string | undefined => { + sortIds: estypes.SortResults | undefined, + page: number +): LoggedRequestsEnabled | undefined => { if (!isLoggedRequestsEnabled) { return undefined; } - return sortIds + const description = sortIds ? i18n.FIND_EVENTS_AFTER_CURSOR_DESCRIPTION(JSON.stringify(sortIds)) : i18n.FIND_EVENTS_DESCRIPTION; + + return { + type: 'findDocuments', + description, + skipRequestQuery: page > 1, // skipping query logging for performance reasons, so we won't overwhelm Kibana with large response size + }; }; export interface SearchAfterAndBulkCreateFactoryParams extends SearchAfterAndBulkCreateParams { @@ -125,9 +133,10 @@ export const searchAfterAndBulkCreateFactory = async ({ trackTotalHits, sortOrder, additionalFilters, - loggedRequestDescription: createLoggedRequestsDescription( + loggedRequestsEnabled: createLoggedRequestsConfig( isLoggedRequestsEnabled, - sortIds + sortIds, + searchingIteration ), }); mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index 90c63f124a686..f3d21fd3a2ca6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -11,6 +11,7 @@ import type { AlertInstanceState, RuleExecutorServices, } from '@kbn/alerting-plugin/server'; +import type { LoggedRequestsEnabled } from '../types'; import type { SignalSearchResponse, SignalSource, OverrideBodyQuery } from '../types'; import { buildEventsSearchQuery } from './build_events_query'; import { createErrorsFromShard, makeFloatString } from './utils'; @@ -37,7 +38,7 @@ export interface SingleSearchAfterParams { runtimeMappings: estypes.MappingRuntimeFields | undefined; additionalFilters?: estypes.QueryDslQueryContainer[]; overrideBody?: OverrideBodyQuery; - loggedRequestDescription?: string; + loggedRequestsEnabled?: LoggedRequestsEnabled; } // utilize search_after for paging results into bulk. @@ -60,7 +61,7 @@ export const singleSearchAfter = async < trackTotalHits, additionalFilters, overrideBody, - loggedRequestDescription, + loggedRequestsEnabled, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; @@ -105,10 +106,13 @@ export const singleSearchAfter = async < errors: nextSearchAfterResult._shards.failures ?? [], }); - if (loggedRequestDescription) { + if (loggedRequestsEnabled) { loggedRequests.push({ - request: logSearchRequest(searchAfterQuery), - description: loggedRequestDescription, + request: loggedRequestsEnabled.skipRequestQuery + ? undefined + : logSearchRequest(searchAfterQuery), + description: loggedRequestsEnabled.description, + request_type: loggedRequestsEnabled.type, duration: Math.round(end - start), }); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts index 3430d2ae903ef..9d08d2c22c521 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts @@ -49,6 +49,7 @@ import { getUnprocessedExceptionsWarnings, getDisabledActionsWarningText, calculateFromValue, + stringifyAfterKey, } from './utils'; import type { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from '../types'; import { @@ -1757,6 +1758,20 @@ describe('utils', () => { }); }); + describe('stringifyAfterKey', () => { + it('should stringify after_key object with single key value', () => { + expect(stringifyAfterKey({ 'agent.name': 'test' })).toBe('agent.name: test'); + }); + it('should stringify after_key object with multiple key values', () => { + expect(stringifyAfterKey({ 'agent.name': 'test', 'destination.ip': '127.0.0.1' })).toBe( + 'agent.name: test, destination.ip: 127.0.0.1' + ); + }); + it('should return undefined if after_key is undefined', () => { + expect(stringifyAfterKey(undefined)).toBeUndefined(); + }); + }); + describe('getDisabledActionsWarningText', () => { const alertsCreated = true; const alertsNotCreated = false; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index 75c8b30dc88df..45f2b47850fc6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -1103,6 +1103,20 @@ export type SequenceSuppressionTermsAndFieldsFactory = ( subAlerts: Array>; }; +/** + * converts ES after_key object into string + * for example: { "agent.name": "test" } would become `agent.name: test` + */ +export const stringifyAfterKey = (afterKey: Record | undefined) => { + if (!afterKey) { + return; + } + + return Object.entries(afterKey) + .map((entry) => entry.join(': ')) + .join(', '); +}; + export const buildShellAlertSuppressionTermsAndFields = ({ shellAlert, buildingBlockAlerts, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts index 970d6ab3ba6ed..b93f92e56c050 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts @@ -1364,5 +1364,103 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('preview logged requests', () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.ip'], + from: ruleExecutionStart, + history_window_start: historicalWindowStart, + query: '*', + }; + + it('should not return requests property when not enabled', async () => { + const { logs } = await previewRule({ + supertest, + rule, + }); + + expect(logs[0].requests).toEqual(undefined); + }); + + it('should return requests property when enable_logged_requests set to true for single new term field', async () => { + // historical window documents + const historicalDocuments = [ + { + host: { name: 'host-0', ip: '127.0.0.1' }, + }, + { + host: { name: 'host-1', ip: '127.0.0.2' }, + }, + ]; + + // rule execution documents + const ruleExecutionDocuments = [ + { + host: { name: 'host-0', ip: '127.0.0.2' }, + }, + { + host: { name: 'host-2', ip: '127.0.0.1' }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const { logs } = await previewRule({ + supertest, + rule: { ...rule, query: `id: "${testId}"`, new_terms_fields: ['host.name'] }, + }); + + expect(logs[0].requests).toEqual(3); + const requests = logs[0].requests ?? []; + + expect(requests[0].description).toBe('Find historical values'); + expect(requests[1].description).toBe('Find new terms'); + expect(requests[2].description).toBe('Find events associated with new events'); + }); + + it('should return requests property when enable_logged_requests set to true for multiple fields', async () => { + // historical window documents + const historicalDocuments = [ + { + host: { name: 'host-0', ip: '127.0.0.1' }, + }, + { + host: { name: 'host-1', ip: '127.0.0.2' }, + }, + ]; + + // rule execution documents + const ruleExecutionDocuments = [ + { + host: { name: 'host-0', ip: '127.0.0.2' }, + }, + { + host: { name: 'host-1', ip: '127.0.0.1' }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const { logs } = await previewRule({ + supertest, + rule: { ...rule, query: `id: "${testId}"` }, + }); + + expect(logs[0].requests).toEqual(3); + const requests = logs[0].requests ?? []; + + expect(requests[0].description).toBe('Find historical values'); + expect(requests[1].description).toBe('Find new terms'); + expect(requests[2].description).toBe('Find events associated with new events'); + }); + }); }); }; From 6f0743c61782e54dd109426c4a2f1ff1bd276aa4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 20 Jan 2025 18:24:38 +0000 Subject: [PATCH 15/28] [CI] Auto-commit changed files from 'yarn openapi:bundle' --- ...ity_solution_detections_api_2023_10_31.bundled.schema.yaml | 4 ++-- ...ity_solution_detections_api_2023_10_31.bundled.schema.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 6c865ab356c1d..9d90c04695569 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -5269,8 +5269,8 @@ components: type: integer request: $ref: '#/components/schemas/NonEmptyString' - required: - - request + request_type: + $ref: '#/components/schemas/NonEmptyString' RulePreviewLogs: type: object properties: diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 583944c5b3435..dfc8fe12d23ff 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -4418,8 +4418,8 @@ components: type: integer request: $ref: '#/components/schemas/NonEmptyString' - required: - - request + request_type: + $ref: '#/components/schemas/NonEmptyString' RulePreviewLogs: type: object properties: From 112be8423ea1026a7a3e18b8233ab6fd72fcaa3d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:10:59 +0000 Subject: [PATCH 16/28] query rule paging --- .../components/rule_preview/logged_requests_pages.tsx | 2 +- .../components/rule_preview/use_accordion_styling.ts | 7 +++++-- .../query/alert_suppression/group_and_bulk_create.ts | 8 +++++++- .../rule_types/utils/search_after_bulk_create_factory.ts | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx index 67f3c1a552f05..4740fba8bddf5 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -59,7 +59,7 @@ const LoggedRequestsPagesComponent: FC<{ { const paddingLarge = useEuiPaddingSize('l'); const paddingSmall = useEuiPaddingSize('s'); - return { 'padding-bottom': paddingLarge, 'padding-top': paddingSmall }; + return useMemo( + () => ({ 'padding-bottom': paddingLarge, 'padding-top': paddingSmall }), + [paddingLarge, paddingSmall] + ); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index ec95b486e22d7..babcb652e0c25 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -33,6 +33,7 @@ import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../../c import type { ExperimentalFeatures } from '../../../../../../common'; import { createEnrichEventsFunction } from '../../utils/enrichments'; import { getNumberOfSuppressedAlerts } from '../../utils/get_number_of_suppressed_alerts'; +import * as i18n from '../../translations'; export interface BucketHistory { key: Record; @@ -201,7 +202,12 @@ export const groupAndBulkCreate = async ({ secondaryTimestamp: runOpts.secondaryTimestamp, runtimeMappings: runOpts.runtimeMappings, additionalFilters: bucketHistoryFilter, - loggedRequestDescription: isLoggedRequestsEnabled ? 'Find events' : undefined, + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findDocuments', + description: i18n.FIND_EVENTS_DESCRIPTION, + } + : undefined, }; const { searchResult, searchDuration, searchErrors, loggedRequests } = await singleSearchAfter(eventsSearchParams); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index 15a741e243230..e40f27932bbe8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -47,7 +47,7 @@ const createLoggedRequestsConfig = ( return { type: 'findDocuments', description, - skipRequestQuery: page > 1, // skipping query logging for performance reasons, so we won't overwhelm Kibana with large response size + skipRequestQuery: page > 2, // skipping query logging for performance reasons, so we won't overwhelm Kibana with large response size }; }; From 92a1c855fed2332f60ebab0cf9d2782471e23864 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 21 Jan 2025 12:14:15 +0000 Subject: [PATCH 17/28] [Security Solution][Detection Engine] threshold rule --- .../rule_preview/logged_requests_item.tsx | 8 ++------ .../rule_preview/logged_requests_pages.tsx | 7 +++++++ .../rule_types/ml/find_ml_signals.ts | 1 + .../threshold/find_threshold_signals.ts | 15 +++++++++++---- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx index 769d413cd3762..eea3c93db8905 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx @@ -17,9 +17,7 @@ import { PreferenceFormattedDate } from '../../../../common/components/formatted import { OptimizedAccordion } from './optimized_accordion'; import { LoggedRequestsQuery } from './logged_requests_query'; import { useAccordionStyling } from './use_accordion_styling'; -import { LoggedRequestsPages } from './logged_requests_pages'; - -const PAGE_VIEW_RULE_TYPES: Type[] = ['query', 'saved_query']; +import { LoggedRequestsPages, isPageViewSupported } from './logged_requests_pages'; const LoggedRequestsItemComponent: FC> = ({ startedAt, @@ -29,8 +27,6 @@ const LoggedRequestsItemComponent: FC { const paddingLarge = useEuiPaddingSize('l'); const cssStyles = useAccordionStyling(); - const isPageViewSupported = PAGE_VIEW_RULE_TYPES.includes(ruleType); - return ( - {isPageViewSupported ? ( + {isPageViewSupported(ruleType) ? ( ) : ( requests.map((request, key) => ) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx index 4740fba8bddf5..8038af8d83140 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -15,6 +15,8 @@ import { OptimizedAccordion } from './optimized_accordion'; import { useAccordionStyling } from './use_accordion_styling'; import { LoggedRequestsQuery } from './logged_requests_query'; +const PAGE_VIEW_RULE_TYPES: Type[] = ['query', 'saved_query', 'threshold']; + const ruleRequestsTypesMap: Record> = { query: { findDocuments: 'pageDelimiter', @@ -22,8 +24,13 @@ const ruleRequestsTypesMap: Record> = { saved_query: { findDocuments: 'pageDelimiter', }, + threshold: { + findThresholdBuckets: 'pageDelimiter', + }, }; +export const isPageViewSupported = (ruleType: Type) => PAGE_VIEW_RULE_TYPES.includes(ruleType); + const transformRequestsToPages = ( requests: RulePreviewLoggedRequest[], ruleType: Type diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts index b5eea99e93071..39050e5678988 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/find_ml_signals.ts @@ -59,6 +59,7 @@ export const findMlSignals = async ({ request: logSearchRequest(searchQuery), description: i18n.ML_SEARCH_ANOMALIES_DESCRIPTION, duration: anomalyResults.took, + request_type: 'findAnomalies', }); } return { anomalyResults, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts index 1f4e817efbd43..d2d3c2f35e255 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts @@ -117,8 +117,12 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(stringifyAfterKey(sortKeys)) + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findThresholdBuckets', + description: i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(stringifyAfterKey(sortKeys)), + skipRequestQuery: loggedRequests.length > 2, + } : undefined, }); @@ -164,8 +168,11 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION() + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findThresholdBuckets', + description: i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(), + } : undefined, }); From b020a2c71a220c101c2eefc505ea61967d885cd1 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:07:32 +0000 Subject: [PATCH 18/28] [Security Solution][Detection Engine] FUNCTIONAL CHANGES --- .../rule_preview/logged_requests_pages.tsx | 7 +++-- .../components/rule_preview/translations.ts | 6 ---- .../new_terms/create_new_terms_alert_type.ts | 30 +++++++++++++++---- .../new_terms/multi_terms_composite.ts | 29 ++++++++++++++---- .../rule_types/utils/single_search_after.ts | 8 +++-- 5 files changed, 57 insertions(+), 23 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx index 8038af8d83140..4d4dba2cf21e2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -15,8 +15,6 @@ import { OptimizedAccordion } from './optimized_accordion'; import { useAccordionStyling } from './use_accordion_styling'; import { LoggedRequestsQuery } from './logged_requests_query'; -const PAGE_VIEW_RULE_TYPES: Type[] = ['query', 'saved_query', 'threshold']; - const ruleRequestsTypesMap: Record> = { query: { findDocuments: 'pageDelimiter', @@ -27,9 +25,12 @@ const ruleRequestsTypesMap: Record> = { threshold: { findThresholdBuckets: 'pageDelimiter', }, + new_terms: { + findAllTerms: 'pageDelimiter', + }, }; -export const isPageViewSupported = (ruleType: Type) => PAGE_VIEW_RULE_TYPES.includes(ruleType); +export const isPageViewSupported = (ruleType: Type) => Boolean(ruleRequestsTypesMap[ruleType]); const transformRequestsToPages = ( requests: RulePreviewLoggedRequest[], diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts index f5899c6f2dbc3..0b25071a76830 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts @@ -178,12 +178,6 @@ export const LOGGED_REQUEST_ITEM_ACCORDION_UNKNOWN_TIME_BUTTON = i18n.translate( defaultMessage: 'Preview logged requests', } ); -export const LOGGED_REQUEST_SHOW_CODE_BLOCK_BUTTON = i18n.translate( - 'xpack.securitySolution.detectionEngine.queryPreview.loggedRequestShowCodeBlockButtonLabel', - { - defaultMessage: 'Show code block', - } -); export const VIEW_DETAILS_FOR_ROW = ({ ariaRowindex, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 1d08f96146a7e..17cd795bf7650 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -163,6 +163,7 @@ export const createNewTermsAlertType = ( if (exceptionsWarning) { result.warningMessages.push(exceptionsWarning); } + let pageNumber = 0; // There are 2 conditions that mean we're finished: either there were still too many alerts to create // after deduplication and the array of alerts was truncated before being submitted to ES, or there were @@ -171,6 +172,7 @@ export const createNewTermsAlertType = ( // in which case createdSignalsCount would still be less than maxSignals. Since valid alerts were truncated from // the array in that case, we stop and report the errors. while (result.createdSignalsCount <= params.maxSignals) { + pageNumber++; // PHASE 1: Fetch a page of terms using a composite aggregation. This will collect a page from // all of the terms seen over the last rule interval. In the next phase we'll determine which // ones are new. @@ -196,8 +198,14 @@ export const createNewTermsAlertType = ( primaryTimestamp, secondaryTimestamp, runtimeMappings, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION(stringifyAfterKey(afterKey)) + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findAllTerms', + description: i18n.FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION( + stringifyAfterKey(afterKey) + ), + skipRequestQuery: pageNumber > 2, + } : undefined, }); loggedRequests.push(...firstPhaseLoggedRequests); @@ -368,8 +376,12 @@ export const createNewTermsAlertType = ( pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(afterKey)) + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findNewTerms', + description: i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(afterKey)), + skipRequestQuery: pageNumber > 2, + } : undefined, }); result.searchAfterTimes.push(pageSearchDuration); @@ -415,8 +427,14 @@ export const createNewTermsAlertType = ( pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION(stringifyAfterKey(afterKey)) + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findDocuments', + description: i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION( + stringifyAfterKey(afterKey) + ), + skipRequestQuery: pageNumber > 2, + } : undefined, }); result.searchAfterTimes.push(docFetchSearchDuration); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts index 8f32f6c53adfb..a66b83842b4bf 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -84,7 +84,10 @@ const multiTermsCompositeNonRetryable = async ({ isAlertSuppressionActive, isLoggedRequestsEnabled, }: MultiTermsCompositeArgs): Promise< - Omit, 'suppressedItemsCount'> | undefined + | (Omit, 'suppressedItemsCount'> & { + loggedRequests?: RulePreviewLoggedRequest; + }) + | undefined > => { const { ruleExecutionLogger, @@ -100,8 +103,10 @@ const multiTermsCompositeNonRetryable = async ({ let internalAfterKey = afterKey ?? undefined; let i = 0; + let pageNumber = 0; while (i < buckets.length) { + pageNumber++; const batch = buckets.slice(i, i + batchSize); i += batchSize; const batchFilters = batch.map((b) => { @@ -147,8 +152,14 @@ const multiTermsCompositeNonRetryable = async ({ pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(internalAfterKey)) + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findNewTerms', + description: i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION( + stringifyAfterKey(internalAfterKey) + ), + skipRequestQuery: Boolean(afterKey) || pageNumber > 2, + } : undefined, }); @@ -191,8 +202,14 @@ const multiTermsCompositeNonRetryable = async ({ pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestDescription: isLoggedRequestsEnabled - ? i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION(stringifyAfterKey(internalAfterKey)) + loggedRequestsEnabled: isLoggedRequestsEnabled + ? { + type: 'findDocuments', + description: i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION( + stringifyAfterKey(internalAfterKey) + ), + skipRequestQuery: Boolean(afterKey) || pageNumber > 2, + } : undefined, }); result.searchAfterTimes.push(docFetchSearchDuration); @@ -206,7 +223,7 @@ const multiTermsCompositeNonRetryable = async ({ } const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs); - // console.log('bulkCreateResult.loggedRequests ', bulkCreateResult.loggedRequests.length); + if (isLoggedRequestsEnabled) { bulkCreateResult.loggedRequests = loggedRequests; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index f3d21fd3a2ca6..689878fc7837a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -11,8 +11,12 @@ import type { AlertInstanceState, RuleExecutorServices, } from '@kbn/alerting-plugin/server'; -import type { LoggedRequestsEnabled } from '../types'; -import type { SignalSearchResponse, SignalSource, OverrideBodyQuery } from '../types'; +import type { + SignalSearchResponse, + SignalSource, + OverrideBodyQuery, + LoggedRequestsEnabled, +} from '../types'; import { buildEventsSearchQuery } from './build_events_query'; import { createErrorsFromShard, makeFloatString } from './utils'; import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; From 448f0386d3218e8542a31f82096e205e5f87db16 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:23:34 +0000 Subject: [PATCH 19/28] refactoring --- .../rule_preview/logged_requests.test.tsx | 6 ++--- .../rule_preview/logged_requests_pages.tsx | 4 +-- .../new_terms/create_new_terms_alert_type.ts | 2 +- .../new_terms/multi_terms_composite.ts | 27 +++++++++---------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx index c0cf8870c162a..ba710fe3d06f2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx @@ -16,13 +16,13 @@ import { previewLogs } from './__mocks__/preview_logs'; describe('LoggedRequests', () => { it('should not render component if logs are empty', () => { - render(, { wrapper: TestProviders }); + render(, { wrapper: TestProviders }); expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeNull(); }); it('should open accordion on click and render list of request items', async () => { - render(, { wrapper: TestProviders }); + render(, { wrapper: TestProviders }); expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeInTheDocument(); @@ -32,7 +32,7 @@ describe('LoggedRequests', () => { }); it('should render code content on logged request item accordion click', async () => { - render(, { wrapper: TestProviders }); + render(, { wrapper: TestProviders }); expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeInTheDocument(); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx index 4d4dba2cf21e2..644415237d231 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -15,7 +15,7 @@ import { OptimizedAccordion } from './optimized_accordion'; import { useAccordionStyling } from './use_accordion_styling'; import { LoggedRequestsQuery } from './logged_requests_query'; -const ruleRequestsTypesMap: Record> = { +const ruleRequestsTypesMap: Partial>> = { query: { findDocuments: 'pageDelimiter', }, @@ -42,7 +42,7 @@ const transformRequestsToPages = ( pages.push([request]); } else if ( request.request_type && - ruleRequestsTypesMap[ruleType][request.request_type] === 'pageDelimiter' + ruleRequestsTypesMap[ruleType]?.[request.request_type] === 'pageDelimiter' ) { pages.push([request]); } else { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 17cd795bf7650..28a65e5e0ec38 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -343,7 +343,7 @@ export const createNewTermsAlertType = ( }); loggedRequests.push(...(bulkCreateResult?.loggedRequests ?? [])); - if (bulkCreateResult?.alertsWereTruncated) { + if (bulkCreateResult && 'alertsWereTruncated' in bulkCreateResult) { break; } } else { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts index a66b83842b4bf..229b4b8b9d244 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -62,6 +62,16 @@ interface MultiTermsCompositeArgs extends MultiTermsCompositeArgsBase { batchSize: number; } +interface LoggedRequestsProps { + loggedRequests?: RulePreviewLoggedRequest[]; +} + +type MultiTermsCompositeResult = + | (Omit, 'suppressedItemsCount'> & + LoggedRequestsProps) + | LoggedRequestsProps + | undefined; + /** * This helper does phase2/phase3(look README) got multiple new terms * It takes full page of results from phase 1 (10,000) @@ -83,12 +93,7 @@ const multiTermsCompositeNonRetryable = async ({ batchSize, isAlertSuppressionActive, isLoggedRequestsEnabled, -}: MultiTermsCompositeArgs): Promise< - | (Omit, 'suppressedItemsCount'> & { - loggedRequests?: RulePreviewLoggedRequest; - }) - | undefined -> => { +}: MultiTermsCompositeArgs): Promise => { const { ruleExecutionLogger, tuple, @@ -224,15 +229,11 @@ const multiTermsCompositeNonRetryable = async ({ const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs); - if (isLoggedRequestsEnabled) { - bulkCreateResult.loggedRequests = loggedRequests; - } - if (bulkCreateResult.alertsWereTruncated) { result.warningMessages.push( isAlertSuppressionActive ? getSuppressionMaxSignalsWarning() : getMaxSignalsWarning() ); - return bulkCreateResult; + return isLoggedRequestsEnabled ? { ...bulkCreateResult, loggedRequests } : bulkCreateResult; } } @@ -249,9 +250,7 @@ const multiTermsCompositeNonRetryable = async ({ */ export const multiTermsComposite = async ( args: MultiTermsCompositeArgsBase -): Promise< - Omit, 'suppressedItemsCount'> | undefined -> => { +): Promise => { let retryBatchSize = BATCH_SIZE; const ruleExecutionLogger = args.runOpts.ruleExecutionLogger; return pRetry( From 36d667dfd41751bd1f6a99b22d8fd63779634639 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:41:07 +0000 Subject: [PATCH 20/28] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.serverless.yaml | 4 ++-- oas_docs/output/kibana.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 9bc95e9034e6f..8d45fa52bf16d 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -45766,8 +45766,8 @@ components: type: integer request: $ref: '#/components/schemas/Security_Detections_API_NonEmptyString' - required: - - request + request_type: + $ref: '#/components/schemas/Security_Detections_API_NonEmptyString' Security_Detections_API_RulePreviewLogs: type: object properties: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 0817106e13b41..ebd5e7a3f129c 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -52634,8 +52634,8 @@ components: type: integer request: $ref: '#/components/schemas/Security_Detections_API_NonEmptyString' - required: - - request + request_type: + $ref: '#/components/schemas/Security_Detections_API_NonEmptyString' Security_Detections_API_RulePreviewLogs: type: object properties: From 26a87a5ae6a95d5225d705a1238b0fe071c133fb Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:29:21 +0000 Subject: [PATCH 21/28] [Security Solution][Detection Engine] add unit tests --- .../rule_preview/__mocks__/preview_logs.ts | 27 ++++ .../rule_preview/logged_requests.test.tsx | 30 +++- .../rule_preview/logged_requests_item.tsx | 2 +- .../logged_requests_pages.test.tsx | 128 ++++++++++++++++++ .../rule_preview/logged_requests_pages.tsx | 33 ++++- .../logged_requests_query.test.tsx | 52 +++++++ .../rule_preview/use_accordion_styling.ts | 2 +- 7 files changed, 264 insertions(+), 10 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.test.tsx create mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.test.tsx diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/__mocks__/preview_logs.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/__mocks__/preview_logs.ts index 1e380d1bb4561..1c09e50cc000c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/__mocks__/preview_logs.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/__mocks__/preview_logs.ts @@ -91,3 +91,30 @@ export const previewLogs: RulePreviewLogs[] = [ ], }, ]; + +export const queryRuleTypePreviewLogs: RulePreviewLogs[] = [ + { + errors: [], + warnings: [ + 'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.', + ], + startedAt: '2025-01-21T16:48:50.891Z', + duration: 1103, + requests: [ + { + request: + 'POST /apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*,very-unique/_search?allow_no_indices=true&ignore_unavailable=true\n{\n "size": 100,\n "query": {\n "bool": {\n "filter": [\n {\n "bool": {\n "must": [],\n "filter": [\n {\n "query_string": {\n "query": "*"\n }\n }\n ],\n "should": [],\n "must_not": []\n }\n },\n {\n "range": {\n "@timestamp": {\n "lte": "2025-01-21T16:48:50.891Z",\n "gte": "2025-01-21T15:17:50.891Z",\n "format": "strict_date_optional_time"\n }\n }\n }\n ]\n }\n },\n "fields": [\n {\n "field": "*",\n "include_unmapped": true\n },\n {\n "field": "@timestamp",\n "format": "strict_date_optional_time"\n }\n ],\n "runtime_mappings": {},\n "sort": [\n {\n "@timestamp": {\n "order": "asc",\n "unmapped_type": "date"\n }\n }\n ]\n}', + description: 'Find documents', + request_type: 'findDocuments', + duration: 137, + }, + { + request: + 'POST /apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*,very-unique/_search?allow_no_indices=true&ignore_unavailable=true\n{\n "size": 100,\n "query": {\n "bool": {\n "filter": [\n {\n "bool": {\n "must": [],\n "filter": [\n {\n "query_string": {\n "query": "*"\n }\n }\n ],\n "should": [],\n "must_not": []\n }\n },\n {\n "range": {\n "@timestamp": {\n "lte": "2025-01-21T16:48:50.891Z",\n "gte": "2025-01-21T15:17:50.891Z",\n "format": "strict_date_optional_time"\n }\n }\n }\n ]\n }\n },\n "fields": [\n {\n "field": "*",\n "include_unmapped": true\n },\n {\n "field": "@timestamp",\n "format": "strict_date_optional_time"\n }\n ],\n "runtime_mappings": {},\n "sort": [\n {\n "@timestamp": {\n "order": "asc",\n "unmapped_type": "date"\n }\n }\n ],\n "search_after": [\n 1737472675562\n ]\n}', + description: 'Find documents after cursor [1737472675562]', + request_type: 'findDocuments', + duration: 192, + }, + ], + }, +]; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx index ba710fe3d06f2..153cd7ac7b5ac 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.test.tsx @@ -12,7 +12,7 @@ import userEvent from '@testing-library/user-event'; import { TestProviders } from '../../../../common/mock/test_providers'; import { LoggedRequests } from './logged_requests'; -import { previewLogs } from './__mocks__/preview_logs'; +import { previewLogs, queryRuleTypePreviewLogs } from './__mocks__/preview_logs'; describe('LoggedRequests', () => { it('should not render component if logs are empty', () => { @@ -65,4 +65,32 @@ describe('LoggedRequests', () => { /POST \/packetbeat-8\.14\.2\/_search\?ignore_unavailable=true/ ); }); + + it('should render code content when rule supports page view', async () => { + render(, { + wrapper: TestProviders, + }); + + expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeInTheDocument(); + + await userEvent.click(screen.getByText('Preview logged requests')); + + const loggedRequestsItem = screen.getAllByTestId('preview-logged-requests-item-accordion')[0]; + + expect(loggedRequestsItem).toHaveTextContent('Rule execution started at'); + expect(loggedRequestsItem).toHaveTextContent('[1103ms]'); + + await userEvent.click(loggedRequestsItem.querySelector('button') as HTMLElement); + + expect(screen.getAllByTestId('preview-logged-requests-page-accordion')).toHaveLength(2); + + await userEvent.click(screen.getByText('Page 1 of search queries')); + + expect(screen.getAllByTestId('preview-logged-request-description')[0]).toHaveTextContent( + 'Find documents [137ms]' + ); + expect(screen.getAllByTestId('preview-logged-request-code-block')[0]).toHaveTextContent( + 'POST /apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*,very-unique/_search?allow_no_indices=true&ignore_unavailable=true' + ); + }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx index eea3c93db8905..71a6c8f5cd78c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx @@ -46,7 +46,7 @@ const LoggedRequestsItemComponent: FC diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.test.tsx new file mode 100644 index 0000000000000..71bd42fcdb00a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.test.tsx @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render, screen } from '@testing-library/react'; + +import { TestProviders } from '../../../../common/mock/test_providers'; +import { LoggedRequestsPages } from './logged_requests_pages'; +import userEvent from '@testing-library/user-event'; + +const customQueryRuleTypeRequests = [ + { + request_type: 'findDocuments', + description: 'request #1', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findDocuments', + description: 'request #2', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findDocuments', + description: 'request #3', + }, +]; + +describe('LoggedRequestsPages', () => { + it('should render 3 pages for query rule', () => { + render(, { + wrapper: TestProviders, + }); + + const pages = screen.getAllByTestId('preview-logged-requests-page-accordion'); + expect(pages).toHaveLength(3); + expect(pages[0]).toHaveTextContent('Page 1 of search queries'); + expect(pages[2]).toHaveTextContent('Page 3 of search queries'); + }); + + it('should render 3 pages for saved_query rule', () => { + render(, { + wrapper: TestProviders, + }); + + const pages = screen.getAllByTestId('preview-logged-requests-page-accordion'); + expect(pages).toHaveLength(3); + }); + + it('should render 2 pages for threshold rule', () => { + const requests = [ + { + request_type: 'findThresholdBuckets', + description: 'request #1', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findThresholdBuckets', + description: 'request #2', + request: 'POST test/_search', + duration: 10, + }, + ]; + render(, { + wrapper: TestProviders, + }); + + const pages = screen.getAllByTestId('preview-logged-requests-page-accordion'); + expect(pages).toHaveLength(2); + expect(pages[0]).toHaveTextContent('Page 1 of search queries'); + expect(pages[1]).toHaveTextContent('Page 2 of search queries'); + }); + + it('should render 2 pages for new_terms rule', async () => { + const requests = [ + { + request_type: 'findAllTerms', + description: 'request #1', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findNewTerms', + description: 'request #2', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findDocuments', + description: 'request #3', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findAllTerms', + description: 'request #4', + request: 'POST test/_search', + duration: 10, + }, + { + request_type: 'findNewTerms', + description: 'request #5', + request: 'POST test/_search', + duration: 10, + }, + ]; + render(, { + wrapper: TestProviders, + }); + + const pages = screen.getAllByTestId('preview-logged-requests-page-accordion'); + expect(pages).toHaveLength(2); + + // renders 3 requests on page 1 + await userEvent.click(screen.getByText('Page 1 of search queries')); + expect(screen.getAllByTestId('preview-logged-request-code-block')).toHaveLength(3); + + // renders 2 additional requests on page 2 + await userEvent.click(screen.getByText('Page 2 of search queries')); + expect(screen.getAllByTestId('preview-logged-request-code-block')).toHaveLength(5); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx index 644415237d231..c11eaef6e08c2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -8,14 +8,16 @@ import type { FC } from 'react'; import React from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; import { useEuiPaddingSize } from '@elastic/eui'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; + import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine'; import { OptimizedAccordion } from './optimized_accordion'; import { useAccordionStyling } from './use_accordion_styling'; import { LoggedRequestsQuery } from './logged_requests_query'; -const ruleRequestsTypesMap: Partial>> = { +const ruleRequestsTypesMap = { query: { findDocuments: 'pageDelimiter', }, @@ -30,11 +32,20 @@ const ruleRequestsTypesMap: Partial>> = { }, }; -export const isPageViewSupported = (ruleType: Type) => Boolean(ruleRequestsTypesMap[ruleType]); +type RuleTypesWithPages = keyof typeof ruleRequestsTypesMap; + +export const isPageViewSupported = (ruleType: Type): ruleType is RuleTypesWithPages => + ruleType in ruleRequestsTypesMap; + +const hasRequestType = ( + ruleType: RuleTypesWithPages, + requestType: string +): requestType is keyof (typeof ruleRequestsTypesMap)[typeof ruleType] => + requestType in ruleRequestsTypesMap[ruleType]; const transformRequestsToPages = ( requests: RulePreviewLoggedRequest[], - ruleType: Type + ruleType: RuleTypesWithPages ): RulePreviewLoggedRequest[][] => { const pages: RulePreviewLoggedRequest[][] = []; requests.forEach((request) => { @@ -42,7 +53,8 @@ const transformRequestsToPages = ( pages.push([request]); } else if ( request.request_type && - ruleRequestsTypesMap[ruleType]?.[request.request_type] === 'pageDelimiter' + hasRequestType(ruleType, request.request_type) && + ruleRequestsTypesMap[ruleType][request.request_type] === 'pageDelimiter' ) { pages.push([request]); } else { @@ -55,7 +67,7 @@ const transformRequestsToPages = ( const LoggedRequestsPagesComponent: FC<{ requests: RulePreviewLoggedRequest[]; - ruleType: Type; + ruleType: RuleTypesWithPages; }> = ({ requests, ruleType }) => { const cssStyles = useAccordionStyling(); const paddingLarge = useEuiPaddingSize('l'); @@ -65,12 +77,19 @@ const LoggedRequestsPagesComponent: FC<{ <> {pages.map((pageRequests, key) => ( + } borders="horizontal" css={{ - 'margin-left': paddingLarge, + marginLeft: paddingLarge, ...cssStyles, }} > diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.test.tsx new file mode 100644 index 0000000000000..9d92d2f335750 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_query.test.tsx @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render, screen } from '@testing-library/react'; + +import { LoggedRequestsQuery } from './logged_requests_query'; + +const description = 'Retrieve source documents when ES|QL query is not aggregable'; +const duration = 8; +const request = + 'POST /packetbeat-8.14.2/_search?ignore_unavailable=true\n{\n "query": {\n "bool": {\n "filter": {\n "ids": {\n "values": [\n "yB7awpEBluhaSO8ejVKZ",\n "yR7awpEBluhaSO8ejVKZ",\n "yh7awpEBluhaSO8ejVKZ",\n "yx7awpEBluhaSO8ejVKZ",\n "zB7awpEBluhaSO8ejVKZ",\n "zR7awpEBluhaSO8ejVKZ",\n "zh7awpEBluhaSO8ejVKZ",\n "zx7awpEBluhaSO8ejVKZ",\n "0B7awpEBluhaSO8ejVKZ",\n "0R7awpEBluhaSO8ejVKZ",\n "0h7awpEBluhaSO8ejVKZ",\n "0x7awpEBluhaSO8ejVKZ",\n "1B7awpEBluhaSO8ejVKZ",\n "1R7awpEBluhaSO8ejVKZ",\n "1h7awpEBluhaSO8ejVKZ",\n "1x7awpEBluhaSO8ejVKZ",\n "2B7awpEBluhaSO8ejVKZ",\n "2R7awpEBluhaSO8ejVKZ",\n "2h7awpEBluhaSO8ejVKZ",\n "2x7awpEBluhaSO8ejVKZ",\n "3B7awpEBluhaSO8ejVKZ",\n "3R7awpEBluhaSO8ejVKZ",\n "3h7awpEBluhaSO8ejVKZ",\n "3x7awpEBluhaSO8ejVKZ",\n "4B7awpEBluhaSO8ejVKZ",\n "4R7awpEBluhaSO8ejVKZ",\n "4h7awpEBluhaSO8ejVKZ",\n "4x7awpEBluhaSO8ejVKZ",\n "5B7awpEBluhaSO8ejVKZ",\n "5R7awpEBluhaSO8ejVKZ",\n "5h7awpEBluhaSO8ejVKZ",\n "5x7awpEBluhaSO8ejVKZ",\n "6B7awpEBluhaSO8ejVKZ",\n "6R7awpEBluhaSO8ejVKZ",\n "6h7awpEBluhaSO8ejVKZ",\n "6x7awpEBluhaSO8ejVKZ",\n "7B7awpEBluhaSO8ejVKZ",\n "7R7awpEBluhaSO8ejVKZ",\n "7h7awpEBluhaSO8ejVKZ",\n "7x7awpEBluhaSO8ejVKZ",\n "8B7awpEBluhaSO8ejVKZ",\n "8R7awpEBluhaSO8ejVKZ",\n "8h7awpEBluhaSO8ejVKZ",\n "8x7awpEBluhaSO8ejVKZ",\n "9B7awpEBluhaSO8ejVKZ",\n "9R7awpEBluhaSO8ejVKZ",\n "9h7awpEBluhaSO8ejVKZ",\n "9x7awpEBluhaSO8ejVKZ",\n "-B7awpEBluhaSO8ejVKZ",\n "-R7awpEBluhaSO8ejVKZ",\n "-h7awpEBluhaSO8ejVKZ",\n "-x7awpEBluhaSO8ejVKZ",\n "_B7awpEBluhaSO8ejVKZ",\n "_R7awpEBluhaSO8ejVKZ",\n "_h7awpEBluhaSO8ejVKZ",\n "_x7awpEBluhaSO8ejVKZ",\n "AB7awpEBluhaSO8ejVOZ",\n "AR7awpEBluhaSO8ejVOZ",\n "Ah7awpEBluhaSO8ejVOZ",\n "Ax7awpEBluhaSO8ejVOZ",\n "BB7awpEBluhaSO8ejVOZ",\n "BR7awpEBluhaSO8ejVOZ",\n "Bh7awpEBluhaSO8ejVOZ",\n "Bx7awpEBluhaSO8ejVOZ",\n "CB7awpEBluhaSO8ejVOZ",\n "CR7awpEBluhaSO8ejVOZ",\n "Ch7awpEBluhaSO8ejVOZ",\n "Cx7awpEBluhaSO8ejVOZ",\n "DB7awpEBluhaSO8ejVOZ",\n "DR7awpEBluhaSO8ejVOZ",\n "Dh7awpEBluhaSO8ejVOZ",\n "Dx7awpEBluhaSO8ejVOZ",\n "EB7awpEBluhaSO8ejVOZ",\n "ER7awpEBluhaSO8ejVOZ",\n "Eh7awpEBluhaSO8ejVOZ",\n "Ex7awpEBluhaSO8ejVOZ",\n "FB7awpEBluhaSO8ejVOZ",\n "FR7awpEBluhaSO8ejVOZ",\n "Fh7awpEBluhaSO8ejVOZ",\n "Fx7awpEBluhaSO8ejVOZ",\n "GB7awpEBluhaSO8ejVOZ",\n "GR7awpEBluhaSO8ejVOZ",\n "Gh7awpEBluhaSO8ejVOZ",\n "Gx7awpEBluhaSO8ejVOZ",\n "HB7awpEBluhaSO8ejVOZ",\n "HR7awpEBluhaSO8ejVOZ",\n "Hh7awpEBluhaSO8ejVOZ",\n "Hx7awpEBluhaSO8ejVOZ",\n "IB7awpEBluhaSO8ejVOZ",\n "IR7awpEBluhaSO8ejVOZ",\n "Ih7awpEBluhaSO8ejVOZ",\n "Ix7awpEBluhaSO8ejVOZ",\n "JB7awpEBluhaSO8ejVOZ",\n "JR7awpEBluhaSO8ejVOZ",\n "Jh7awpEBluhaSO8ejVOZ",\n "Jx7awpEBluhaSO8ejVOZ",\n "KB7awpEBluhaSO8ejVOZ",\n "KR7awpEBluhaSO8ejVOZ",\n "Kh7awpEBluhaSO8ejVOZ",\n "Kx7awpEBluhaSO8ejVOZ",\n "LB7awpEBluhaSO8ejVOZ"\n ]\n }\n }\n }\n },\n "_source": false,\n "fields": [\n "*"\n ]\n}'; + +describe('LoggedRequestsQuery', () => { + it('should not render code block when request field is empty', () => { + render(); + + expect(screen.queryByTestId('preview-logged-request-code-block')).toBeNull(); + }); + + it('should render code block', () => { + render(); + + expect(screen.queryByTestId('preview-logged-request-code-block')).toHaveTextContent( + 'POST /packetbeat-8.14.2/_search?ignore_unavailable=true' + ); + }); + + it('should render duration', () => { + render(); + + expect(screen.queryByTestId('preview-logged-request-description')).toHaveTextContent('8ms'); + }); + + it('should not render duration when it absent', () => { + render(); + + expect(screen.queryByTestId('preview-logged-request-description')).not.toHaveTextContent('ms'); + }); + + it('should render description', () => { + render(); + + expect(screen.queryByTestId('preview-logged-request-description')).toHaveTextContent( + description + ); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts index 9b44ddf9a10f9..ee06a812d75e0 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts @@ -12,7 +12,7 @@ export const useAccordionStyling = () => { const paddingSmall = useEuiPaddingSize('s'); return useMemo( - () => ({ 'padding-bottom': paddingLarge, 'padding-top': paddingSmall }), + () => ({ paddingBottom: paddingLarge, paddingTop: paddingSmall }), [paddingLarge, paddingSmall] ); }; From 82dc3da2d1d1bf5713bba46d39f100f416a99ca7 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:42:16 +0000 Subject: [PATCH 22/28] fix tests --- .../trial_license_complete_tier/new_terms.ts | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts index b93f92e56c050..c58b43c31f8e2 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/new_terms/trial_license_complete_tier/new_terms.ts @@ -1413,14 +1413,22 @@ export default ({ getService }: FtrProviderContext) => { const { logs } = await previewRule({ supertest, rule: { ...rule, query: `id: "${testId}"`, new_terms_fields: ['host.name'] }, + enableLoggedRequests: true, }); - expect(logs[0].requests).toEqual(3); + expect(logs[0].requests?.length).toEqual(4); const requests = logs[0].requests ?? []; - expect(requests[0].description).toBe('Find historical values'); - expect(requests[1].description).toBe('Find new terms'); - expect(requests[2].description).toBe('Find events associated with new events'); + expect(requests[0].description).toBe('Find all values'); + expect(requests[0].request_type).toBe('findAllTerms'); + + expect(requests[1].description).toBe('Find new values'); + expect(requests[1].request_type).toBe('findNewTerms'); + + expect(requests[2].description).toBe('Find documents associated with new values'); + expect(requests[2].request_type).toBe('findDocuments'); + + expect(requests[3].description).toBe('Find all values after host.name: host-2'); }); it('should return requests property when enable_logged_requests set to true for multiple fields', async () => { @@ -1452,14 +1460,20 @@ export default ({ getService }: FtrProviderContext) => { const { logs } = await previewRule({ supertest, rule: { ...rule, query: `id: "${testId}"` }, + enableLoggedRequests: true, }); - expect(logs[0].requests).toEqual(3); + expect(logs[0].requests?.length).toEqual(4); const requests = logs[0].requests ?? []; - expect(requests[0].description).toBe('Find historical values'); - expect(requests[1].description).toBe('Find new terms'); - expect(requests[2].description).toBe('Find events associated with new events'); + expect(requests[0].description).toBe('Find all values'); + expect(requests[0].request_type).toBe('findAllTerms'); + + expect(requests[1].description).toBe('Find new values'); + expect(requests[1].request_type).toBe('findNewTerms'); + + expect(requests[2].description).toBe('Find documents associated with new values'); + expect(requests[2].request_type).toBe('findDocuments'); }); }); }); From abf429ca4b0e1b585cb22b62092149a2bc6dcb5a Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 22 Jan 2025 15:56:56 +0000 Subject: [PATCH 23/28] [Security Solution][Detection Engine] fix more tests --- .../server/lib/detection_engine/rule_types/translations.ts | 4 ++-- .../detection_engine/rule_edit/preview.cy.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts index f473f59190f5b..b9bd6f858b9ae 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts @@ -72,14 +72,14 @@ export const FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION = (afterKey?: string) => ? i18n.translate( 'xpack.securitySolution.detectionEngine.newTermsRuleType.findAllNewTermsFieldsAfterDescription', { - defaultMessage: 'Find all values in history window after {afterKey}', + defaultMessage: 'Find all values after {afterKey}', values: { afterKey }, } ) : i18n.translate( 'xpack.securitySolution.detectionEngine.newTermsRuleType.findAllNewTermsFieldsDescription', { - defaultMessage: 'Find all values in history window', + defaultMessage: 'Find all values', } ); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts index 268968c76ecc0..8998d91ae3b28 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts @@ -22,6 +22,7 @@ import { submitRulePreview, toggleLoggedRequestsAccordion, toggleLoggedRequestsItemAccordion, + selectIndicatorMatchType, } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; @@ -69,7 +70,8 @@ describe( }); }); - it('does not show preview logged requests checkbox', () => { + it('does not show preview logged requests checkbox fro Indicator Match rule', () => { + selectIndicatorMatchType(); cy.get(RULES_CREATION_PREVIEW_REFRESH_BUTTON).should('be.visible'); cy.get(PREVIEW_LOGGED_REQUESTS_CHECKBOX).should('not.exist'); }); From 804e316d88e395af4117d11cc3f1d75203f871fb Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 22 Jan 2025 18:38:21 +0000 Subject: [PATCH 24/28] [Security Solution][Detection Engine] refactoring --- .../rule_preview/logged_requests_item.tsx | 17 ++++++++++++++++- .../components/rule_preview/translations.ts | 7 +++++++ .../detection_engine/rule_edit/preview.cy.ts | 6 ++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx index 71a6c8f5cd78c..a66e58b9280ba 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx @@ -8,7 +8,7 @@ import type { FC, PropsWithChildren } from 'react'; import React from 'react'; -import { useEuiPaddingSize } from '@elastic/eui'; +import { useEuiPaddingSize, EuiText, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { RulePreviewLogs } from '../../../../../common/api/detection_engine'; @@ -50,6 +50,21 @@ const LoggedRequestsItemComponent: FC + {requests.length > 2 ? ( + <> + + + {i18n.REQUESTS_SAMPLE_WARNING} + + + + ) : null} {isPageViewSupported(ruleType) ? ( ) : ( diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts index 0b25071a76830..b65b181a72128 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/translations.ts @@ -206,3 +206,10 @@ export const RULE_PREVIEW_DESCRIPTION = i18n.translate( 'Rule preview reflects the current configuration of your rule settings and exceptions, click refresh icon to see the updated preview.', } ); + +export const REQUESTS_SAMPLE_WARNING = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.rulePreviewRequestSampleWarningText', + { + defaultMessage: 'Sample search queries logged only for first 2 requests of each type.', + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts index 8998d91ae3b28..7d0a42a4c4ecc 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { getEsqlRule, getSimpleCustomQueryRule } from '../../../../objects/rule'; +import { getEsqlRule, getNewThreatIndicatorRule } from '../../../../objects/rule'; import { PREVIEW_LOGGED_REQUEST_DESCRIPTION, @@ -22,7 +22,6 @@ import { submitRulePreview, toggleLoggedRequestsAccordion, toggleLoggedRequestsItemAccordion, - selectIndicatorMatchType, } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; @@ -65,13 +64,12 @@ describe( describe('does not support preview logged requests', () => { beforeEach(() => { - createRule(getSimpleCustomQueryRule()).then((createdRule) => { + createRule(getNewThreatIndicatorRule()).then((createdRule) => { visitEditRulePage(createdRule.body.id); }); }); it('does not show preview logged requests checkbox fro Indicator Match rule', () => { - selectIndicatorMatchType(); cy.get(RULES_CREATION_PREVIEW_REFRESH_BUTTON).should('be.visible'); cy.get(PREVIEW_LOGGED_REQUESTS_CHECKBOX).should('not.exist'); }); From 5a0ca5a9ca4a3dd968c5025ad4b69e8354f65ca8 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:33:50 +0000 Subject: [PATCH 25/28] [Security Solution][Detection Engine]fix styles --- .../components/rule_preview/logged_requests.tsx | 5 ++++- .../rule_preview/logged_requests_item.tsx | 15 ++++++++------- .../rule_preview/logged_requests_pages.tsx | 9 +++++---- .../rule_preview/use_accordion_styling.ts | 8 +++----- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx index 2e99eca3449c6..73dd4cdca57fa 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests.tsx @@ -9,6 +9,7 @@ import type { FC } from 'react'; import React, { useMemo } from 'react'; import { EuiSpacer } from '@elastic/eui'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import { css } from '@emotion/react'; import type { RulePreviewLogs } from '../../../../../common/api/detection_engine'; import * as i18n from './translations'; @@ -47,7 +48,9 @@ const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[]; ruleType: Type }> = data-test-subj="preview-logged-requests-accordion" buttonContent={i18n.LOGGED_REQUESTS_ACCORDION_BUTTON} borders="horizontal" - css={cssStyles} + css={css` + ${cssStyles} + `} > {AccordionContent} diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx index a66e58b9280ba..d1cb6ad0e318d 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_item.tsx @@ -7,6 +7,7 @@ import type { FC, PropsWithChildren } from 'react'; import React from 'react'; +import { css } from '@emotion/react'; import { useEuiPaddingSize, EuiText, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -45,10 +46,10 @@ const LoggedRequestsItemComponent: FC } id={`ruleExecution-${startedAt}`} - css={{ - marginLeft: paddingLarge, - ...cssStyles, - }} + css={css` + margin-left: ${paddingLarge}; + ${cssStyles} + `} > {requests.length > 2 ? ( <> @@ -56,9 +57,9 @@ const LoggedRequestsItemComponent: FC {i18n.REQUESTS_SAMPLE_WARNING} diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx index c11eaef6e08c2..a7a2955f4e43b 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/logged_requests_pages.tsx @@ -7,6 +7,7 @@ import type { FC } from 'react'; import React from 'react'; +import { css } from '@emotion/react'; import { FormattedMessage } from '@kbn/i18n-react'; import { useEuiPaddingSize } from '@elastic/eui'; @@ -88,10 +89,10 @@ const LoggedRequestsPagesComponent: FC<{ /> } borders="horizontal" - css={{ - marginLeft: paddingLarge, - ...cssStyles, - }} + css={css` + margin-left: ${paddingLarge}; + ${cssStyles} + `} > {pageRequests.map((request, requestKey) => ( diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts index ee06a812d75e0..be05d90836c94 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/use_accordion_styling.ts @@ -4,15 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { useMemo } from 'react'; + import { useEuiPaddingSize } from '@elastic/eui'; export const useAccordionStyling = () => { const paddingLarge = useEuiPaddingSize('l'); const paddingSmall = useEuiPaddingSize('s'); - return useMemo( - () => ({ paddingBottom: paddingLarge, paddingTop: paddingSmall }), - [paddingLarge, paddingSmall] - ); + return `padding-bottom: ${paddingLarge}; + padding-top: ${paddingSmall};`; }; From 154ec1fb028e243837f2da922ecb978de88cd96d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:12:32 +0000 Subject: [PATCH 26/28] [Security Solution][Detection Engine] tests --- .../components/rule_preview/preview_logs.tsx | 5 ++++- .../query/trial_license_complete_tier/custom_query.ts | 2 ++ .../threshold/trial_license_complete_tier/threshold.ts | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx index a204d89e1c641..d8ef8d3cd22ae 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/preview_logs.tsx @@ -7,6 +7,7 @@ import type { FC, PropsWithChildren } from 'react'; import React, { Fragment, useMemo } from 'react'; +import { css } from '@emotion/css'; import { EuiCallOut, EuiText, EuiSpacer, EuiAccordion } from '@elastic/eui'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; @@ -112,7 +113,9 @@ const LogAccordion: FC> = ({ logs, isError, isError ? i18n.QUERY_PREVIEW_SEE_ALL_ERRORS : i18n.QUERY_PREVIEW_SEE_ALL_WARNINGS } borders="horizontal" - css={cssStyles} + css={css` + ${cssStyles} + `} > {restOfLogs.map((log, key) => ( { expect(requests).toHaveLength(1); expect(requests![0].description).toBe('Find events'); + expect(requests![0].request_type).toBe('findDocuments'); expect(requests![0].request).toContain('POST /ecs_compliant/_search?allow_no_indices=true'); }); @@ -2843,6 +2844,7 @@ export default ({ getService }: FtrProviderContext) => { expect(requests).toHaveLength(1); expect(requests![0].description).toBe('Find events'); + expect(requests![0].request_type).toBe('findDocuments'); expect(requests![0].request).toContain('POST /ecs_compliant/_search?allow_no_indices=true'); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts index 0f37077da10af..83e0d924ee3b7 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts @@ -718,6 +718,7 @@ export default ({ getService }: FtrProviderContext) => { expect(requests).toHaveLength(2); expect(requests![0].description).toBe('Find all terms that exceeds threshold value'); expect(requests![0].request).toContain('POST /auditbeat-*/_search?allow_no_indices=true'); + expect(requests![0].request_type).toBe('findThresholdBuckets'); expect(requests![1].description).toBe( 'Find all terms that exceeds threshold value after host.id: f9c7ca2d33f548a8b37667f6fffc59ce' ); From ec9707d0877de8f500934a6f5ec4bc28947c7438 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:08:54 +0000 Subject: [PATCH 27/28] [Security Solution][Detection Engine]additional test assertion --- .../threshold/trial_license_complete_tier/threshold.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts index 83e0d924ee3b7..9b455583a51f6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/threshold/trial_license_complete_tier/threshold.ts @@ -717,6 +717,11 @@ export default ({ getService }: FtrProviderContext) => { expect(requests).toHaveLength(2); expect(requests![0].description).toBe('Find all terms that exceeds threshold value'); + + const requestWithoutSpaces = requests![0].request?.replace(/\s/g, ''); + expect(requestWithoutSpaces).toContain( + `aggregations":{"thresholdTerms":{"composite":{"sources":[{"host.id":{"terms":{"field":"host.id"}}}],"size":10000},"aggs":{"max_timestamp":{"max":{"field":"@timestamp"}},"min_timestamp":{"min":{"field":"@timestamp"}},"count_check":{"bucket_selector":{"buckets_path":{"docCount":"_count"},"script":"params.docCount>=100"}}}}` + ); expect(requests![0].request).toContain('POST /auditbeat-*/_search?allow_no_indices=true'); expect(requests![0].request_type).toBe('findThresholdBuckets'); expect(requests![1].description).toBe( From dfeb094033b300950caf60e5b97432d735a8ed35 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 28 Jan 2025 10:18:21 +0000 Subject: [PATCH 28/28] [Security Solution][Detection Engine] rename argument name --- .../new_terms/create_new_terms_alert_type.ts | 6 +++--- .../rule_types/new_terms/multi_terms_composite.ts | 4 ++-- .../alert_suppression/group_and_bulk_create.ts | 2 +- .../rule_types/threshold/find_threshold_signals.ts | 4 ++-- .../lib/detection_engine/rule_types/types.ts | 2 +- .../utils/search_after_bulk_create_factory.ts | 6 +++--- .../rule_types/utils/single_search_after.ts | 14 +++++++------- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 28a65e5e0ec38..c4927ff75a0d8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -198,7 +198,7 @@ export const createNewTermsAlertType = ( primaryTimestamp, secondaryTimestamp, runtimeMappings, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findAllTerms', description: i18n.FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION( @@ -376,7 +376,7 @@ export const createNewTermsAlertType = ( pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findNewTerms', description: i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(afterKey)), @@ -427,7 +427,7 @@ export const createNewTermsAlertType = ( pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findDocuments', description: i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts index 229b4b8b9d244..7fea5857a6e44 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -157,7 +157,7 @@ const multiTermsCompositeNonRetryable = async ({ pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findNewTerms', description: i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION( @@ -207,7 +207,7 @@ const multiTermsCompositeNonRetryable = async ({ pageSize: 0, primaryTimestamp, secondaryTimestamp, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findDocuments', description: i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index babcb652e0c25..12a8e0d41fb9f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -202,7 +202,7 @@ export const groupAndBulkCreate = async ({ secondaryTimestamp: runOpts.secondaryTimestamp, runtimeMappings: runOpts.runtimeMappings, additionalFilters: bucketHistoryFilter, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findDocuments', description: i18n.FIND_EVENTS_DESCRIPTION, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts index d2d3c2f35e255..f14f22acf0a0d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/find_threshold_signals.ts @@ -117,7 +117,7 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findThresholdBuckets', description: i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(stringifyAfterKey(sortKeys)), @@ -168,7 +168,7 @@ export const findThresholdSignals = async ({ runtimeMappings, primaryTimestamp, secondaryTimestamp, - loggedRequestsEnabled: isLoggedRequestsEnabled + loggedRequestsConfig: isLoggedRequestsEnabled ? { type: 'findThresholdBuckets', description: i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 6224ae13352d2..27b14db5c1bb4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -409,7 +409,7 @@ export interface SearchAfterAndBulkCreateReturnType { loggedRequests?: RulePreviewLoggedRequest[]; } -export interface LoggedRequestsEnabled { +export interface LoggedRequestsConfig { type: string; description: string; skipRequestQuery?: boolean; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index e40f27932bbe8..700799729cb3e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -23,7 +23,7 @@ import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType, SignalSourceHit, - LoggedRequestsEnabled, + LoggedRequestsConfig, } from '../types'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { GenericBulkCreateResponse } from '../factories'; @@ -36,7 +36,7 @@ const createLoggedRequestsConfig = ( isLoggedRequestsEnabled: boolean | undefined, sortIds: estypes.SortResults | undefined, page: number -): LoggedRequestsEnabled | undefined => { +): LoggedRequestsConfig | undefined => { if (!isLoggedRequestsEnabled) { return undefined; } @@ -133,7 +133,7 @@ export const searchAfterAndBulkCreateFactory = async ({ trackTotalHits, sortOrder, additionalFilters, - loggedRequestsEnabled: createLoggedRequestsConfig( + loggedRequestsConfig: createLoggedRequestsConfig( isLoggedRequestsEnabled, sortIds, searchingIteration diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts index 689878fc7837a..96b48f385d513 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts @@ -15,7 +15,7 @@ import type { SignalSearchResponse, SignalSource, OverrideBodyQuery, - LoggedRequestsEnabled, + LoggedRequestsConfig, } from '../types'; import { buildEventsSearchQuery } from './build_events_query'; import { createErrorsFromShard, makeFloatString } from './utils'; @@ -42,7 +42,7 @@ export interface SingleSearchAfterParams { runtimeMappings: estypes.MappingRuntimeFields | undefined; additionalFilters?: estypes.QueryDslQueryContainer[]; overrideBody?: OverrideBodyQuery; - loggedRequestsEnabled?: LoggedRequestsEnabled; + loggedRequestsConfig?: LoggedRequestsConfig; } // utilize search_after for paging results into bulk. @@ -65,7 +65,7 @@ export const singleSearchAfter = async < trackTotalHits, additionalFilters, overrideBody, - loggedRequestsEnabled, + loggedRequestsConfig, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; @@ -110,13 +110,13 @@ export const singleSearchAfter = async < errors: nextSearchAfterResult._shards.failures ?? [], }); - if (loggedRequestsEnabled) { + if (loggedRequestsConfig) { loggedRequests.push({ - request: loggedRequestsEnabled.skipRequestQuery + request: loggedRequestsConfig.skipRequestQuery ? undefined : logSearchRequest(searchAfterQuery), - description: loggedRequestsEnabled.description, - request_type: loggedRequestsEnabled.type, + description: loggedRequestsConfig.description, + request_type: loggedRequestsConfig.type, duration: Math.round(end - start), }); }