From ed85aa5d61024034285624714bd11587118e6d75 Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Fri, 6 Dec 2024 12:02:28 +0100 Subject: [PATCH] [eem] _search accepts kql filters (#203089) ## Summary `searchEntities` now accepts kql filters instead of esql and translates that to dsl filters at the query level (cherry picked from commit 5470fb71339bb76661ad0bd2dfe6cf617a03a0dc) --- .../server/lib/v2/entity_client.ts | 7 ++- .../server/lib/v2/queries/index.test.ts | 57 ++++++++++++++++++- .../server/lib/v2/queries/index.ts | 17 +++--- .../server/lib/v2/run_esql_query.ts | 11 +++- .../public/pages/overview/index.tsx | 2 +- 5 files changed, 76 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/v2/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/v2/entity_client.ts index 9eb2127ddc818..bb40fc2849a46 100644 --- a/x-pack/plugins/entity_manager/server/lib/v2/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/v2/entity_client.ts @@ -98,7 +98,7 @@ export class EntityClient { ); } - const query = getEntityInstancesQuery({ + const { query, filter } = getEntityInstancesQuery({ source: { ...source, metadata_fields: availableMetadataFields, @@ -109,10 +109,13 @@ export class EntityClient { sort, limit, }); - this.options.logger.debug(`Entity query: ${query}`); + this.options.logger.debug( + () => `Entity query: ${query}\nfilter: ${JSON.stringify(filter, null, 2)}` + ); const rawEntities = await runESQLQuery('resolve entities', { query, + filter, esClient: this.options.clusterClient.asCurrentUser, logger: this.options.logger, }); diff --git a/x-pack/plugins/entity_manager/server/lib/v2/queries/index.test.ts b/x-pack/plugins/entity_manager/server/lib/v2/queries/index.test.ts index e77be7d4172ca..9bc475d031923 100644 --- a/x-pack/plugins/entity_manager/server/lib/v2/queries/index.test.ts +++ b/x-pack/plugins/entity_manager/server/lib/v2/queries/index.test.ts @@ -10,7 +10,7 @@ import { getEntityInstancesQuery } from '.'; describe('getEntityInstancesQuery', () => { describe('getEntityInstancesQuery', () => { it('generates a valid esql query', () => { - const query = getEntityInstancesQuery({ + const { query, filter } = getEntityInstancesQuery({ source: { id: 'service_source', type_id: 'service', @@ -29,14 +29,65 @@ describe('getEntityInstancesQuery', () => { expect(query).toEqual( 'FROM logs-*, metrics-* | ' + - 'WHERE service.name::keyword IS NOT NULL | ' + - 'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z" AND custom_timestamp_field <= "2024-11-20T20:00:00.000Z" | ' + 'STATS host.name = VALUES(host.name::keyword), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id::keyword) BY service.name::keyword | ' + 'RENAME `service.name::keyword` AS service.name | ' + 'EVAL entity.type = "service", entity.id = service.name, entity.display_name = COALESCE(service.id, entity.id) | ' + 'SORT entity.id DESC | ' + 'LIMIT 5' ); + + expect(filter).toEqual({ + bool: { + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'service.name', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + custom_timestamp_field: { + gte: '2024-11-20T19:00:00.000Z', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + custom_timestamp_field: { + lte: '2024-11-20T20:00:00.000Z', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + ], + }, + }); }); }); }); diff --git a/x-pack/plugins/entity_manager/server/lib/v2/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/v2/queries/index.ts index 43c73fe7debad..5ce7a54eb1d1c 100644 --- a/x-pack/plugins/entity_manager/server/lib/v2/queries/index.ts +++ b/x-pack/plugins/entity_manager/server/lib/v2/queries/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; import { asKeyword } from './utils'; import { EntitySourceDefinition, SortBy } from '../types'; @@ -21,7 +22,7 @@ const sourceCommand = ({ source }: { source: EntitySourceDefinition }) => { return query; }; -const whereCommand = ({ +const dslFilter = ({ source, start, end, @@ -30,10 +31,7 @@ const whereCommand = ({ start: string; end: string; }) => { - const filters = [ - source.identity_fields.map((field) => `${asKeyword(field)} IS NOT NULL`).join(' AND '), - ...source.filters, - ]; + const filters = [...source.filters, ...source.identity_fields.map((field) => `${field}: *`)]; if (source.timestamp_field) { filters.push( @@ -41,7 +39,8 @@ const whereCommand = ({ ); } - return filters.map((filter) => `WHERE ${filter}`).join(' | '); + const kuery = filters.map((filter) => '(' + filter + ')').join(' AND '); + return toElasticsearchQuery(fromKueryExpression(kuery)); }; const statsCommand = ({ source }: { source: EntitySourceDefinition }) => { @@ -108,16 +107,16 @@ export function getEntityInstancesQuery({ start: string; end: string; sort?: SortBy; -}): string { +}) { const commands = [ sourceCommand({ source }), - whereCommand({ source, start, end }), statsCommand({ source }), renameCommand({ source }), evalCommand({ source }), sortCommand({ source, sort }), `LIMIT ${limit}`, ]; + const filter = dslFilter({ source, start, end }); - return commands.join(' | '); + return { query: commands.join(' | '), filter }; } diff --git a/x-pack/plugins/entity_manager/server/lib/v2/run_esql_query.ts b/x-pack/plugins/entity_manager/server/lib/v2/run_esql_query.ts index eda36a007ffe6..ccccacd0174df 100644 --- a/x-pack/plugins/entity_manager/server/lib/v2/run_esql_query.ts +++ b/x-pack/plugins/entity_manager/server/lib/v2/run_esql_query.ts @@ -7,6 +7,7 @@ import { withSpan } from '@kbn/apm-utils'; import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { ESQLColumn, ESQLRow, ESQLSearchResponse } from '@kbn/es-types'; export interface SourceAs { @@ -19,19 +20,24 @@ export async function runESQLQuery( esClient, logger, query, + filter, }: { esClient: ElasticsearchClient; logger: Logger; query: string; + filter?: QueryDslQueryContainer; } ): Promise { - logger.trace(() => `Request (${operationName}):\n${query}`); + logger.trace( + () => `Request (${operationName}):\nquery: ${query}\nfilter: ${JSON.stringify(filter, null, 2)}` + ); return withSpan( { name: operationName, labels: { plugin: '@kbn/entityManager-plugin' } }, async () => esClient.esql.query( { query, + filter, format: 'json', }, { querystring: { drop_null_columns: true } } @@ -62,8 +68,7 @@ function rowToObject(row: ESQLRow, columns: ESQLColumn[]) { return object; } - // Removes the type suffix from the column name - const name = column.name.replace(/\.(text|keyword)$/, ''); + const name = column.name; if (!object[name]) { object[name] = value; } diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx index d628ab306a1b1..e3d634557d4fb 100644 --- a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx @@ -70,7 +70,7 @@ function EntitySourceForm({ - +