From d5d3f42aa812ea834936f021e00b0df7941ea4a3 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Thu, 18 Jul 2024 10:51:37 +0200 Subject: [PATCH] [EEM] Fallback to source for metadata if no destination defined (#188515) This PR closes https://github.com/elastic/elastic-entity-model/issues/116 by ensuring that `destination` is always set when the schema is parsed along with ensuring that if for some reason desitnation is not set, we fallback in the actual metadata code as well. I also added a unit test for each of the different `metadata` formats: - String - Object with only `source` - Object with `source` and `limit` - Object with `source`, `limit`, and `destination` --------- Co-authored-by: Chris Cowan Co-authored-by: Chris Cowan Co-authored-by: Nathan L Smith --- .../kbn-entities-schema/src/schema/common.ts | 5 + .../helpers/fixtures/entity_definition.ts | 5 +- .../generate_metadata_aggregations.test.ts | 183 ++++++++++++++++++ .../generate_metadata_aggregations.ts | 2 +- 4 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index bb9d07b1957f4..df6d20ef1d44b 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -88,6 +88,11 @@ export const metadataSchema = z destination: z.optional(z.string()), limit: z.optional(z.number().default(1000)), }) + .transform((metadata) => ({ + ...metadata, + destination: metadata.destination ?? metadata.source, + limit: metadata.limit ?? 1000, + })) .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))); export const identityFieldsSchema = z diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index 137560df13385..2d6bceda8b077 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -6,7 +6,7 @@ */ import { entityDefinitionSchema } from '@kbn/entities-schema'; -export const entityDefinition = entityDefinitionSchema.parse({ +export const rawEntityDefinition = { id: 'admin-console-services', version: '999.999.999', name: 'Services for Admin Console', @@ -43,4 +43,5 @@ export const entityDefinition = entityDefinitionSchema.parse({ ], }, ], -}); +}; +export const entityDefinition = entityDefinitionSchema.parse(rawEntityDefinition); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts new file mode 100644 index 0000000000000..0ae3c4a81c870 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { entityDefinitionSchema } from '@kbn/entities-schema'; +import { rawEntityDefinition } from '../helpers/fixtures/entity_definition'; +import { + generateHistoryMetadataAggregations, + generateLatestMetadataAggregations, +} from './generate_metadata_aggregations'; + +describe('Generate Metadata Aggregations for history and latest', () => { + describe('generateHistoryMetadataAggregations()', () => { + it('should generate metadata aggregations for string format', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: ['host.name'], + }); + expect(generateHistoryMetadataAggregations(definition)).toEqual({ + 'entity.metadata.host.name': { + terms: { + field: 'host.name', + size: 1000, + }, + }, + }); + }); + + it('should generate metadata aggregations for object format with only source', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [{ source: 'host.name' }], + }); + expect(generateHistoryMetadataAggregations(definition)).toEqual({ + 'entity.metadata.host.name': { + terms: { + field: 'host.name', + size: 1000, + }, + }, + }); + }); + + it('should generate metadata aggregations for object format with source and limit', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [{ source: 'host.name', limit: 10 }], + }); + expect(generateHistoryMetadataAggregations(definition)).toEqual({ + 'entity.metadata.host.name': { + terms: { + field: 'host.name', + size: 10, + }, + }, + }); + }); + + it('should generate metadata aggregations for object format with source, limit, and destination', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }], + }); + expect(generateHistoryMetadataAggregations(definition)).toEqual({ + 'entity.metadata.hostName': { + terms: { + field: 'host.name', + size: 10, + }, + }, + }); + }); + }); + + describe('generateLatestMetadataAggregations()', () => { + it('should generate metadata aggregations for string format', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: ['host.name'], + }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ + 'entity.metadata.host.name': { + filter: { + range: { + 'event.ingested': { + gte: 'now-1m', + }, + }, + }, + aggs: { + data: { + terms: { + field: 'host.name', + size: 1000, + }, + }, + }, + }, + }); + }); + + it('should generate metadata aggregations for object format with only source', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [{ source: 'host.name' }], + }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ + 'entity.metadata.host.name': { + filter: { + range: { + 'event.ingested': { + gte: 'now-1m', + }, + }, + }, + aggs: { + data: { + terms: { + field: 'host.name', + size: 1000, + }, + }, + }, + }, + }); + }); + + it('should generate metadata aggregations for object format with source and limit', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [{ source: 'host.name', limit: 10 }], + }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ + 'entity.metadata.host.name': { + filter: { + range: { + 'event.ingested': { + gte: 'now-1m', + }, + }, + }, + aggs: { + data: { + terms: { + field: 'host.name', + size: 10, + }, + }, + }, + }, + }); + }); + + it('should generate metadata aggregations for object format with source, limit, and destination', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }], + }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ + 'entity.metadata.hostName': { + filter: { + range: { + 'event.ingested': { + gte: 'now-1m', + }, + }, + }, + aggs: { + data: { + terms: { + field: 'hostName', + size: 10, + }, + }, + }, + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 31ba3e9add0dc..809ed5f2b57b9 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -34,7 +34,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) return definition.metadata.reduce( (aggs, metadata) => ({ ...aggs, - [`entity.metadata.${metadata.destination}`]: { + [`entity.metadata.${metadata.destination ?? metadata.source}`]: { filter: { range: { 'event.ingested': {