From ab308aa2d0dfe1547f17cb1cf6ac41ecf7e3ce3e Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 17 Dec 2024 22:14:39 +1100 Subject: [PATCH] [8.x] [ECO][Inventory v2] APM changes (#202497) (#204524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.x`: - [[ECO][Inventory v2] APM changes (#202497)](https://github.com/elastic/kibana/pull/202497) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Cauê Marcondes <55978943+cauemarcondes@users.noreply.github.com> --- .../observability_solution/apm/kibana.jsonc | 1 + .../entities/entity_link/entity_link.test.tsx | 8 +- .../server/routes/entities/get_entities.ts | 67 ---- .../entities/get_entity_latest_services.ts | 59 ---- .../entities/services/get_service_entities.ts | 61 ---- .../services/get_service_entity_summary.ts | 25 +- .../server/routes/entities/services/routes.ts | 47 +-- .../apm/server/routes/entities/types.ts | 22 +- .../entities/utils/merge_entities.test.ts | 319 ++++++++---------- .../routes/entities/utils/merge_entities.ts | 27 +- .../apm/server/routes/services/route.ts | 16 +- .../apm/server/types.ts | 70 ++-- .../observability_solution/apm/tsconfig.json | 1 + .../common/entity/entity_types.ts | 1 + .../common/field_names/elasticsearch.ts | 2 + .../observability_shared/common/index.ts | 11 +- 16 files changed, 249 insertions(+), 488 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts delete mode 100644 x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_latest_services.ts delete mode 100644 x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts diff --git a/x-pack/plugins/observability_solution/apm/kibana.jsonc b/x-pack/plugins/observability_solution/apm/kibana.jsonc index bcb0801fc6394..0b8e8bd70a9a6 100644 --- a/x-pack/plugins/observability_solution/apm/kibana.jsonc +++ b/x-pack/plugins/observability_solution/apm/kibana.jsonc @@ -39,6 +39,7 @@ "uiActions", "logsDataAccess", "savedSearch", + "entityManager" ], "optionalPlugins": [ "actions", diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx index cdf6f23eb53d9..4515c2cdd5714 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx @@ -173,7 +173,9 @@ describe('Entity link', () => { renderEntityLink({ isEntityCentricExperienceEnabled: true, serviceEntitySummaryMockReturnValue: { - serviceEntitySummary: { dataStreamTypes: ['metrics'] } as unknown as ServiceEntitySummary, + serviceEntitySummary: { + ['data_stream.type']: ['metrics'], + } as unknown as ServiceEntitySummary, serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, }, hasApmDataFetcherMockReturnValue: { @@ -200,7 +202,9 @@ describe('Entity link', () => { renderEntityLink({ isEntityCentricExperienceEnabled: true, serviceEntitySummaryMockReturnValue: { - serviceEntitySummary: { dataStreamTypes: ['metrics'] } as unknown as ServiceEntitySummary, + serviceEntitySummary: { + ['data_stream.type']: ['metrics'], + } as unknown as ServiceEntitySummary, serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, }, hasApmDataFetcherMockReturnValue: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts deleted file mode 100644 index 6cedb09efa7c2..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { - ENTITY_FIRST_SEEN, - ENTITY_LAST_SEEN, -} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; -import type { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; -import { getEntityLatestServices } from './get_entity_latest_services'; -import type { EntityLatestServiceRaw } from './types'; - -export function entitiesRangeQuery(start?: number, end?: number): QueryDslQueryContainer[] { - if (!start || !end) { - return []; - } - - return [ - { - range: { - [ENTITY_LAST_SEEN]: { - gte: start, - }, - }, - }, - { - range: { - [ENTITY_FIRST_SEEN]: { - lte: end, - }, - }, - }, - ]; -} - -export async function getEntities({ - entitiesESClient, - start, - end, - environment, - kuery, - size, - serviceName, -}: { - entitiesESClient: EntitiesESClient; - start: number; - end: number; - environment: string; - kuery?: string; - size: number; - serviceName?: string; -}): Promise { - const entityLatestServices = await getEntityLatestServices({ - entitiesESClient, - start, - end, - environment, - kuery, - size, - serviceName, - }); - - return entityLatestServices; -} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_latest_services.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_latest_services.ts deleted file mode 100644 index e08be75072b6f..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_latest_services.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { kqlQuery, termQuery } from '@kbn/observability-plugin/server'; -import { - ENTITY, - ENTITY_TYPE, - SOURCE_DATA_STREAM_TYPE, -} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; -import { AGENT_NAME, SERVICE_ENVIRONMENT, SERVICE_NAME } from '../../../common/es_fields/apm'; -import { environmentQuery } from '../../../common/utils/environment_query'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; -import { entitiesRangeQuery } from './get_entities'; -import { EntityLatestServiceRaw, EntityType } from './types'; - -export async function getEntityLatestServices({ - entitiesESClient, - start, - end, - environment, - kuery, - size, - serviceName, -}: { - entitiesESClient: EntitiesESClient; - start?: number; - end?: number; - environment: string; - kuery?: string; - size: number; - serviceName?: string; -}): Promise { - const latestEntityServices = ( - await entitiesESClient.searchLatest(`get_entity_latest_services`, { - body: { - size, - track_total_hits: false, - _source: [AGENT_NAME, ENTITY, SOURCE_DATA_STREAM_TYPE, SERVICE_NAME, SERVICE_ENVIRONMENT], - query: { - bool: { - filter: [ - ...kqlQuery(kuery), - ...environmentQuery(environment, SERVICE_ENVIRONMENT), - ...entitiesRangeQuery(start, end), - ...termQuery(ENTITY_TYPE, EntityType.SERVICE), - ...termQuery(SERVICE_NAME, serviceName), - ], - }, - }, - }, - }) - ).hits.hits.map((hit) => hit._source); - - return latestEntityServices; -} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts deleted file mode 100644 index 9e6bb34bceafe..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { errors } from '@elastic/elasticsearch'; -import { Logger } from '@kbn/core/server'; -import { WrappedElasticsearchClientError } from '@kbn/observability-plugin/server'; -import { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; -import { getEntities } from '../get_entities'; -import { mergeEntities } from '../utils/merge_entities'; - -export const MAX_NUMBER_OF_SERVICES = 1_000; - -export async function getServiceEntities({ - entitiesESClient, - start, - end, - kuery, - environment, - logger, -}: { - entitiesESClient: EntitiesESClient; - start: number; - end: number; - kuery: string; - environment: string; - logger: Logger; -}) { - return withApmSpan('get_service_entities', async () => { - try { - const entities = await getEntities({ - entitiesESClient, - start, - end, - kuery, - environment, - size: MAX_NUMBER_OF_SERVICES, - }); - - return mergeEntities({ entities }); - } catch (error) { - // If the index does not exist, handle it gracefully - if ( - error instanceof WrappedElasticsearchClientError && - error.originalError instanceof errors.ResponseError - ) { - const type = error.originalError.body.error.type; - - if (type === 'index_not_found_exception') { - logger.error(`Entities index does not exist. Unable to fetch services.`); - return []; - } - } - - throw error; - } - }); -} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts index 3ab3b907f5be2..90a44a9b609e6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts @@ -5,28 +5,31 @@ * 2.0. */ -import type { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; +import { SERVICE_NAME, AGENT_NAME, SERVICE_ENVIRONMENT } from '@kbn/apm-types'; +import { BUILT_IN_ENTITY_TYPES, DATA_STREAM_TYPE } from '@kbn/observability-shared-plugin/common'; +import moment from 'moment'; +import type { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; import { withApmSpan } from '../../../utils/with_apm_span'; -import { getEntityLatestServices } from '../get_entity_latest_services'; import { mergeEntities } from '../utils/merge_entities'; -import { MAX_NUMBER_OF_SERVICES } from './get_service_entities'; interface Params { - entitiesESClient: EntitiesESClient; + entityManagerClient: EntityClient; serviceName: string; environment: string; } -export function getServiceEntitySummary({ entitiesESClient, environment, serviceName }: Params) { +export function getServiceEntitySummary({ entityManagerClient, environment, serviceName }: Params) { return withApmSpan('get_service_entity_summary', async () => { - const entityLatestServices = await getEntityLatestServices({ - entitiesESClient, - environment, - size: MAX_NUMBER_OF_SERVICES, - serviceName, + const serviceEntitySummary = await entityManagerClient.v2.searchEntities({ + start: moment().subtract(15, 'm').toISOString(), + end: moment().toISOString(), + type: BUILT_IN_ENTITY_TYPES.SERVICE_V2, + filters: [`${SERVICE_NAME}: "${serviceName}"`], + limit: 1, + metadata_fields: [DATA_STREAM_TYPE, AGENT_NAME, SERVICE_ENVIRONMENT], }); - const serviceEntity = mergeEntities({ entities: entityLatestServices }); + const serviceEntity = mergeEntities({ entities: serviceEntitySummary?.entities }); return serviceEntity[0]; }); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts index ab89f9417ec88..e3d44645ad394 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts @@ -6,10 +6,8 @@ */ import * as t from 'io-ts'; import { environmentQuery } from '../../../../common/utils/environment_query'; -import { createEntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { createApmServerRoute } from '../../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../../default_api_types'; -import { getServiceEntities } from './get_service_entities'; import { getServiceEntitySummary } from './get_service_entity_summary'; const serviceEntitiesSummaryRoute = createApmServerRoute({ @@ -20,52 +18,20 @@ const serviceEntitiesSummaryRoute = createApmServerRoute({ }), security: { authz: { requiredPrivileges: ['apm'] } }, async handler(resources) { - const { context, params, request } = resources; - const coreContext = await context.core; - - const entitiesESClient = await createEntitiesESClient({ - request, - esClient: coreContext.elasticsearch.client.asCurrentUser, - }); + const { params, request, plugins } = resources; + const entityManagerStart = await plugins.entityManager.start(); + const entityManagerClient = await entityManagerStart.getScopedClient({ request }); const { serviceName } = params.path; const { environment } = params.query; - return getServiceEntitySummary({ - entitiesESClient, + const serviceEntitySummary = await getServiceEntitySummary({ + entityManagerClient, serviceName, environment, }); - }, -}); - -const servicesEntitiesRoute = createApmServerRoute({ - endpoint: 'GET /internal/apm/entities/services', - params: t.type({ - query: t.intersection([environmentRt, kueryRt, rangeRt]), - }), - security: { authz: { requiredPrivileges: ['apm'] } }, - async handler(resources) { - const { context, params, request } = resources; - const coreContext = await context.core; - - const entitiesESClient = await createEntitiesESClient({ - request, - esClient: coreContext.elasticsearch.client.asCurrentUser, - }); - - const { start, end, kuery, environment } = params.query; - - const services = await getServiceEntities({ - entitiesESClient, - start, - end, - kuery, - environment, - logger: resources.logger, - }); - return { services }; + return serviceEntitySummary; }, }); @@ -137,7 +103,6 @@ const serviceLogErrorRateTimeseriesRoute = createApmServerRoute({ }); export const servicesEntitiesRoutesRepository = { - ...servicesEntitiesRoute, ...serviceLogRateTimeseriesRoute, ...serviceLogErrorRateTimeseriesRoute, ...serviceEntitiesSummaryRoute, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts index 833e565ec00ef..c65af1a05a26a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts @@ -4,28 +4,10 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; +import type { EntityMetadata, EntityV2 } from '@kbn/entities-schema'; export enum EntityType { SERVICE = 'service', } -export interface EntityLatestServiceRaw { - agent: { - name: AgentName[]; - }; - source_data_stream: { - type: string[]; - }; - service: { - name: string; - environment?: string; - }; - entity: Entity; -} - -interface Entity { - id: string; - last_seen_timestamp: string; - identity_fields: string[]; -} +export type EntityLatestServiceRaw = EntityV2 & EntityMetadata; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts index e3dd0ef5e0d4e..91f1eff244def 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts @@ -13,17 +13,14 @@ describe('mergeEntities', () => { it('modifies one service', () => { const entities: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - environment: 'test', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['metrics', 'logs'] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-1:test', - }, + 'data_stream.type': ['metrics', 'logs'], + 'agent.name': 'nodejs', + 'service.environment': 'test', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1', + 'entity.display_name': 'service-1', }, ]; const result = mergeEntities({ entities }); @@ -32,7 +29,7 @@ describe('mergeEntities', () => { agentName: 'nodejs' as AgentName, dataStreamTypes: ['metrics', 'logs'], environments: ['test'], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); @@ -41,56 +38,44 @@ describe('mergeEntities', () => { it('joins two service with the same name ', () => { const entities: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - environment: 'env-service-1', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['foo'] }, - entity: { - last_seen_timestamp: '2024-03-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-1:env-service-1', - }, + 'data_stream.type': ['foo'], + 'agent.name': 'nodejs', + 'service.environment': 'env-service-1', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:env-service-1', + 'entity.display_name': 'service-1', }, { - service: { - name: 'service-1', - environment: 'env-service-2', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['bar'] }, - entity: { - last_seen_timestamp: '2024-03-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'apm-only-1:synthtrace-env-2', - }, + 'data_stream.type': ['bar'], + 'agent.name': 'nodejs', + 'service.environment': 'env-service-2', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:env-service-2', + 'entity.display_name': 'service-1', }, { - service: { - name: 'service-2', - environment: 'env-service-3', - }, - agent: { name: ['java'] }, - source_data_stream: { type: ['baz'] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-2:env-service-3', - }, + 'data_stream.type': ['baz'], + 'agent.name': 'java', + 'service.environment': 'env-service-3', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-2', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-2:env-service-3', + 'entity.display_name': 'service-2', }, { - service: { - name: 'service-2', - environment: 'env-service-4', - }, - agent: { name: ['java'] }, - source_data_stream: { type: ['baz'] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-2:env-service-3', - }, + 'data_stream.type': ['baz'], + 'agent.name': ['java'], + 'service.environment': 'env-service-4', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-2', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-2:env-service-4', + 'entity.display_name': 'service-2', }, ]; @@ -100,14 +85,14 @@ describe('mergeEntities', () => { agentName: 'nodejs' as AgentName, dataStreamTypes: ['foo', 'bar'], environments: ['env-service-1', 'env-service-2'], - lastSeenTimestamp: '2024-03-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, { agentName: 'java' as AgentName, dataStreamTypes: ['baz'], environments: ['env-service-3', 'env-service-4'], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-2', }, ]); @@ -115,43 +100,34 @@ describe('mergeEntities', () => { it('handles duplicate environments and data streams', () => { const entities: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - environment: 'test', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['metrics', 'logs'] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-1:test', - }, + 'data_stream.type': ['metrics', 'logs'], + 'agent.name': ['nodejs'], + 'service.environment': 'test', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, { - service: { - name: 'service-1', - environment: 'test', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['metrics', 'logs'] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-1:test', - }, + 'data_stream.type': ['metrics', 'logs'], + 'agent.name': ['nodejs'], + 'service.environment': 'test', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, { - service: { - name: 'service-1', - environment: 'prod', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['foo'] }, - entity: { - last_seen_timestamp: '2024-23-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-1:prod', - }, + 'data_stream.type': ['foo'], + 'agent.name': ['nodejs'], + 'service.environment': 'prod', + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:prod', + 'entity.display_name': 'service-1', }, ]; const result = mergeEntities({ entities }); @@ -160,7 +136,7 @@ describe('mergeEntities', () => { agentName: 'nodejs' as AgentName, dataStreamTypes: ['metrics', 'logs', 'foo'], environments: ['test', 'prod'], - lastSeenTimestamp: '2024-23-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); @@ -168,17 +144,14 @@ describe('mergeEntities', () => { it('handles null environment', () => { const entity: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - environment: undefined, - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: [] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name'], - id: 'service-1:test', - }, + 'data_stream.type': [], + 'agent.name': ['nodejs'], + 'service.environment': null, + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, ]; const entityResult = mergeEntities({ entities: entity }); @@ -187,35 +160,31 @@ describe('mergeEntities', () => { agentName: 'nodejs' as AgentName, dataStreamTypes: [], environments: [], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); const entities: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: [] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name'], - id: 'service-1:test', - }, + 'data_stream.type': [], + 'agent.name': ['nodejs'], + 'service.environment': null, + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, { - service: { - name: 'service-1', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: [] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name'], - id: 'service-1:test', - }, + 'data_stream.type': [], + 'agent.name': ['nodejs'], + 'service.environment': null, + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, ]; const result = mergeEntities({ entities }); @@ -224,7 +193,7 @@ describe('mergeEntities', () => { agentName: 'nodejs' as AgentName, dataStreamTypes: [], environments: [], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); @@ -233,16 +202,13 @@ describe('mergeEntities', () => { it('handles undefined environment', () => { const entity: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: [] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name'], - id: 'service-1:test', - }, + 'data_stream.type': [], + 'agent.name': ['nodejs'], + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, ]; const entityResult = mergeEntities({ entities: entity }); @@ -251,35 +217,29 @@ describe('mergeEntities', () => { agentName: 'nodejs', dataStreamTypes: [], environments: [], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); const entities: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: [] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name'], - id: 'service-1:test', - }, + 'data_stream.type': [], + 'agent.name': ['nodejs'], + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, { - service: { - name: 'service-1', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: [] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name'], - id: 'service-1:test', - }, + 'data_stream.type': [], + 'agent.name': ['nodejs'], + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, ]; const result = mergeEntities({ entities }); @@ -288,7 +248,7 @@ describe('mergeEntities', () => { agentName: 'nodejs', dataStreamTypes: [], environments: [], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); @@ -297,17 +257,14 @@ describe('mergeEntities', () => { it('has no logs when log rate is not returned', () => { const entities: EntityLatestServiceRaw[] = [ { - service: { - name: 'service-1', - environment: 'test', - }, - agent: { name: ['nodejs'] }, - source_data_stream: { type: ['metrics'] }, - entity: { - last_seen_timestamp: '2024-06-05T10:34:40.810Z', - identity_fields: ['service.name', 'service.environment'], - id: 'service-1:test', - }, + 'data_stream.type': ['metrics'], + 'agent.name': ['nodejs'], + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'service.environment': 'test', + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', }, ]; const result = mergeEntities({ entities }); @@ -316,7 +273,31 @@ describe('mergeEntities', () => { agentName: 'nodejs' as AgentName, dataStreamTypes: ['metrics'], environments: ['test'], - lastSeenTimestamp: '2024-06-05T10:34:40.810Z', + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', + serviceName: 'service-1', + }, + ]); + }); + it('has multiple duplicate environments and data stream types', () => { + const entities: EntityLatestServiceRaw[] = [ + { + 'data_stream.type': ['metrics', 'metrics', 'logs', 'logs'], + 'agent.name': ['nodejs', 'nodejs'], + 'entity.last_seen_timestamp': '2024-12-13T14:52:35.461Z', + 'service.name': 'service-1', + 'service.environment': ['test', 'test', 'test'], + 'entity.type': 'built_in_services_from_ecs_data', + 'entity.id': 'service-1:test', + 'entity.display_name': 'service-1', + }, + ]; + const result = mergeEntities({ entities }); + expect(result).toEqual([ + { + agentName: 'nodejs' as AgentName, + dataStreamTypes: ['metrics', 'logs'], + environments: ['test'], + lastSeenTimestamp: '2024-12-13T14:52:35.461Z', serviceName: 'service-1', }, ]); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts index 2f33c4728bd1a..1e95656cb1f8e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts @@ -23,7 +23,7 @@ export function mergeEntities({ entities: EntityLatestServiceRaw[]; }): MergedServiceEntity[] { const mergedEntities = entities.reduce((map, current) => { - const key = current.service.name; + const key = current['service.name']; if (map.has(key)) { const existingEntity = map.get(key); map.set(key, mergeFunc(current, existingEntity)); @@ -33,28 +33,37 @@ export function mergeEntities({ return map; }, new Map()); - return [...mergedEntities.values()]; + return [...new Set(mergedEntities.values())]; } function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServiceEntity) { const commonEntityFields = { - serviceName: entity.service.name, - agentName: entity.agent.name[0], - lastSeenTimestamp: entity.entity.last_seen_timestamp, + serviceName: entity['service.name'], + agentName: + Array.isArray(entity['agent.name']) && entity['agent.name'].length > 0 + ? entity['agent.name'][0] + : entity['agent.name'], + lastSeenTimestamp: entity['entity.last_seen_timestamp'], }; if (!existingEntity) { return { ...commonEntityFields, - dataStreamTypes: entity.source_data_stream.type, - environments: compact([entity?.service.environment]), + dataStreamTypes: uniq(entity['data_stream.type']), + environments: uniq( + compact( + Array.isArray(entity['service.environment']) + ? entity['service.environment'] + : [entity['service.environment']] + ) + ), }; } return { ...commonEntityFields, dataStreamTypes: uniq( - compact([...(existingEntity?.dataStreamTypes ?? []), ...entity.source_data_stream.type]) + compact([...(existingEntity?.dataStreamTypes ?? []), ...entity['data_stream.type']]) ), - environments: uniq(compact([...existingEntity?.environments, entity?.service.environment])), + environments: uniq(compact([...existingEntity?.environments, entity['service.environment']])), }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts index d6c3b5b73e3d8..71d570d2708f7 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts @@ -80,7 +80,6 @@ import { import { getThroughput, ServiceThroughputResponse } from './get_throughput'; import { getServiceEntitySummary } from '../entities/services/get_service_entity_summary'; import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; -import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; const servicesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/services', @@ -297,16 +296,11 @@ const serviceAgentRoute = createApmServerRoute({ }), security: { authz: { requiredPrivileges: ['apm'] } }, handler: async (resources): Promise => { - const { context, request } = resources; - const coreContext = await context.core; + const { request, plugins } = resources; + const entityManagerStart = await plugins.entityManager.start(); - const [apmEventClient, entitiesESClient] = await Promise.all([ - getApmEventClient(resources), - createEntitiesESClient({ - request, - esClient: coreContext.elasticsearch.client.asCurrentUser, - }), - ]); + const apmEventClient = await getApmEventClient(resources); + const entityManagerClient = await entityManagerStart.getScopedClient({ request }); const { params } = resources; const { serviceName } = params.path; const { start, end } = params.query; @@ -320,7 +314,7 @@ const serviceAgentRoute = createApmServerRoute({ }), getServiceEntitySummary({ serviceName, - entitiesESClient, + entityManagerClient, environment: ENVIRONMENT_ALL.value, }), ]); diff --git a/x-pack/plugins/observability_solution/apm/server/types.ts b/x-pack/plugins/observability_solution/apm/server/types.ts index e99860b9d441f..ba1d17d6af6b9 100644 --- a/x-pack/plugins/observability_solution/apm/server/types.ts +++ b/x-pack/plugins/observability_solution/apm/server/types.ts @@ -5,52 +5,50 @@ * 2.0. */ -import { SharePluginSetup } from '@kbn/share-plugin/server'; -import { Observable } from 'rxjs'; -import { - RuleRegistryPluginSetupContract, - RuleRegistryPluginStartContract, -} from '@kbn/rule-registry-plugin/server'; -import { - PluginSetup as DataPluginSetup, - PluginStart as DataPluginStart, -} from '@kbn/data-plugin/server'; -import { +import type { ApmDataAccessPluginSetup, ApmDataAccessPluginStart, } from '@kbn/apm-data-access-plugin/server'; - -import { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server'; +import type { + PluginSetup as DataPluginSetup, + PluginStart as DataPluginStart, +} from '@kbn/data-plugin/server'; +import type { + RuleRegistryPluginSetupContract, + RuleRegistryPluginStartContract, +} from '@kbn/rule-registry-plugin/server'; +import type { SharePluginSetup } from '@kbn/share-plugin/server'; +import type { Observable } from 'rxjs'; +import type { ActionsPlugin } from '@kbn/actions-plugin/server'; +import type { CloudSetup } from '@kbn/cloud-plugin/server'; +import type { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; +import type { FeaturesPluginSetup, FeaturesPluginStart } from '@kbn/features-plugin/server'; import { HomeServerPluginSetup, HomeServerPluginStart } from '@kbn/home-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; -import { ActionsPlugin } from '@kbn/actions-plugin/server'; import type { AlertingServerSetup, AlertingServerStart } from '@kbn/alerting-plugin/server'; -import { CloudSetup } from '@kbn/cloud-plugin/server'; -import { FeaturesPluginSetup, FeaturesPluginStart } from '@kbn/features-plugin/server'; import { LicensingPluginSetup, LicensingPluginStart } from '@kbn/licensing-plugin/server'; import { MlPluginSetup, MlPluginStart } from '@kbn/ml-plugin/server'; import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; -import { - TaskManagerSetupContract, - TaskManagerStartContract, -} from '@kbn/task-manager-plugin/server'; import { FleetSetupContract as FleetPluginSetup, FleetStartContract as FleetPluginStart, } from '@kbn/fleet-plugin/server'; -import { MetricsDataPluginSetup } from '@kbn/metrics-data-access-plugin/server'; -import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; - -import { +import type { MetricsDataPluginSetup } from '@kbn/metrics-data-access-plugin/server'; +import type { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { CustomIntegrationsPluginSetup, CustomIntegrationsPluginStart, } from '@kbn/custom-integrations-plugin/server'; -import { - ProfilingDataAccessPluginSetup, - ProfilingDataAccessPluginStart, -} from '@kbn/profiling-data-access-plugin/server'; -import { +import type { + EntityManagerServerPluginSetup, + EntityManagerServerPluginStart, +} from '@kbn/entityManager-plugin/server'; +import type { LogsDataAccessPluginSetup, LogsDataAccessPluginStart, } from '@kbn/logs-data-access-plugin/server'; @@ -58,6 +56,10 @@ import type { ObservabilityAIAssistantServerSetup, ObservabilityAIAssistantServerStart, } from '@kbn/observability-ai-assistant-plugin/server'; +import type { + ProfilingDataAccessPluginSetup, + ProfilingDataAccessPluginStart, +} from '@kbn/profiling-data-access-plugin/server'; import { APMConfig } from '.'; export interface APMPluginSetup { @@ -75,8 +77,10 @@ export interface APMPluginSetupDependencies { metricsDataAccess: MetricsDataPluginSetup; dataViews: {}; share: SharePluginSetup; - observabilityAIAssistant?: ObservabilityAIAssistantServerSetup; + logsDataAccess: LogsDataAccessPluginSetup; + entityManager: EntityManagerServerPluginSetup; // optional dependencies + observabilityAIAssistant?: ObservabilityAIAssistantServerSetup; actions?: ActionsPlugin['setup']; alerting?: AlertingServerSetup; cloud?: CloudSetup; @@ -89,7 +93,6 @@ export interface APMPluginSetupDependencies { usageCollection?: UsageCollectionSetup; customIntegrations?: CustomIntegrationsPluginSetup; profilingDataAccess?: ProfilingDataAccessPluginSetup; - logsDataAccess: LogsDataAccessPluginSetup; } export interface APMPluginStartDependencies { // required dependencies @@ -102,8 +105,10 @@ export interface APMPluginStartDependencies { metricsDataAccess: MetricsDataPluginSetup; dataViews: DataViewsServerPluginStart; share: undefined; - observabilityAIAssistant?: ObservabilityAIAssistantServerStart; + logsDataAccess: LogsDataAccessPluginStart; + entityManager: EntityManagerServerPluginStart; // optional dependencies + observabilityAIAssistant?: ObservabilityAIAssistantServerStart; actions?: ActionsPlugin['start']; alerting?: AlertingServerStart; cloud?: undefined; @@ -116,5 +121,4 @@ export interface APMPluginStartDependencies { usageCollection?: undefined; customIntegrations?: CustomIntegrationsPluginStart; profilingDataAccess?: ProfilingDataAccessPluginStart; - logsDataAccess: LogsDataAccessPluginStart; } diff --git a/x-pack/plugins/observability_solution/apm/tsconfig.json b/x-pack/plugins/observability_solution/apm/tsconfig.json index b2fda13c3f76f..82dd827086033 100644 --- a/x-pack/plugins/observability_solution/apm/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm/tsconfig.json @@ -129,6 +129,7 @@ "@kbn/alerting-comparators", "@kbn/saved-search-component", "@kbn/saved-search-plugin", + "@kbn/entityManager-plugin", ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts b/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts index db3e91fbf493a..222780a1fc31a 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/entity/entity_types.ts @@ -14,6 +14,7 @@ export const BUILT_IN_ENTITY_TYPES = { HOST: 'host', CONTAINER: 'container', SERVICE: 'service', + SERVICE_V2: 'built_in_services_from_ecs_data', KUBERNETES: { CLUSTER: createKubernetesEntity('cluster'), CONTAINER: createKubernetesEntity('container'), diff --git a/x-pack/plugins/observability_solution/observability_shared/common/field_names/elasticsearch.ts b/x-pack/plugins/observability_solution/observability_shared/common/field_names/elasticsearch.ts index e703cd487259c..5569dac69b8be 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/field_names/elasticsearch.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/field_names/elasticsearch.ts @@ -146,6 +146,8 @@ export const PROFILE_ALLOC_SPACE = 'profile.alloc_space.bytes'; export const PROFILE_INUSE_OBJECTS = 'profile.inuse_objects.count'; export const PROFILE_INUSE_SPACE = 'profile.inuse_space.bytes'; +export const DATA_STREAM_TYPE = 'data_stream.type'; + export const ENTITY = 'entity'; export const ENTITY_ID = 'entity.id'; export const ENTITY_TYPE = 'entity.type'; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/index.ts index a8e26366ab4b3..24d12362d7cfa 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/index.ts @@ -128,15 +128,16 @@ export { PROFILE_ALLOC_SPACE, PROFILE_INUSE_OBJECTS, PROFILE_INUSE_SPACE, + DATA_STREAM_TYPE, ENTITY, - ENTITY_DEFINITION_ID, - ENTITY_DISPLAY_NAME, - ENTITY_FIRST_SEEN, ENTITY_ID, - ENTITY_LAST_SEEN, ENTITY_TYPE, - SOURCE_DATA_STREAM_TYPE, + ENTITY_LAST_SEEN, + ENTITY_FIRST_SEEN, + ENTITY_DISPLAY_NAME, + ENTITY_DEFINITION_ID, ENTITY_IDENTITY_FIELDS, + SOURCE_DATA_STREAM_TYPE, } from './field_names/elasticsearch'; export {