Skip to content

Commit

Permalink
[8.x] [EEM] Add create and read APIs for v2 type and source definitio…
Browse files Browse the repository at this point in the history 
…ns (elastic#201470) (elastic#202928)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[EEM] Add create and read APIs for v2 type and source definitions
(elastic#201470)](elastic#201470)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Milton
Hultgren","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-12-04T13:53:57Z","message":"[EEM]
Add create and read APIs for v2 type and source definitions
(elastic#201470)\n\n## Summary\r\n\r\nThis PR adds:\r\n* Some basic features
and privileges for the EEM app\r\n* A function that sets up an index
with a template for the new\r\ndefinitions\r\n* 4 API endpoints to read
and create entity types and sources\r\n * `POST
/internal/entities/v2/definitions/types`\r\n * `GET
/internal/entities/v2/definitions/types`\r\n * `POST
/internal/entities/v2/definitions/sources`\r\n * `GET
/internal/entities/v2/definitions/sources`\r\n* Some v2 shuffling around
of code\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Maxim Palenov
<[email protected]>","sha":"26de7a85ba8dfcfca082a463a67de58ba376ce13","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:all-open","ci:project-deploy-observability","Feature:EEM"],"title":"[EEM]
Add create and read APIs for v2 type and source
definitions","number":201470,"url":"https://github.com/elastic/kibana/pull/201470","mergeCommit":{"message":"[EEM]
Add create and read APIs for v2 type and source definitions
(elastic#201470)\n\n## Summary\r\n\r\nThis PR adds:\r\n* Some basic features
and privileges for the EEM app\r\n* A function that sets up an index
with a template for the new\r\ndefinitions\r\n* 4 API endpoints to read
and create entity types and sources\r\n * `POST
/internal/entities/v2/definitions/types`\r\n * `GET
/internal/entities/v2/definitions/types`\r\n * `POST
/internal/entities/v2/definitions/sources`\r\n * `GET
/internal/entities/v2/definitions/sources`\r\n* Some v2 shuffling around
of code\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Maxim Palenov
<[email protected]>","sha":"26de7a85ba8dfcfca082a463a67de58ba376ce13"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/201470","number":201470,"mergeCommit":{"message":"[EEM]
Add create and read APIs for v2 type and source definitions
(elastic#201470)\n\n## Summary\r\n\r\nThis PR adds:\r\n* Some basic features
and privileges for the EEM app\r\n* A function that sets up an index
with a template for the new\r\ndefinitions\r\n* 4 API endpoints to read
and create entity types and sources\r\n * `POST
/internal/entities/v2/definitions/types`\r\n * `GET
/internal/entities/v2/definitions/types`\r\n * `POST
/internal/entities/v2/definitions/sources`\r\n * `GET
/internal/entities/v2/definitions/sources`\r\n* Some v2 shuffling around
of code\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Maxim Palenov
<[email protected]>","sha":"26de7a85ba8dfcfca082a463a67de58ba376ce13"}}]}]
BACKPORT-->

Co-authored-by: Milton Hultgren <[email protected]>
  • Loading branch information
@kibanamachine @miltonhultgren
kibanamachine and miltonhultgren authored Dec 5, 2024
1 parent 8aa7419 commit c9e60df
Show file tree
Hide file tree
Showing 32 changed files with 893 additions and 336 deletions.
3 changes: 2 additions & 1 deletion x-pack/plugins/entity_manager/kibana.jsonc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
"requiredPlugins": [
"security",
"encryptedSavedObjects",
"licensing"
"licensing",
"features"
],
"requiredBundles": []
}
Expand Down
8 changes: 8 additions & 0 deletions x-pack/plugins/entity_manager/server/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ export type {
};
export { config };

export {
CREATE_ENTITY_TYPE_DEFINITION_PRIVILEGE,
CREATE_ENTITY_SOURCE_DEFINITION_PRIVILEGE,
READ_ENTITY_TYPE_DEFINITION_PRIVILEGE,
READ_ENTITY_SOURCE_DEFINITION_PRIVILEGE,
READ_ENTITIES_PRIVILEGE,
} from './lib/v2/constants';

export const plugin = async (context: PluginInitializerContext<EntityManagerConfig>) => {
const { EntityManagerServerPlugin } = await import('./plugin');
return new EntityManagerServerPlugin(context);
Expand Down
29 changes: 0 additions & 29 deletions x-pack/plugins/entity_manager/server/lib/client/index.ts
View file

This file was deleted.

190 changes: 40 additions & 150 deletions x-pack/plugins/entity_manager/server/lib/entity_client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
* 2.0.
*/

import { without } from 'lodash';
import { EntityV2, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
import { Logger } from '@kbn/logging';
import {
installEntityDefinition,
Expand All @@ -19,40 +18,24 @@ import { startTransforms } from './entities/start_transforms';
import { findEntityDefinitionById, findEntityDefinitions } from './entities/find_entity_definition';
import { uninstallEntityDefinition } from './entities/uninstall_entity_definition';
import { EntityDefinitionNotFound } from './entities/errors/entity_not_found';

import { stopTransforms } from './entities/stop_transforms';
import { deleteIndices } from './entities/delete_index';
import { EntityDefinitionWithState } from './entities/types';
import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
import { EntitySource, SortBy, getEntityInstancesQuery } from './queries';
import { mergeEntitiesList, runESQLQuery } from './queries/utils';
import { UnknownEntityType } from './entities/errors/unknown_entity_type';

interface SearchCommon {
start: string;
end: string;
sort?: SortBy;
metadataFields?: string[];
filters?: string[];
limit?: number;
}

export type SearchByType = SearchCommon & {
type: string;
};

export type SearchBySources = SearchCommon & {
sources: EntitySource[];
};
import { EntityClient as EntityClient_v2 } from './v2/entity_client';

export class EntityClient {
public v2: EntityClient_v2;

constructor(
private options: {
esClient: ElasticsearchClient;
clusterClient: IScopedClusterClient;
soClient: SavedObjectsClientContract;
logger: Logger;
}
) {}
) {
this.v2 = new EntityClient_v2(options);
}

async createEntityDefinition({
definition,
Expand All @@ -66,13 +49,17 @@ export class EntityClient {
);
const installedDefinition = await installEntityDefinition({
definition,
esClient: this.options.esClient,
esClient: this.options.clusterClient.asCurrentUser,
soClient: this.options.soClient,
logger: this.options.logger,
});

if (!installOnly) {
await startTransforms(this.options.esClient, installedDefinition, this.options.logger);
await startTransforms(
this.options.clusterClient.asCurrentUser,
installedDefinition,
this.options.logger
);
}

return installedDefinition;
Expand All @@ -88,7 +75,7 @@ export class EntityClient {
const definition = await findEntityDefinitionById({
id,
soClient: this.options.soClient,
esClient: this.options.esClient,
esClient: this.options.clusterClient.asCurrentUser,
includeState: true,
});

Expand All @@ -115,20 +102,24 @@ export class EntityClient {
definition,
definitionUpdate,
soClient: this.options.soClient,
esClient: this.options.esClient,
esClient: this.options.clusterClient.asCurrentUser,
logger: this.options.logger,
});

if (shouldRestartTransforms) {
await startTransforms(this.options.esClient, updatedDefinition, this.options.logger);
await startTransforms(
this.options.clusterClient.asCurrentUser,
updatedDefinition,
this.options.logger
);
}
return updatedDefinition;
}

async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) {
const definition = await findEntityDefinitionById({
id,
esClient: this.options.esClient,
esClient: this.options.clusterClient.asCurrentUser,
soClient: this.options.soClient,
});

Expand All @@ -141,13 +132,17 @@ export class EntityClient {
);
await uninstallEntityDefinition({
definition,
esClient: this.options.esClient,
esClient: this.options.clusterClient.asCurrentUser,
soClient: this.options.soClient,
logger: this.options.logger,
});

if (deleteData) {
await deleteIndices(this.options.esClient, definition, this.options.logger);
await deleteIndices(
this.options.clusterClient.asCurrentUser,
definition,
this.options.logger
);
}
}

Expand All @@ -167,7 +162,7 @@ export class EntityClient {
builtIn?: boolean;
}) {
const definitions = await findEntityDefinitions({
esClient: this.options.esClient,
esClient: this.options.clusterClient.asCurrentUser,
soClient: this.options.soClient,
page,
perPage,
Expand All @@ -182,124 +177,19 @@ export class EntityClient {

async startEntityDefinition(definition: EntityDefinition) {
this.options.logger.info(`Starting transforms for definition [${definition.id}]`);
return startTransforms(this.options.esClient, definition, this.options.logger);
return startTransforms(
this.options.clusterClient.asCurrentUser,
definition,
this.options.logger
);
}

async stopEntityDefinition(definition: EntityDefinition) {
this.options.logger.info(`Stopping transforms for definition [${definition.id}]`);
return stopTransforms(this.options.esClient, definition, this.options.logger);
}

async getEntitySources({ type }: { type: string }) {
const result = await this.options.esClient.search<EntitySource>({
index: 'kibana_entity_definitions',
query: {
bool: {
must: {
term: { entity_type: type },
},
},
},
});

return result.hits.hits.map((hit) => hit._source) as EntitySource[];
}

async searchEntities({
type,
start,
end,
sort,
metadataFields = [],
filters = [],
limit = 10,
}: SearchByType) {
const sources = await this.getEntitySources({ type });
if (sources.length === 0) {
throw new UnknownEntityType(`No sources found for entity type [${type}]`);
}

return this.searchEntitiesBySources({
sources,
start,
end,
metadataFields,
filters,
sort,
limit,
});
}

async searchEntitiesBySources({
sources,
start,
end,
sort,
metadataFields = [],
filters = [],
limit = 10,
}: SearchBySources) {
const entities = await Promise.all(
sources.map(async (source) => {
const mandatoryFields = [
...source.identity_fields,
...(source.timestamp_field ? [source.timestamp_field] : []),
...(source.display_name ? [source.display_name] : []),
];
const metaFields = [...metadataFields, ...source.metadata_fields];

// operations on an unmapped field result in a failing query so we verify
// field capabilities beforehand
const { fields } = await this.options.esClient.fieldCaps({
index: source.index_patterns,
fields: [...mandatoryFields, ...metaFields],
});

const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
if (!sourceHasMandatoryFields) {
// we can't build entities without id fields so we ignore the source.
// TODO filters should likely behave similarly. we should also throw
const missingFields = mandatoryFields.filter((field) => !fields[field]);
this.options.logger.info(
`Ignoring source for type [${source.type}] with index_patterns [${
source.index_patterns
}] because some mandatory fields [${missingFields.join(', ')}] are not mapped`
);
return [];
}

// but metadata field not being available is fine
const availableMetadataFields = metaFields.filter((field) => fields[field]);
if (availableMetadataFields.length < metaFields.length) {
this.options.logger.info(
`Ignoring unmapped fields [${without(metaFields, ...availableMetadataFields).join(
', '
)}]`
);
}

const query = getEntityInstancesQuery({
source: {
...source,
metadata_fields: availableMetadataFields,
filters: [...source.filters, ...filters],
},
start,
end,
sort,
limit,
});
this.options.logger.debug(`Entity query: ${query}`);

const rawEntities = await runESQLQuery<EntityV2>({
query,
esClient: this.options.esClient,
});

return rawEntities;
})
).then((results) => results.flat());

return mergeEntitiesList(sources, entities).slice(0, limit);
return stopTransforms(
this.options.clusterClient.asCurrentUser,
definition,
this.options.logger
);
}
}
19 changes: 19 additions & 0 deletions x-pack/plugins/entity_manager/server/lib/v2/constants.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

// Definitions index

export const DEFINITIONS_ALIAS = '.kibana-entities-definitions';
export const TEMPLATE_VERSION = 1;

// Privileges

export const CREATE_ENTITY_TYPE_DEFINITION_PRIVILEGE = 'create_entity_type_definition';
export const CREATE_ENTITY_SOURCE_DEFINITION_PRIVILEGE = 'create_entity_source_definition';
export const READ_ENTITY_TYPE_DEFINITION_PRIVILEGE = 'read_entity_type_definition';
export const READ_ENTITY_SOURCE_DEFINITION_PRIVILEGE = 'read_entity_source_definition';
export const READ_ENTITIES_PRIVILEGE = 'read_entities';
Loading

0 comments on commit c9e60df

Please sign in to comment.