diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index fca2bcc83f3ab..9a14031d4cc28 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -42,3 +42,6 @@ export const MASKED_FIELD_PLACEHOLDER = ''; export const UNKOWN_FIELD_PLACEHOLDER = ''; export const KNOWN_TYPES: DataStreamType[] = ['logs', 'metrics', 'traces', 'synthetics']; + +export const ALL_PATTERNS_SELECTOR = '::*'; +export const FAILURE_STORE_SELECTOR = '::failures'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts index 5cf347ed09304..08944df9efcdc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts @@ -40,7 +40,6 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => { }; export const extractIndexNameFromBackingIndex = (indexString: string): string => { - // TODO: Undo this change once ::failures is supported const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; const match = indexString.match(pattern); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts index 4e76999318f57..a278d999e731f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts @@ -9,6 +9,7 @@ import { i18n } from '@kbn/i18n'; import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public'; import { v4 as uuidv4 } from 'uuid'; +import { ALL_PATTERNS_SELECTOR, FAILURE_STORE_SELECTOR } from '../../../../../../common/constants'; import { flyoutFailedDocsTrendText, flyoutFailedDocsPercentageText, @@ -24,6 +25,7 @@ enum DatasetQualityLensColumn { } const MAX_BREAKDOWN_SERIES = 5; +const FAILED_DOCS_QUERY = `_index: "${FAILURE_STORE_SELECTOR}"`; interface GetLensAttributesParams { color: string; @@ -130,7 +132,6 @@ export function getLensAttributes({ } function getAdHocDataViewState(id: string, dataStream: string, title: string) { - // TODO: Need to fix the index pattern used here (aka ::failures) return { internalReferences: [ { @@ -147,14 +148,14 @@ function getAdHocDataViewState(id: string, dataStream: string, title: string) { adHocDataViews: { [id]: { id, - title: dataStream, + title: `${dataStream}${ALL_PATTERNS_SELECTOR}`, timeFieldName: '@timestamp', sourceFilters: [], fieldFormats: {}, runtimeFieldMap: {}, fieldAttrs: {}, allowNoIndex: false, - name: title, + name: `${dataStream}${ALL_PATTERNS_SELECTOR}`, }, }, }; @@ -181,7 +182,7 @@ function getChartColumns(breakdownField?: string): Record { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts index 928fecc8e3096..f73995da493c9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts @@ -7,6 +7,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { rangeQuery } from '@kbn/observability-plugin/server'; +import { FAILURE_STORE_SELECTOR } from '../../../../common/constants'; import { TIMESTAMP } from '../../../../common/es_fields'; import { FailedDocsDetails } from '../../../../common/api_types'; import { getFieldIntervalInSeconds } from '../../../utils/get_interval'; @@ -48,7 +49,7 @@ export async function getFailedDocsDetails({ }; const response = await datasetQualityESClient.search({ - index: dataStream, + index: `${dataStream}${FAILURE_STORE_SELECTOR}`, track_total_hits: true, size: 0, query: { @@ -57,7 +58,6 @@ export async function getFailedDocsDetails({ }, }, aggs, - failure_store: 'only', }); return { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts index c8fd640b4cd21..6db11b478ec2f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts @@ -8,6 +8,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { SearchHit } from '@kbn/es-types'; import { rangeQuery } from '@kbn/observability-plugin/server'; +import { FAILURE_STORE_SELECTOR } from '../../../../common/constants'; import { createDatasetQualityESClient } from '../../../utils'; import { TIMESTAMP } from '../../../../common/es_fields'; @@ -29,7 +30,7 @@ export async function getFailedDocsErrors({ }; const response = await datasetQualityESClient.search({ - index: dataStream, + index: `${dataStream}${FAILURE_STORE_SELECTOR}`, size: 10000, query: { bool, @@ -41,7 +42,7 @@ export async function getFailedDocsErrors({ }, }, ], - failure_store: 'only', + /* failure_store: 'only', */ }); const errors = extractAndDeduplicateValues(response.hits.hits); diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 53db8217d18d6..8a78b4163da95 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -22,8 +22,6 @@ import { type DatasetQualityESSearchParams = ESSearchRequest & { size: number; - // TODO: Remove search parameter once ::failures is supported - failure_store?: 'only'; }; export type DatasetQualityESClient = ReturnType; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts index 0fdd4a0681148..72a1c2ea649ea 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import expect from '@kbn/expect'; import merge from 'lodash/merge'; import { DatasetQualityFtrProviderContext } from './config'; @@ -70,13 +71,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Enable failure store for logs await synthtrace.createCustomPipeline(customLogLevelProcessor, 'logs-apache.access@custom'); - await synthtrace.createComponentTemplate('logs-apache.access@custom', undefined, { - 'index.default_pipeline': 'logs-apache.access@custom', + await synthtrace.createComponentTemplate({ + name: 'logs-apache.access@custom', + settings: { + 'index.default_pipeline': 'logs-apache.access@custom', + }, }); await synthtrace.updateIndexTemplate( 'logs-apache.access', - (template: Record): Record => { - const next: Record = { + (template): IndicesPutIndexTemplateRequest => { + const next = { name: 'logs-apache.access', data_stream: { failure_store: true,