using new failure store selector syntax

elastic · Jan 15, 2025 · 5289ba5 · 5289ba5
1 parent 394320d
commit 5289ba5
Show file tree

Hide file tree

Showing 8 changed files with 26 additions and 21 deletions.
diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts
@@ -42,3 +42,6 @@ export const MASKED_FIELD_PLACEHOLDER = '<custom field>';
 export const UNKOWN_FIELD_PLACEHOLDER = '<unkwon>';
 
 export const KNOWN_TYPES: DataStreamType[] = ['logs', 'metrics', 'traces', 'synthetics'];
+
+export const ALL_PATTERNS_SELECTOR = '::*';
+export const FAILURE_STORE_SELECTOR = '::failures';
diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts
@@ -40,7 +40,6 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => {
 };
 
 export const extractIndexNameFromBackingIndex = (indexString: string): string => {
-  // TODO: Undo this change once ::failures is supported
   const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
   const match = indexString.match(pattern);
 

diff --git a/...omponents/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts b/...omponents/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts
@@ -9,6 +9,7 @@ import { i18n } from '@kbn/i18n';
 import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public';
 import { v4 as uuidv4 } from 'uuid';
 
+import { ALL_PATTERNS_SELECTOR, FAILURE_STORE_SELECTOR } from '../../../../../../common/constants';
 import {
   flyoutFailedDocsTrendText,
   flyoutFailedDocsPercentageText,
@@ -24,6 +25,7 @@ enum DatasetQualityLensColumn {
 }
 
 const MAX_BREAKDOWN_SERIES = 5;
+const FAILED_DOCS_QUERY = `_index: "${FAILURE_STORE_SELECTOR}"`;
 
 interface GetLensAttributesParams {
   color: string;
@@ -130,7 +132,6 @@ export function getLensAttributes({
 }
 
 function getAdHocDataViewState(id: string, dataStream: string, title: string) {
-  // TODO: Need to fix the index pattern used here (aka ::failures)
   return {
     internalReferences: [
       {
@@ -147,14 +148,14 @@ function getAdHocDataViewState(id: string, dataStream: string, title: string) {
     adHocDataViews: {
       [id]: {
         id,
-        title: dataStream,
+        title: `${dataStream}${ALL_PATTERNS_SELECTOR}`,
         timeFieldName: '@timestamp',
         sourceFilters: [],
         fieldFormats: {},
         runtimeFieldMap: {},
         fieldAttrs: {},
         allowNoIndex: false,
-        name: title,
+        name: `${dataStream}${ALL_PATTERNS_SELECTOR}`,
       },
     },
   };
@@ -181,7 +182,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
       scale: 'ratio',
       sourceField: '___records___',
       filter: {
-        query: '',
+        query: FAILED_DOCS_QUERY,
         language: 'kuery',
       },
       params: {
@@ -216,7 +217,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
             min: 0,
             max: 34,
           },
-          text: "count(kql='') / count()",
+          text: `count(kql='${FAILED_DOCS_QUERY}') / count()`,
         },
       },
       references: ['count_failed', 'count_total'],
@@ -230,7 +231,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
       references: [DatasetQualityLensColumn.Math],
       isBucketed: false,
       params: {
-        formula: "count(kql='') / count()",
+        formula: `count(kql='${FAILED_DOCS_QUERY}') / count()`,
         format: {
           id: 'percent',
           params: {
@@ -257,7 +258,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
               },
               orderDirection: 'desc',
               otherBucket: true,
-              missingBucket: true,
+              missingBucket: false,
               parentFormat: {
                 id: 'terms',
               },

diff --git a/...ty_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs.ts b/...ty_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs.ts
@@ -7,6 +7,7 @@
 
 import type { ElasticsearchClient } from '@kbn/core/server';
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { FAILURE_STORE_SELECTOR } from '../../../../common/constants';
 import {
   extractIndexNameFromBackingIndex,
   streamPartsToIndexPattern,
@@ -32,9 +33,8 @@ async function getPaginatedResults(options: {
     filter: [...rangeQuery(start, end)],
   };
 
-  // TODO: Fix index for accesing failure store (::failures) and remove the search parameter
   const response = await datasetQualityESClient.search({
-    index,
+    index: `${index}${FAILURE_STORE_SELECTOR}`,
     size: 0,
     query: {
       bool,
@@ -48,7 +48,6 @@ async function getPaginatedResults(options: {
         },
       },
     },
-    failure_store: 'only',
   });
 
   const currResults = (response.aggregations?.datasets.buckets ?? []).reduce((acc, curr) => {

diff --git a/...ion/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts b/...ion/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts
@@ -7,6 +7,7 @@
 
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { FAILURE_STORE_SELECTOR } from '../../../../common/constants';
 import { TIMESTAMP } from '../../../../common/es_fields';
 import { FailedDocsDetails } from '../../../../common/api_types';
 import { getFieldIntervalInSeconds } from '../../../utils/get_interval';
@@ -48,7 +49,7 @@ export async function getFailedDocsDetails({
   };
 
   const response = await datasetQualityESClient.search({
-    index: dataStream,
+    index: `${dataStream}${FAILURE_STORE_SELECTOR}`,
     track_total_hits: true,
     size: 0,
     query: {
@@ -57,7 +58,6 @@ export async function getFailedDocsDetails({
       },
     },
     aggs,
-    failure_store: 'only',
   });
 
   return {

diff --git a/...tion/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts b/...tion/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts
@@ -8,6 +8,7 @@
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { SearchHit } from '@kbn/es-types';
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { FAILURE_STORE_SELECTOR } from '../../../../common/constants';
 import { createDatasetQualityESClient } from '../../../utils';
 import { TIMESTAMP } from '../../../../common/es_fields';
 
@@ -29,7 +30,7 @@ export async function getFailedDocsErrors({
   };
 
   const response = await datasetQualityESClient.search({
-    index: dataStream,
+    index: `${dataStream}${FAILURE_STORE_SELECTOR}`,
     size: 10000,
     query: {
       bool,
@@ -41,7 +42,7 @@ export async function getFailedDocsErrors({
         },
       },
     ],
-    failure_store: 'only',
+    /* failure_store: 'only', */
   });
 
   const errors = extractAndDeduplicateValues(response.hits.hits);

diff --git a/...s/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/...s/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts
@@ -22,8 +22,6 @@ import {
 
 type DatasetQualityESSearchParams = ESSearchRequest & {
   size: number;
-  // TODO: Remove search parameter once ::failures is supported
-  failure_store?: 'only';
 };
 
 export type DatasetQualityESClient = ReturnType<typeof createDatasetQualityESClient>;

diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
 import expect from '@kbn/expect';
 import merge from 'lodash/merge';
 import { DatasetQualityFtrProviderContext } from './config';
@@ -70,13 +71,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
 
       // Enable failure store for logs
       await synthtrace.createCustomPipeline(customLogLevelProcessor, 'logs-apache.access@custom');
-      await synthtrace.createComponentTemplate('logs-apache.access@custom', undefined, {
-        'index.default_pipeline': 'logs-apache.access@custom',
+      await synthtrace.createComponentTemplate({
+        name: 'logs-apache.access@custom',
+        settings: {
+          'index.default_pipeline': 'logs-apache.access@custom',
+        },
       });
       await synthtrace.updateIndexTemplate(
         'logs-apache.access',
-        (template: Record<string, any>): Record<string, any> => {
-          const next: Record<string, any> = {
+        (template): IndicesPutIndexTemplateRequest => {
+          const next = {
             name: 'logs-apache.access',
             data_stream: {
               failure_store: true,