[ML] Field Stats: Use field caps option include_empty_fields=false …

…to identify populated fields. (#205417)

## Summary

Part of #178606.

Uses `dataViews.getFieldsForIndexPattern()` instead of custom code to
identify populated fields for field stats and the data grid used in the
Data Frame Analytics wizard.

- The previous custom code supported abort signals to cancel requests as
well as runtime fields. This was not yet supported by
`getFieldsForIndexPattern/getFieldsForWildcard`, so this PR adds that
capability.
- This also tweaks the options interface for `getFieldsForIndexPattern`
so you no longer have to pass in the empty `pattern: ''`.

This GIF demonstrates cancelling the request by navigating away from the
Data Frame Analytics wizard while the page is still loading (done with
3G throttling in dev tools):

![field-caps-cancel-0001](https://github.com/user-attachments/assets/8865ef08-76f0-4c84-a459-211230b2608e)

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.

---------

Co-authored-by: kibanamachine <[email protected]>

src/platform/packages/shared/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts

@@ -51,8 +51,6 @@ export async function fetchFieldExistence({
   dataViewsService: DataViewsContract;
 }) {
   const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
-    // filled in by data views service
-    pattern: '',
     indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
     includeEmptyFields: false,
   });

src/platform/plugins/shared/data_views/common/data_views/data_views.test.ts

@@ -219,9 +219,7 @@ describe('IndexPatterns', () => {
   });
 
   test('getFieldsForIndexPattern called with allowHidden set to undefined as default', async () => {
-    await indexPatterns.getFieldsForIndexPattern({ id: '1' } as DataViewSpec, {
-      pattern: 'something',
-    });
+    await indexPatterns.getFieldsForIndexPattern({ id: '1' } as DataViewSpec);
     expect(apiClient.getFieldsForWildcard).toBeCalledWith({
       allowHidden: undefined,
       allowNoIndex: true,
@@ -233,9 +231,7 @@ describe('IndexPatterns', () => {
   });
 
   test('getFieldsForIndexPattern called with allowHidden set to true', async () => {
-    await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: true } as DataViewSpec, {
-      pattern: 'something',
-    });
+    await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: true } as DataViewSpec);
     expect(apiClient.getFieldsForWildcard).toBeCalledWith({
       allowHidden: true,
       allowNoIndex: true,
@@ -247,9 +243,7 @@ describe('IndexPatterns', () => {
   });
 
   test('getFieldsForIndexPattern called with allowHidden set to false', async () => {
-    await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: false } as DataViewSpec, {
-      pattern: 'something',
-    });
+    await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: false } as DataViewSpec);
     expect(apiClient.getFieldsForWildcard).toBeCalledWith({
       allowHidden: false,
       allowNoIndex: true,
@@ -261,12 +255,10 @@ describe('IndexPatterns', () => {
   });
 
   test('getFieldsForIndexPattern called with getAllowHidden returning true', async () => {
-    await indexPatterns.getFieldsForIndexPattern(
-      { id: '1', getAllowHidden: () => true } as DataView,
-      {
-        pattern: 'something',
-      }
-    );
+    await indexPatterns.getFieldsForIndexPattern({
+      id: '1',
+      getAllowHidden: () => true,
+    } as DataView);
     expect(apiClient.getFieldsForWildcard).toBeCalledWith({
       allowHidden: true,
       allowNoIndex: true,
@@ -278,12 +270,10 @@ describe('IndexPatterns', () => {
   });
 
   test('getFieldsForIndexPattern called with getAllowHidden returning false', async () => {
-    await indexPatterns.getFieldsForIndexPattern(
-      { id: '1', getAllowHidden: () => false } as DataView,
-      {
-        pattern: 'something',
-      }
-    );
+    await indexPatterns.getFieldsForIndexPattern({
+      id: '1',
+      getAllowHidden: () => false,
+    } as DataView);
     expect(apiClient.getFieldsForWildcard).toBeCalledWith({
       allowHidden: false,
       allowNoIndex: true,

src/platform/plugins/shared/data_views/common/data_views/data_views.ts

@@ -242,7 +242,7 @@ export interface DataViewsServicePublicMethods {
    */
   getFieldsForIndexPattern: (
     indexPattern: DataView | DataViewSpec,
-    options?: GetFieldsOptions | undefined
+    options?: Omit<GetFieldsOptions, 'allowNoIndex' | 'pattern'>
   ) => Promise<FieldSpec[]>;
   /**
    * Get fields for index pattern string
@@ -593,13 +593,13 @@ export class DataViewsService {
   };
 
   /**
-   * Get field list by providing an index patttern (or spec).
+   * Get field list by providing an index pattern (or spec).
    * @param options options for getting field list
    * @returns FieldSpec[]
    */
   getFieldsForIndexPattern = async (
     indexPattern: DataView | DataViewSpec,
-    options?: Omit<GetFieldsOptions, 'allowNoIndex'>
+    options?: Omit<GetFieldsOptions, 'allowNoIndex' | 'pattern'>
   ) =>
     this.getFieldsForWildcard({
       type: indexPattern.type,

src/platform/plugins/shared/data_views/common/types.ts

@@ -324,6 +324,8 @@ export interface GetFieldsOptions {
   forceRefresh?: boolean;
   fieldTypes?: string[];
   includeEmptyFields?: boolean;
+  abortSignal?: AbortSignal;
+  runtimeMappings?: estypes.MappingRuntimeFields;
 }
 
 /**

src/platform/plugins/shared/data_views/public/data_views/data_views_api_client.test.mock.ts

@@ -12,3 +12,6 @@ import { setup } from '@kbn/core-test-helpers-http-setup-browser';
 export const { http } = setup((injectedMetadata) => {
   injectedMetadata.getBasePath.mockReturnValue('/hola/daro/');
 });
+
+export const indexFilterMock = { bool: { must: [{ match_all: {} }] } };
+export const runtimeMappingsMock = { myField: { type: 'keyword' } };

src/platform/plugins/shared/data_views/public/data_views/data_views_api_client.test.ts

@@ -8,9 +8,10 @@
  */
 
 import type { HttpSetup } from '@kbn/core/public';
-import { http } from './data_views_api_client.test.mock';
-import { DataViewsApiClient } from './data_views_api_client';
+import { http, indexFilterMock, runtimeMappingsMock } from './data_views_api_client.test.mock';
+import { getFieldsForWildcardRequestBody, DataViewsApiClient } from './data_views_api_client';
 import { FIELDS_PATH as expectedPath } from '../../common/constants';
+import type { GetFieldsOptions } from '../../common';
 
 describe('IndexPatternsApiClient', () => {
   let fetchSpy: jest.SpyInstance;
@@ -56,3 +57,36 @@ describe('IndexPatternsApiClient', () => {
     expect(fetchSpy.mock.calls[0][1].query.field_types).toEqual(fieldTypes);
   });
 });
+
+describe('getFieldsForWildcardRequestBody', () => {
+  test('returns undefined if no indexFilter or runtimeMappings', () => {
+    expect(getFieldsForWildcardRequestBody({} as unknown as GetFieldsOptions)).toBeUndefined();
+  });
+
+  test('returns just indexFilter if no runtimeMappings', () => {
+    expect(
+      getFieldsForWildcardRequestBody({
+        indexFilter: indexFilterMock,
+      } as unknown as GetFieldsOptions)
+    ).toEqual(JSON.stringify({ index_filter: indexFilterMock }));
+  });
+
+  test('returns just runtimeMappings if no indexFilter', () => {
+    expect(
+      getFieldsForWildcardRequestBody({
+        runtimeMappings: runtimeMappingsMock,
+      } as unknown as GetFieldsOptions)
+    ).toEqual(JSON.stringify({ runtime_mappings: runtimeMappingsMock }));
+  });
+
+  test('returns both indexFilter and runtimeMappings', () => {
+    expect(
+      getFieldsForWildcardRequestBody({
+        indexFilter: indexFilterMock,
+        runtimeMappings: runtimeMappingsMock,
+      } as unknown as GetFieldsOptions)
+    ).toEqual(
+      JSON.stringify({ index_filter: indexFilterMock, runtime_mappings: runtimeMappingsMock })
+    );
+  });
+});

src/platform/plugins/shared/data_views/public/data_views/data_views_api_client.ts

@@ -29,6 +29,24 @@ async function sha1(str: string) {
   }
 }
 
+/**
+ * Helper function to get the request body for the getFieldsForWildcard request
+ * @param options options for fields request
+ * @returns string | undefined
+ */
+export function getFieldsForWildcardRequestBody(options: GetFieldsOptions): string | undefined {
+  const { indexFilter, runtimeMappings } = options;
+
+  if (!indexFilter && !runtimeMappings) {
+    return;
+  }
+
+  return JSON.stringify({
+    ...(indexFilter && { index_filter: indexFilter }),
+    ...(runtimeMappings && { runtime_mappings: runtimeMappings }),
+  });
+}
+
 /**
  * Data Views API Client - client implementation
  */
@@ -49,7 +67,8 @@ export class DataViewsApiClient implements IDataViewsApiClient {
     url: string,
     query?: {},
     body?: string,
-    forceRefresh?: boolean
+    forceRefresh?: boolean,
+    abortSignal?: AbortSignal
   ): Promise<HttpResponse<T> | undefined> {
     const asResponse = true;
     const cacheOptions: { cache?: RequestCache } = forceRefresh ? { cache: 'no-cache' } : {};
@@ -59,21 +78,33 @@ export class DataViewsApiClient implements IDataViewsApiClient {
     const headers = userHash ? { 'user-hash': userHash } : undefined;
 
     const request = body
-      ? this.http.post<T>(url, { query, body, version, asResponse })
+      ? this.http.post<T>(url, { query, body, version, asResponse, signal: abortSignal })
       : this.http.fetch<T>(url, {
           query,
           version,
           ...cacheOptions,
           asResponse,
           headers,
+          signal: abortSignal,
         });
 
     return request.catch((resp) => {
-      if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') {
-        throw new DataViewMissingIndices(resp.body.message);
+      // Custom errors with a body
+      if (resp?.body) {
+        if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') {
+          throw new DataViewMissingIndices(resp.body.message);
+        }
+
+        throw new Error(resp.body.message || resp.body.error || `${resp.body.statusCode} Response`);
       }
 
-      throw new Error(resp.body.message || resp.body.error || `${resp.body.statusCode} Response`);
+      // Regular errors including AbortError
+      if (typeof resp?.name === 'string' && typeof resp?.message === 'string') {
+        throw resp;
+      }
+
+      // Other unknown errors
+      throw new Error('Unknown error');
     });
   }
 
@@ -99,6 +130,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
       allowHidden,
       fieldTypes,
       includeEmptyFields,
+      abortSignal,
     } = options;
     const path = indexFilter ? FIELDS_FOR_WILDCARD_PATH : FIELDS_PATH;
     const versionQueryParam = indexFilter ? {} : { apiVersion: version };
@@ -119,8 +151,9 @@ export class DataViewsApiClient implements IDataViewsApiClient {
         include_empty_fields: includeEmptyFields,
         ...versionQueryParam,
       },
-      indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined,
-      forceRefresh
+      getFieldsForWildcardRequestBody(options),
+      forceRefresh,
+      abortSignal
     ).then((response) => {
       return {
         indices: response?.body?.indices || [],

src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -82,6 +82,8 @@ export class IndexPatternsFetcher {
     allowHidden?: boolean;
     fieldTypes?: string[];
     includeEmptyFields?: boolean;
+    abortSignal?: AbortSignal;
+    runtimeMappings?: estypes.MappingRuntimeFields;
   }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
     const {
       pattern,
@@ -93,6 +95,8 @@ export class IndexPatternsFetcher {
       allowHidden,
       fieldTypes,
       includeEmptyFields,
+      abortSignal,
+      runtimeMappings,
     } = options;
     const allowNoIndices = fieldCapsOptions?.allow_no_indices || this.allowNoIndices;
 
@@ -112,6 +116,8 @@ export class IndexPatternsFetcher {
       expandWildcards,
       fieldTypes,
       includeEmptyFields,
+      runtimeMappings,
+      abortSignal,
     });
 
     if (this.rollupsEnabled && type === DataViewType.ROLLUP && rollupIndex) {

src/platform/plugins/shared/data_views/server/fetcher/lib/es_api.ts

@@ -8,7 +8,10 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
-import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  ExpandWildcard,
+  MappingRuntimeFields,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { QueryDslQueryContainer } from '../../../common/types';
 import { convertEsError } from './errors';
 
@@ -50,6 +53,8 @@ interface FieldCapsApiParams {
   expandWildcards?: ExpandWildcard;
   fieldTypes?: string[];
   includeEmptyFields?: boolean;
+  runtimeMappings?: MappingRuntimeFields;
+  abortSignal?: AbortSignal;
 }
 
 /**
@@ -77,6 +82,8 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
     expandWildcards,
     fieldTypes,
     includeEmptyFields,
+    runtimeMappings,
+    abortSignal,
   } = params;
   try {
     return await callCluster.fieldCaps(
@@ -88,9 +95,10 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
         expand_wildcards: expandWildcards,
         types: fieldTypes,
         include_empty_fields: includeEmptyFields ?? true,
+        runtime_mappings: runtimeMappings,
         ...fieldCapsOptions,
       },
-      { meta: true }
+      { meta: true, signal: abortSignal }
     );
   } catch (error) {
     // return an empty set for closed indices

src/platform/plugins/shared/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js

@@ -34,13 +34,15 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
 
   const fillUndefinedParams = (args) => ({
     callCluster: undefined,
+    abortSignal: undefined,
     indices: undefined,
     expandWildcards: undefined,
     fieldCapsOptions: undefined,
     fieldTypes: undefined,
     indexFilter: undefined,
     fields: undefined,
     includeEmptyFields: undefined,
+    runtimeMappings: undefined,
     ...args,
   });
 

src/platform/plugins/shared/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts

@@ -9,7 +9,10 @@
 
 import { defaults, keyBy, sortBy } from 'lodash';
 
-import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  ExpandWildcard,
+  MappingRuntimeFields,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server';
 import { callFieldCapsApi } from '../es_api';
 import { readFieldCapsResponse } from './field_caps_response';
@@ -30,6 +33,8 @@ interface FieldCapabilitiesParams {
   expandWildcards?: ExpandWildcard;
   fieldTypes?: string[];
   includeEmptyFields?: boolean;
+  runtimeMappings?: MappingRuntimeFields;
+  abortSignal?: AbortSignal;
 }
 
 /**
@@ -54,6 +59,8 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     expandWildcards,
     fieldTypes,
     includeEmptyFields,
+    runtimeMappings,
+    abortSignal,
   } = params;
 
   const excludedTiers = await uiSettingsClient?.get<string>(DATA_VIEWS_FIELDS_EXCLUDED_TIERS);
@@ -66,6 +73,8 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     expandWildcards,
     fieldTypes,
     includeEmptyFields,
+    runtimeMappings,
+    abortSignal,
   });
   const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
   const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');