lens esql generation

src/platform/plugins/shared/data/common/search/aggs/buckets/index.ts

@@ -30,7 +30,7 @@ export * from './lib/cidr_mask';
 export * from './lib/date_range';
 export * from './lib/ip_range';
 export * from './lib/time_buckets/calc_auto_interval';
-export { TimeBuckets } from './lib/time_buckets';
+export { TimeBuckets, convertDurationToNormalizedEsInterval } from './lib/time_buckets';
 export * from './migrate_include_exclude_format';
 export * from './range_fn';
 export * from './range';

src/platform/plugins/shared/data/common/search/aggs/buckets/lib/time_buckets/index.ts

@@ -8,3 +8,4 @@
  */
 
 export { TimeBuckets } from './time_buckets';
+export { convertDurationToNormalizedEsInterval } from './calc_es_interval';

src/platform/plugins/shared/data/common/search/expressions/esql.ts

@@ -22,6 +22,7 @@ import { zipObject } from 'lodash';
 import { catchError, defer, map, Observable, switchMap, tap, throwError } from 'rxjs';
 import { buildEsQuery, type Filter } from '@kbn/es-query';
 import type { ESQLSearchParams, ESQLSearchResponse } from '@kbn/es-types';
+import DateMath from '@kbn/datemath';
 import { getEsQueryConfig } from '../../es_query';
 import { getTime } from '../../query';
 import {
@@ -328,6 +329,13 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           );
           const indexPattern = getIndexPatternFromESQLQuery(query);
 
+          const appliedTimeRange = input?.timeRange
+            ? {
+                from: DateMath.parse(input.timeRange.from),
+                to: DateMath.parse(input.timeRange.to),
+              }
+            : undefined;
+
           const allColumns =
             (body.all_columns ?? body.columns)?.map(({ name, type }) => ({
               id: name,
@@ -338,7 +346,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
                 sourceParams:
                   type === 'date'
                     ? {
-                        appliedTimeRange: input?.timeRange,
+                        appliedTimeRange,
                         params: {},
                         indexPattern,
                       }

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_range_select.test.ts

@@ -97,6 +97,7 @@ describe('brushEvent', () => {
             name: '1',
             meta: {
               type: 'date',
+              sourceParams: {},
             },
           },
         ],
@@ -247,5 +248,20 @@ describe('brushEvent', () => {
         expect(rangeFilter.query.range['1']).toHaveProperty('format', 'strict_date_optional_time');
       }
     });
+
+    test('for column with different name than source field', async () => {
+      const rangeBegin = JAN_01_2014;
+      const rangeEnd = rangeBegin + DAY_IN_MS;
+      esqlEventContext.range = [rangeBegin, rangeEnd];
+      esqlEventContext.table.columns[0].meta!.sourceParams!.sourceField = 'time';
+      esqlEventContext.table.columns[0].name = 'time over 12h';
+
+      const filter = await createFiltersFromRangeSelectAction(esqlEventContext);
+
+      expect(filter).toBeDefined();
+      expect(filter.length).toEqual(1);
+      expect(filter[0].query).toBeDefined();
+      expect(filter[0].query!.range.time).toBeDefined();
+    });
   });
 });

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_range_select.ts

@@ -28,9 +28,9 @@ export interface RangeSelectDataContext {
 const getParameters = async (event: RangeSelectDataContext) => {
   const column: Record<string, any> = event.table.columns[event.column];
   // Handling of the ES|QL datatable
-  if (isOfAggregateQueryType(event.query)) {
+  if (isOfAggregateQueryType(event.query) || event.table.meta?.type === 'es_ql') {
     const field = new DataViewField({
-      name: column.name,
+      name: column.meta?.sourceParams?.sourceField || column.name,
       type: column.meta?.type ?? 'unknown',
       esTypes: column.meta?.esType ? ([column.meta.esType] as string[]) : undefined,
       searchable: true,
@@ -43,8 +43,14 @@ const getParameters = async (event: RangeSelectDataContext) => {
     };
   }
   if (column.meta && 'sourceParams' in column.meta) {
-    const { indexPatternId, ...aggConfigs } = column.meta.sourceParams;
+    const { sourceField, ...aggConfigs } = column.meta.sourceParams;
+    const indexPatternId =
+      column.meta.sourceParams.indexPatternId || column.meta.sourceParams.indexPattern;
     const indexPattern = await getIndexPatterns().get(indexPatternId);
+    // if (event.table.meta?.type === 'es_ql') {
+    //   const field = indexPattern.fields.getByName(sourceField);
+    //   return { field, indexPattern };
+    // }
     const aggConfigsInstance = getSearchService().aggs.createAggConfigs(indexPattern, [
       aggConfigs as AggConfigSerialized,
     ]);

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_value_click.test.ts

@@ -13,9 +13,11 @@ import { setIndexPatterns, setSearchService } from '../../services';
 import {
   createFiltersFromValueClickAction,
   appendFilterToESQLQueryFromValueClickAction,
+  createFilterESQL,
 } from './create_filters_from_value_click';
 import { FieldFormatsGetConfigFn, BytesFormat } from '@kbn/field-formats-plugin/common';
 import { RangeFilter } from '@kbn/es-query';
+import { Datatable } from '@kbn/expressions-plugin/common';
 
 const mockField = {
   name: 'bytes',
@@ -105,6 +107,118 @@ describe('createFiltersFromClickEvent', () => {
       expect(filters.length).toEqual(1);
     });
   });
+
+  describe('createFilterESQL', () => {
+    let table: Datatable;
+
+    beforeEach(() => {
+      table = {
+        type: 'datatable',
+        columns: [
+          {
+            name: 'test',
+            id: '1-1',
+            meta: {
+              type: 'number',
+              sourceParams: {
+                sourceField: 'bytes',
+              },
+            },
+          },
+        ],
+        rows: [
+          {
+            '1-1': '2048',
+          },
+        ],
+      };
+    });
+
+    test('ignores event when sourceField is missing', async () => {
+      table.columns[0].meta.sourceParams = {};
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toBeUndefined();
+    });
+
+    test('ignores event when value for rows is not provided', async () => {
+      table.rows[0]['1-1'] = null;
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toBeUndefined();
+    });
+
+    test('handles an event when operation type is a date histogram', async () => {
+      (table.columns[0].meta.sourceParams as any).operationType = 'date_histogram';
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "meta": Object {
+              "field": "bytes",
+              "index": "2048",
+              "params": Object {},
+            },
+            "query": Object {
+              "range": Object {
+                "bytes": Object {
+                  "format": NaN,
+                  "gte": 2048,
+                  "lt": 2048,
+                },
+              },
+            },
+          },
+        ]
+      `);
+    });
+
+    test('handles an event when operation type is histogram', async () => {
+      (table.columns[0].meta.sourceParams as any).operationType = 'histogram';
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "meta": Object {
+              "field": "bytes",
+              "index": "2048",
+              "params": Object {},
+            },
+            "query": Object {
+              "range": Object {
+                "bytes": Object {
+                  "gte": 2048,
+                  "lt": 2048,
+                },
+              },
+            },
+          },
+        ]
+      `);
+    });
+
+    test('handles an event when operation type is not date histogram', async () => {
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "meta": Object {
+              "index": undefined,
+            },
+            "query": Object {
+              "exists": Object {
+                "field": "bytes",
+              },
+            },
+          },
+        ]
+      `);
+    });
+  });
+
   describe('appendFilterToESQLQueryFromValueClickAction', () => {
     let dataPoints: Parameters<typeof appendFilterToESQLQueryFromValueClickAction>[0]['data'];
     beforeEach(() => {

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_value_click.ts

@@ -8,7 +8,7 @@
  */
 
 import _ from 'lodash';
-import { Datatable } from '@kbn/expressions-plugin/public';
+import { Datatable, isSourceParamsESQL } from '@kbn/expressions-plugin/public';
 import {
   compareFilters,
   COMPARE_ALL_OPTIONS,
@@ -17,13 +17,17 @@ import {
   type AggregateQuery,
 } from '@kbn/es-query';
 import { appendWhereClauseToESQLQuery } from '@kbn/esql-utils';
+import {
+  buildSimpleExistFilter,
+  buildSimpleNumberRangeFilter,
+} from '@kbn/es-query/src/filters/build_filters';
 import { getIndexPatterns, getSearchService } from '../../services';
 import { AggConfigSerialized } from '../../../common/search/aggs';
 import { mapAndFlattenFilters } from '../../query';
 
 interface ValueClickDataContext {
   data: Array<{
-    table: Pick<Datatable, 'rows' | 'columns'>;
+    table: Pick<Datatable, 'rows' | 'columns' | 'meta'>;
     column: number;
     row: number;
     value: any;
@@ -129,6 +133,51 @@ export const createFilter = async (
   return filter;
 };
 
+export const createFilterESQL = async (
+  table: Pick<Datatable, 'rows' | 'columns'>,
+  columnIndex: number,
+  rowIndex: number
+) => {
+  const column = table?.columns?.[columnIndex];
+  if (
+    !column?.meta?.sourceParams?.sourceField ||
+    column.meta.sourceParams?.sourceField === '___records___'
+  ) {
+    return [];
+  }
+  const sourceParams = column.meta.sourceParams;
+  if (!isSourceParamsESQL(sourceParams)) {
+    return [];
+  }
+  const { indexPattern, sourceField, operationType, interval } = sourceParams;
+
+  const value = rowIndex > -1 ? table.rows[rowIndex][column.id] : null;
+  if (value == null) {
+    return [];
+  }
+
+  const filters: Filter[] = [];
+
+  if (['date_histogram', 'histogram'].includes(operationType)) {
+    filters.push(
+      buildSimpleNumberRangeFilter(
+        sourceField,
+        {
+          gte: value,
+          lt: value + interval,
+          ...(operationType === 'date_hisotgram' ? { format: 'strict_date_optional_time' } : {}),
+        },
+        value,
+        indexPattern
+      )
+    );
+  } else {
+    filters.push(buildSimpleExistFilter(sourceField, indexPattern));
+  }
+
+  return filters;
+};
+
 /** @public */
 export const createFiltersFromValueClickAction = async ({
   data,
@@ -141,7 +190,10 @@ export const createFiltersFromValueClickAction = async ({
       .filter((point) => point)
       .map(async (val) => {
         const { table, column, row } = val;
-        const filter: Filter[] = (await createFilter(table, column, row)) || [];
+        const filter =
+          table.meta?.type === 'es_ql'
+            ? await createFilterESQL(table, column, row)
+            : (await createFilter(table, column, row)) || [];
         if (filter) {
           filter.forEach((f) => {
             if (negate) {

src/platform/plugins/shared/data/public/index.ts

@@ -22,6 +22,7 @@ export {
   getIndexPatternFromFilter,
 } from './query';
 
+export { convertIntervalToEsInterval } from '../common/search/aggs/buckets/lib/time_buckets/calc_es_interval';
 /**
  * Exporters (CSV)
  */

src/platform/plugins/shared/expressions/common/expression_types/specs/datatable.ts

@@ -90,6 +90,23 @@ export interface DatatableColumnMeta {
   sourceParams?: SerializableRecord;
 }
 
+interface SourceParamsESQL extends Record<string, unknown> {
+  indexPattern: string;
+  sourceField: string;
+  operationType: string;
+  interval: number;
+}
+
+export function isSourceParamsESQL(obj: Record<string, unknown>): obj is SourceParamsESQL {
+  return (
+    obj &&
+    typeof obj.indexPattern === 'string' &&
+    typeof obj.sourceField === 'string' &&
+    typeof obj.operationType === 'string' &&
+    typeof obj.interval === 'number'
+  );
+}
+
 /**
  * This type represents the shape of a column in a `Datatable`.
  */

src/platform/plugins/shared/expressions/public/index.ts

@@ -117,3 +117,5 @@ export {
   parseExpression,
   createDefaultInspectorAdapters,
 } from '../common';
+
+export { isSourceParamsESQL } from '../common/expression_types';

x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns.test.ts

@@ -319,8 +319,8 @@ describe('map_to_columns', () => {
       );
 
       expect(result.columns).toStrictEqual([
-        { id: 'a', name: 'A', meta: { type: 'number', field: undefined, params: undefined } },
-        { id: 'b', name: 'B', meta: { type: 'number', field: undefined, params: undefined } },
+        { id: 'a', name: 'A', meta: { type: 'number', sourceParams: {} } },
+        { id: 'b', name: 'B', meta: { type: 'number', sourceParams: {} } },
       ]);
 
       expect(result.rows).toStrictEqual([

x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns_fn.ts

@@ -10,7 +10,7 @@ import type { OriginalColumn, MapToColumnsExpressionFunction } from './types';
 
 function getColumnName(originalColumn: OriginalColumn, newColumn: DatatableColumn) {
   if (originalColumn?.operationType === 'date_histogram') {
-    const fieldName = originalColumn.sourceField;
+    const fieldName = originalColumn.sourceField as string;
 
     // HACK: This is a hack, and introduces some fragility into
     // column naming. Eventually, this should be calculated and