lens esql generation

src/platform/plugins/shared/data/common/search/aggs/buckets/index.ts

@@ -30,7 +30,7 @@ export * from './lib/cidr_mask';
 export * from './lib/date_range';
 export * from './lib/ip_range';
 export * from './lib/time_buckets/calc_auto_interval';
-export { TimeBuckets } from './lib/time_buckets';
+export { TimeBuckets, convertDurationToNormalizedEsInterval } from './lib/time_buckets';
 export * from './migrate_include_exclude_format';
 export * from './range_fn';
 export * from './range';

src/platform/plugins/shared/data/common/search/aggs/buckets/lib/time_buckets/index.ts

@@ -8,3 +8,4 @@
  */
 
 export { TimeBuckets } from './time_buckets';
+export { convertDurationToNormalizedEsInterval } from './calc_es_interval';

src/platform/plugins/shared/data/common/search/expressions/esql.ts

@@ -22,6 +22,7 @@ import { zipObject } from 'lodash';
 import { catchError, defer, map, Observable, switchMap, tap, throwError } from 'rxjs';
 import { buildEsQuery, type Filter } from '@kbn/es-query';
 import type { ESQLSearchParams, ESQLSearchResponse } from '@kbn/es-types';
+import DateMath from '@kbn/datemath';
 import { getEsQueryConfig } from '../../es_query';
 import { getTime } from '../../query';
 import {
@@ -328,6 +329,13 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           );
           const indexPattern = getIndexPatternFromESQLQuery(query);
 
+          const appliedTimeRange = input?.timeRange
+            ? {
+                from: DateMath.parse(input.timeRange.from),
+                to: DateMath.parse(input.timeRange.to),
+              }
+            : undefined;
+
           const allColumns =
             (body.all_columns ?? body.columns)?.map(({ name, type }) => ({
               id: name,
@@ -338,7 +346,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
                 sourceParams:
                   type === 'date'
                     ? {
-                        appliedTimeRange: input?.timeRange,
+                        appliedTimeRange,
                         params: {},
                         indexPattern,
                       }

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_range_select.test.ts

@@ -97,6 +97,7 @@ describe('brushEvent', () => {
             name: '1',
             meta: {
               type: 'date',
+              sourceParams: {},
             },
           },
         ],
@@ -247,5 +248,20 @@ describe('brushEvent', () => {
         expect(rangeFilter.query.range['1']).toHaveProperty('format', 'strict_date_optional_time');
       }
     });
+
+    test('for column with different name than source field', async () => {
+      const rangeBegin = JAN_01_2014;
+      const rangeEnd = rangeBegin + DAY_IN_MS;
+      esqlEventContext.range = [rangeBegin, rangeEnd];
+      esqlEventContext.table.columns[0].meta!.sourceParams!.sourceField = 'time';
+      esqlEventContext.table.columns[0].name = 'time over 12h';
+
+      const filter = await createFiltersFromRangeSelectAction(esqlEventContext);
+
+      expect(filter).toBeDefined();
+      expect(filter.length).toEqual(1);
+      expect(filter[0].query).toBeDefined();
+      expect(filter[0].query!.range.time).toBeDefined();
+    });
   });
 });

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_range_select.ts

@@ -28,9 +28,9 @@ export interface RangeSelectDataContext {
 const getParameters = async (event: RangeSelectDataContext) => {
   const column: Record<string, any> = event.table.columns[event.column];
   // Handling of the ES|QL datatable
-  if (isOfAggregateQueryType(event.query)) {
+  if (isOfAggregateQueryType(event.query) || event.table.meta?.type === 'es_ql') {
     const field = new DataViewField({
-      name: column.name,
+      name: column.meta?.sourceParams?.sourceField || column.name,
       type: column.meta?.type ?? 'unknown',
       esTypes: column.meta?.esType ? ([column.meta.esType] as string[]) : undefined,
       searchable: true,
@@ -43,8 +43,14 @@ const getParameters = async (event: RangeSelectDataContext) => {
     };
   }
   if (column.meta && 'sourceParams' in column.meta) {
-    const { indexPatternId, ...aggConfigs } = column.meta.sourceParams;
+    const { sourceField, ...aggConfigs } = column.meta.sourceParams;
+    const indexPatternId =
+      column.meta.sourceParams.indexPatternId || column.meta.sourceParams.indexPattern;
     const indexPattern = await getIndexPatterns().get(indexPatternId);
+    // if (event.table.meta?.type === 'es_ql') {
+    //   const field = indexPattern.fields.getByName(sourceField);
+    //   return { field, indexPattern };
+    // }
     const aggConfigsInstance = getSearchService().aggs.createAggConfigs(indexPattern, [
       aggConfigs as AggConfigSerialized,
     ]);

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_value_click.test.ts

@@ -13,9 +13,11 @@ import { setIndexPatterns, setSearchService } from '../../services';
 import {
   createFiltersFromValueClickAction,
   appendFilterToESQLQueryFromValueClickAction,
+  createFilterESQL,
 } from './create_filters_from_value_click';
 import { FieldFormatsGetConfigFn, BytesFormat } from '@kbn/field-formats-plugin/common';
 import { RangeFilter } from '@kbn/es-query';
+import { Datatable } from '@kbn/expressions-plugin/common';
 
 const mockField = {
   name: 'bytes',
@@ -105,6 +107,118 @@ describe('createFiltersFromClickEvent', () => {
       expect(filters.length).toEqual(1);
     });
   });
+
+  describe('createFilterESQL', () => {
+    let table: Datatable;
+
+    beforeEach(() => {
+      table = {
+        type: 'datatable',
+        columns: [
+          {
+            name: 'test',
+            id: '1-1',
+            meta: {
+              type: 'number',
+              sourceParams: {
+                sourceField: 'bytes',
+              },
+            },
+          },
+        ],
+        rows: [
+          {
+            '1-1': '2048',
+          },
+        ],
+      };
+    });
+
+    test('ignores event when sourceField is missing', async () => {
+      table.columns[0].meta.sourceParams = {};
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toBeUndefined();
+    });
+
+    test('ignores event when value for rows is not provided', async () => {
+      table.rows[0]['1-1'] = null;
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toBeUndefined();
+    });
+
+    test('handles an event when operation type is a date histogram', async () => {
+      (table.columns[0].meta.sourceParams as any).operationType = 'date_histogram';
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "meta": Object {
+              "field": "bytes",
+              "index": "2048",
+              "params": Object {},
+            },
+            "query": Object {
+              "range": Object {
+                "bytes": Object {
+                  "format": NaN,
+                  "gte": 2048,
+                  "lt": 2048,
+                },
+              },
+            },
+          },
+        ]
+      `);
+    });
+
+    test('handles an event when operation type is histogram', async () => {
+      (table.columns[0].meta.sourceParams as any).operationType = 'histogram';
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "meta": Object {
+              "field": "bytes",
+              "index": "2048",
+              "params": Object {},
+            },
+            "query": Object {
+              "range": Object {
+                "bytes": Object {
+                  "gte": 2048,
+                  "lt": 2048,
+                },
+              },
+            },
+          },
+        ]
+      `);
+    });
+
+    test('handles an event when operation type is not date histogram', async () => {
+      const filter = await createFilterESQL(table, 0, 0);
+
+      expect(filter).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "meta": Object {
+              "index": undefined,
+            },
+            "query": Object {
+              "exists": Object {
+                "field": "bytes",
+              },
+            },
+          },
+        ]
+      `);
+    });
+  });
+
   describe('appendFilterToESQLQueryFromValueClickAction', () => {
     let dataPoints: Parameters<typeof appendFilterToESQLQueryFromValueClickAction>[0]['data'];
     beforeEach(() => {

src/platform/plugins/shared/data/public/actions/filters/create_filters_from_value_click.ts

@@ -17,13 +17,17 @@ import {
   type AggregateQuery,
 } from '@kbn/es-query';
 import { appendWhereClauseToESQLQuery } from '@kbn/esql-utils';
+import {
+  buildSimpleExistFilter,
+  buildSimpleNumberRangeFilter,
+} from '@kbn/es-query/src/filters/build_filters';
 import { getIndexPatterns, getSearchService } from '../../services';
 import { AggConfigSerialized } from '../../../common/search/aggs';
 import { mapAndFlattenFilters } from '../../query';
 
 interface ValueClickDataContext {
   data: Array<{
-    table: Pick<Datatable, 'rows' | 'columns'>;
+    table: Pick<Datatable, 'rows' | 'columns' | 'meta'>;
     column: number;
     row: number;
     value: any;
@@ -129,6 +133,69 @@ export const createFilter = async (
   return filter;
 };
 
+export const createFilterESQL = async (
+  table: Pick<Datatable, 'rows' | 'columns'>,
+  columnIndex: number,
+  rowIndex: number
+) => {
+  if (
+    !table ||
+    !table.columns ||
+    !table.columns[columnIndex] ||
+    !table.columns[columnIndex].meta ||
+    !table.columns[columnIndex].meta.sourceParams?.sourceField ||
+    table.columns[columnIndex].meta.sourceParams?.sourceField === '___records___'
+  ) {
+    return;
+  }
+  const column = table.columns[columnIndex];
+  const { indexPattern, sourceField, operationType, interval } = table.columns[columnIndex].meta
+    .sourceParams as {
+    indexPattern: string;
+    sourceField: string;
+    operationType: string;
+    interval: number;
+  };
+
+  const value = rowIndex > -1 ? table.rows[rowIndex][column.id] : null;
+  if (value === null || value === undefined) {
+    return;
+  }
+
+  const filters: Filter[] = [];
+
+  if (operationType === 'date_histogram') {
+    filters.push(
+      buildSimpleNumberRangeFilter(
+        sourceField,
+        {
+          gte: value,
+          lt: value + interval,
+          format: 'strict_date_optional_time',
+        },
+        indexPattern,
+        value
+      )
+    );
+  } else if (operationType === 'histogram') {
+    filters.push(
+      buildSimpleNumberRangeFilter(
+        sourceField,
+        {
+          gte: value,
+          lt: value + interval,
+        },
+        indexPattern,
+        value
+      )
+    );
+  } else {
+    filters.push(buildSimpleExistFilter(sourceField, indexPattern));
+  }
+
+  return filters;
+};
+
 /** @public */
 export const createFiltersFromValueClickAction = async ({
   data,
@@ -141,7 +208,10 @@ export const createFiltersFromValueClickAction = async ({
       .filter((point) => point)
       .map(async (val) => {
         const { table, column, row } = val;
-        const filter: Filter[] = (await createFilter(table, column, row)) || [];
+        const filter =
+          table.meta?.type === 'es_ql'
+            ? await createFilterESQL(table, column, row)
+            : (await createFilter(table, column, row)) || [];
         if (filter) {
           filter.forEach((f) => {
             if (negate) {

src/platform/plugins/shared/data/public/index.ts

@@ -22,6 +22,7 @@ export {
   getIndexPatternFromFilter,
 } from './query';
 
+export { convertIntervalToEsInterval } from '../common/search/aggs/buckets/lib/time_buckets/calc_es_interval';
 /**
  * Exporters (CSV)
  */

x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns.test.ts

@@ -319,8 +319,8 @@ describe('map_to_columns', () => {
       );
 
       expect(result.columns).toStrictEqual([
-        { id: 'a', name: 'A', meta: { type: 'number', field: undefined, params: undefined } },
-        { id: 'b', name: 'B', meta: { type: 'number', field: undefined, params: undefined } },
+        { id: 'a', name: 'A', meta: { type: 'number', sourceParams: {} } },
+        { id: 'b', name: 'B', meta: { type: 'number', sourceParams: {} } },
       ]);
 
       expect(result.rows).toStrictEqual([

x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns_fn.ts

@@ -10,7 +10,7 @@ import type { OriginalColumn, MapToColumnsExpressionFunction } from './types';
 
 function getColumnName(originalColumn: OriginalColumn, newColumn: DatatableColumn) {
   if (originalColumn?.operationType === 'date_histogram') {
-    const fieldName = originalColumn.sourceField;
+    const fieldName = originalColumn.sourceField as string;
 
     // HACK: This is a hack, and introduces some fragility into
     // column naming. Eventually, this should be calculated and

x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns_fn_textbased.ts

@@ -38,8 +38,24 @@ export const mapToOriginalColumnsTextBased: MapToColumnsExpressionFunction['fn']
         name: originalColumn.label,
         meta: {
           ...column.meta,
-          field: originalColumn.sourceField,
-          params: originalColumn.format,
+          ...('sourceField' in originalColumn ? { field: originalColumn.sourceField } : {}),
+          ...('format' in originalColumn ? { params: originalColumn.format } : {}),
+          sourceParams: {
+            ...(column.meta?.sourceParams ?? {}),
+            ...('sourceField' in originalColumn ? { sourceField: originalColumn.sourceField } : {}),
+            ...('operationType' in originalColumn
+              ? { operationType: originalColumn.operationType }
+              : {}),
+            ...('interval' in originalColumn ? { interval: originalColumn.interval } : {}),
+            ...('params' in originalColumn
+              ? {
+                  params: {
+                    ...(originalColumn.params as object),
+                    used_interval: `${originalColumn.interval}ms`,
+                  },
+                }
+              : {}),
+          },
         },
       }));
     }),

x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/types.ts

@@ -9,8 +9,8 @@ import { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-plugin
 import { SerializedFieldFormat } from '@kbn/field-formats-plugin/common';
 
 export type OriginalColumn = { id: string; label: string; format?: SerializedFieldFormat } & (
-  | { operationType: 'date_histogram'; sourceField: string }
-  | { operationType: string; sourceField: never }
+  | { operationType: 'date_histogram'; sourceField: string; interval: number }
+  | { operationType: string; sourceField?: string; interval: never }
 );
 
 export type MapToColumnsExpressionFunction = ExpressionFunctionDefinition<