[ES|QL] JOIN command autocomplete and validation (#205762)

## Summary Part of #200858 Main goal of this PR is to introduce initial autocomplete for the `JOIN` command: ![Kapture 2025-01-09 at 19 02 17](https://github.com/user-attachments/assets/5ecaddb7-d8c1-4768-a22d-82d2adc521ce) In this PR: - Adds `JOIN` command and `AS` function definition - Adds `JOIN` command validation - Adds `JOIN` command autocomplete - New command suggestion, including command type - Command suggestion on partial command entry - Suggests lookup indices - Fetches them from the server and caches - Also suggests lookup index aliases - Suggests `ON` keyword - Suggests join condition fields - Suggests `,` or `|` after join condition fields - Autocomplete behaviour that could be improve in followup - After index suggestion selection, the "ON" suggestion does not appear automatically, user needs to enter space ` `. - When suggesting `ON <condition>` fields, compute lookup index and source index field intersection and show only those. - Only `LOOKUP JOIN` is exposed. `LEFT JOIN` and `RIGTH JOIN` are defined in code, but commented out. - The aliasing using `AS` operator will validate, but autocomplete does not actively suggest it to the user. --- ### Testing To test, you can create lookup indices in dev console using the following queries: ``` PUT /lookup_index { "settings": { "index.mode": "lookup" }, "mappings": { "properties": { "currency": { "type": "keyword" } } } } PUT /lookup_index_with_alias { "settings": { "index.mode": "lookup" }, "aliases": { "lookup_index2_alias1": {}, "lookup_index2_alias2": {} } } ``` Add some sample data: ``` POST /lookup_index/_doc { "currency": "EUR", "continenet": "Europe", "name": "Euro" } POST /lookup_index/_doc { "currency": "USD", "continenet": "North America", "name": "US Dollar" } POST /lookup_index/_doc { "currency": "USD", "continenet": "North America", "name": "Canadian Dollar" } ``` Add `kibana_sample_data_ecommerce` sample data and execute a query: ``` FROM kibana_sample_data_ecommerce | LOOKUP JOIN lookup_index ON currency ``` --- ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Jan 15, 2025 · 571ee96 · 571ee96
1 parent e7f0771
commit 571ee96
Show file tree

Hide file tree

Showing 42 changed files with 1,256 additions and 118 deletions.
diff --git a/src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx b/src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx
@@ -382,6 +382,7 @@ export const ESQLEditor = memo(function ESQLEditor({
       },
       // @ts-expect-error To prevent circular type import, type defined here is partial of full client
       getFieldsMetadata: fieldsMetadata?.getClient(),
+      getJoinIndices: kibana.services?.esql?.getJoinIndicesAutocomplete,
     };
     return callbacks;
   }, [
@@ -397,6 +398,7 @@ export const ESQLEditor = memo(function ESQLEditor({
     indexManagementApiService,
     histogramBarTarget,
     fieldsMetadata,
+    kibana.services?.esql?.getJoinIndicesAutocomplete,
   ]);
 
   const queryRunButtonProperties = useMemo(() => {

diff --git a/src/platform/packages/private/kbn-esql-editor/src/types.ts b/src/platform/packages/private/kbn-esql-editor/src/types.ts
@@ -71,6 +71,20 @@ export interface ESQLEditorProps {
   disableAutoFocus?: boolean;
 }
 
+export interface JoinIndicesAutocompleteResult {
+  indices: JoinIndexAutocompleteItem[];
+}
+
+export interface JoinIndexAutocompleteItem {
+  name: string;
+  mode: 'lookup' | string;
+  aliases: string[];
+}
+
+export interface EsqlPluginStartBase {
+  getJoinIndicesAutocomplete: () => Promise<JoinIndicesAutocompleteResult>;
+}
+
 export interface ESQLEditorDeps {
   core: CoreStart;
   dataViews: DataViewsPublicPluginStart;
@@ -79,4 +93,5 @@ export interface ESQLEditorDeps {
   indexManagementApiService?: IndexManagementPluginSetup['apiService'];
   fieldsMetadata?: FieldsMetadataPublicStart;
   usageCollection?: UsageCollectionStart;
+  esql?: EsqlPluginStartBase;
 }
diff --git a/src/platform/packages/private/kbn-esql-editor/tsconfig.json b/src/platform/packages/private/kbn-esql-editor/tsconfig.json
@@ -32,7 +32,7 @@
     "@kbn/usage-collection-plugin",
     "@kbn/content-management-favorites-common",
     "@kbn/kibana-utils-plugin",
-    "@kbn/shared-ux-table-persist",
+    "@kbn/shared-ux-table-persist"
   ],
   "exclude": [
     "target/**/*",

diff --git a/src/platform/packages/shared/kbn-esql-ast/index.ts b/src/platform/packages/shared/kbn-esql-ast/index.ts
@@ -30,6 +30,18 @@ export type {
   ESQLAstNode,
 } from './src/types';
 
+export {
+  isBinaryExpression,
+  isColumn,
+  isDoubleLiteral,
+  isFunctionExpression,
+  isIdentifier,
+  isIntegerLiteral,
+  isLiteral,
+  isParamLiteral,
+  isProperNode,
+} from './src/ast/helpers';
+
 export { Builder, type AstNodeParserFields, type AstNodeTemplate } from './src/builder';
 
 export {

diff --git a/src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts b/src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts
@@ -12,6 +12,7 @@ import type {
   ESQLBinaryExpression,
   ESQLColumn,
   ESQLFunction,
+  ESQLIdentifier,
   ESQLIntegerLiteral,
   ESQLLiteral,
   ESQLParamLiteral,
@@ -55,6 +56,9 @@ export const isParamLiteral = (node: unknown): node is ESQLParamLiteral =>
 export const isColumn = (node: unknown): node is ESQLColumn =>
   isProperNode(node) && node.type === 'column';
 
+export const isIdentifier = (node: unknown): node is ESQLIdentifier =>
+  isProperNode(node) && node.type === 'identifier';
+
 /**
  * Returns the group of a binary expression:
  *

diff --git a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/__tests__/helpers.ts b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/__tests__/helpers.ts
@@ -8,8 +8,9 @@
  */
 
 import { camelCase } from 'lodash';
-import { ESQLRealField } from '../validation/types';
+import { ESQLRealField, JoinIndexAutocompleteItem } from '../validation/types';
 import { fieldTypes } from '../definitions/types';
+import { ESQLCallbacks } from '../shared/types';
 
 export const fields: ESQLRealField[] = [
   ...fieldTypes.map((type) => ({ name: `${camelCase(type)}Field`, type })),
@@ -52,9 +53,22 @@ export const policies = [
   },
 ];
 
-export function getCallbackMocks() {
+export const joinIndices: JoinIndexAutocompleteItem[] = [
+  {
+    name: 'join_index',
+    mode: 'lookup',
+    aliases: [],
+  },
+  {
+    name: 'join_index_with_alias',
+    mode: 'lookup',
+    aliases: ['join_index_alias_1', 'join_index_alias_2'],
+  },
+];
+
+export function getCallbackMocks(): ESQLCallbacks {
   return {
-    getColumnsFor: jest.fn(async ({ query }) => {
+    getColumnsFor: jest.fn(async ({ query } = {}) => {
       if (/enrich/.test(query)) {
         return enrichFields;
       }
@@ -75,5 +89,6 @@ export function getCallbackMocks() {
       }))
     ),
     getPolicies: jest.fn(async () => policies),
+    getJoinIndices: jest.fn(async () => ({ indices: joinIndices })),
   };
 }
diff --git a/...esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.join.test.ts b/...esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.join.test.ts
@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { setup, getFieldNamesByType } from './helpers';
+
+describe('autocomplete.suggest', () => {
+  describe('<type> JOIN <index> [ AS <alias> ] ON <condition> [, <condition> [, ...]]', () => {
+    describe('<type> JOIN ...', () => {
+      test('suggests join commands', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | /');
+        const filtered = suggestions
+          .filter((s) => s.label.includes('JOIN'))
+          .map((s) => [s.label, s.text, s.detail]);
+
+        expect(filtered.map((s) => s[0])).toEqual(['LOOKUP JOIN']);
+
+        // TODO: Uncomment when other join types are implemented
+        // expect(filtered.map((s) => s[0])).toEqual(['LEFT JOIN', 'RIGHT JOIN', 'LOOKUP JOIN']);
+      });
+
+      test('can infer full command name based on the unique command type', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKU/');
+        const filtered = suggestions.filter((s) => s.label.toUpperCase() === 'LOOKUP JOIN');
+
+        expect(filtered[0].label).toBe('LOOKUP JOIN');
+      });
+
+      test('suggests command on first character', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP J/');
+        const filtered = suggestions.filter((s) => s.label.toUpperCase() === 'LOOKUP JOIN');
+
+        expect(filtered[0].label).toBe('LOOKUP JOIN');
+      });
+
+      test('returns command description, correct type, and suggestion continuation', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP J/');
+
+        expect(suggestions[0]).toMatchObject({
+          label: 'LOOKUP JOIN',
+          text: 'LOOKUP JOIN $0',
+          detail: 'Join with a "lookup" mode index',
+          kind: 'Keyword',
+        });
+      });
+    });
+
+    describe('... <index> ...', () => {
+      test('can suggest lookup indices (and aliases)', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LEFT JOIN /');
+        const labels = suggestions.map((s) => s.label);
+
+        expect(labels).toEqual([
+          'join_index',
+          'join_index_with_alias',
+          'join_index_alias_1',
+          'join_index_alias_2',
+        ]);
+      });
+
+      test('discriminates between indices and aliases', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LEFT JOIN /');
+        const indices: string[] = suggestions
+          .filter((s) => s.detail === 'Index')
+          .map((s) => s.label)
+          .sort();
+        const aliases: string[] = suggestions
+          .filter((s) => s.detail === 'Alias')
+          .map((s) => s.label)
+          .sort();
+
+        expect(indices).toEqual(['join_index', 'join_index_with_alias']);
+        expect(aliases).toEqual(['join_index_alias_1', 'join_index_alias_2']);
+      });
+    });
+
+    describe('... ON <condition>', () => {
+      test('shows "ON" keyword suggestion', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP JOIN join_index /');
+        const labels = suggestions.map((s) => s.label);
+
+        expect(labels).toEqual(['ON']);
+      });
+
+      test('suggests fields after ON keyword', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP JOIN join_index ON /');
+        const labels = suggestions.map((s) => s.text).sort();
+        const expected = getFieldNamesByType('any')
+          .sort()
+          .map((field) => field + ' ');
+
+        expect(labels).toEqual(expected);
+      });
+
+      test('more field suggestions after comma', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP JOIN join_index ON stringField, /');
+        const labels = suggestions.map((s) => s.text).sort();
+        const expected = getFieldNamesByType('any')
+          .sort()
+          .map((field) => field + ' ');
+
+        expect(labels).toEqual(expected);
+      });
+
+      test('suggests pipe and comma after a field', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP JOIN join_index ON stringField /');
+        const labels = suggestions.map((s) => s.label).sort();
+
+        expect(labels).toEqual([',', '|']);
+      });
+
+      test('suggests pipe and comma after a field (no space)', async () => {
+        const { suggest } = await setup();
+
+        const suggestions = await suggest('FROM index | LOOKUP JOIN join_index ON stringField/');
+        const labels = suggestions.map((s) => s.label).sort();
+
+        expect(labels).toEqual([',', '|']);
+      });
+    });
+  });
+});
diff --git a/.../kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.test.ts b/.../kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.test.ts
@@ -42,6 +42,6 @@ describe('autocomplete.suggest', () => {
     await suggest('sHoW ?');
     await suggest('row ? |');
 
-    expect(callbacks.getColumnsFor.mock.calls.length).toBe(0);
+    expect((callbacks.getColumnsFor as any).mock.calls.length).toBe(0);
   });
 });
diff --git a/...rm/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts b/...rm/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts
@@ -27,6 +27,7 @@ import {
   FunctionReturnType,
   SupportedDataType,
 } from '../../definitions/types';
+import { joinIndices } from '../../__tests__/helpers';
 
 export interface Integration {
   name: string;
@@ -281,6 +282,7 @@ export function createCustomCallbackMocks(
     getColumnsFor: jest.fn(async () => finalColumnsSinceLastCommand),
     getSources: jest.fn(async () => finalSources),
     getPolicies: jest.fn(async () => finalPolicies),
+    getJoinIndices: jest.fn(async () => ({ indices: joinIndices })),
   };
 }