Skip to content

Commit

Permalink
Dbt autocompletion (#94)
Browse files Browse the repository at this point in the history
* Provide autocompletions for dbt models

* Provide autocompletions for dbt sources

* Provide autocompletions for dbt macros

* Provide all available sources in completion requests

* Show package name in source completion suggestions

* Simplify macros definition search

* Simplify dbt autocompletion interface

* Remove unused params

* Provide extended autocompletions for dbt models

* Refactoring

* Provide extended autocompletions for dbt sources

* Refactoring

* Sort macros completions

* Show dbt autocompletions for incomplete jinjas

* Provide sources names as autocompletions

* Provide sources names as autocompletions

* Add e2e tests for sources autocompletions

* Add e2e tests for macros autocompletions

* Fix autocompletion feature for SQL

* Fix autocompletion feature for SQL

* Fix e2e test

* Calculate dbt nodes maps when manifest changes

* Fix dbt autocompletions

* Describe dbt autocompletion feature in README

* Add Completion for dbt feature into features table

* Add unit tests for dbt completion providers

* Fix README

* Remove unused const

* Rename CompletionProvider to SqlCompletionProvider

* Refactoring

* Refactoring

* Clear manifest nodes maps before grouping

* Refactoring

* Refactoring

* Refactoring

* Refactoring

* Refactoring

* Remove unused consts

* Refactoring

* Refactoring

* Refactoring

* Refactoring

* Refactoring

* Create const with macros completions array

* Refactoring

* Refactoring

* Refactoring

* Refactoring

* Add unit tests for cases when completion providers should return undefined

* Improve StringBuilder

* Use vscode type instead of custom one

* Simplify completions provisioning

* Simplify completions provisioning

* Refactoring

* Refactoring
  • Loading branch information
fivetran-dmitryboykov authored Apr 12, 2022
1 parent 4036dcb commit 7e858ea
Show file tree
Hide file tree
Showing 44 changed files with 1,117 additions and 302 deletions.
12 changes: 9 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Also, it provides additional features like [Highlighting errors](#highlighting-e
- [Highlighting errors](#highlighting-errors)
- [Signature help](#signature-help)
- [Completion for SQL](#completion-for-sql)
- [Completion for dbt models](#completion-for-dbt-models)
- [Completion for dbt](#completion-for-dbt)
- [dbt compile preview](#dbt-compile-preview)
- [Information on hover](#information-on-hover)
- [How to use](#how-to-use)
Expand All @@ -31,6 +31,7 @@ Also, it provides additional features like [Highlighting errors](#highlighting-e
| [Highlighting errors](#highlighting-errors) | [BigQuery](https://docs.getdbt.com/reference/warehouse-profiles/bigquery-profile) |
| [Signature help](#signature-help) | All |
| [Completion for SQL](#completion-for-sql) | [BigQuery](https://docs.getdbt.com/reference/warehouse-profiles/bigquery-profile) |
| [Completion for dbt](#completion-for-dbt) | All |
| [Completion for dbt models](#completion-for-dbt-models) | [BigQuery](https://docs.getdbt.com/reference/warehouse-profiles/bigquery-profile) |
| [dbt compile preview](#dbt-compile-preview) | All |
| [Information on hover](#information-on-hover) | [BigQuery](https://docs.getdbt.com/reference/warehouse-profiles/bigquery-profile) |
Expand All @@ -56,9 +57,14 @@ Also, it provides additional features like [Highlighting errors](#highlighting-e

![Completion for SQL](images/Completion.png)

### Completion for dbt models
### Completion for dbt

![Completion for dbt models](images/CompletionForModels.png)
#### Macros
![Completion for macros](images/CompletionForMacros.png)
#### Models
![Completion for models](images/CompletionForModels.png)
#### Sources
![Completion for sources](images/CompletionForSources.png)

### dbt compile preview

Expand Down
2 changes: 1 addition & 1 deletion e2e/projects/postgres/models/active_users_orders_count.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
select u.id, {{ extract_first_name('u.full_name') }} as first_name, {{ extract_last_name('u.full_name') }} as last_name, count(*)
select u.id, {{ extract_first_name('u.full_name') }} as first_name, {{ dbt_postgres_test.extract_last_name('u.full_name') }} as last_name, count(*)
from {{ ref('dbt_postgres_test', 'active_users') }} u
join orders o on u.id = o.user_id
group by u.id, u.full_name
23 changes: 20 additions & 3 deletions e2e/src/asserts.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import assert = require('assert');
import { assertThat, hasSize } from 'hamjest';
import { DefinitionLink, Diagnostic, DiagnosticRelatedInformation, languages, Location, Position, Range, Uri } from 'vscode';
import { PREVIEW_URI, sleep, triggerDefinition } from './helper';
import { assertThat, greaterThanOrEqualTo, hasSize } from 'hamjest';
import { CompletionItem, DefinitionLink, Diagnostic, DiagnosticRelatedInformation, languages, Location, Position, Range, Uri } from 'vscode';
import { PREVIEW_URI, sleep, triggerCompletion, triggerDefinition } from './helper';

export async function assertDiagnostics(uri: Uri, diagnostics: Diagnostic[]): Promise<void> {
await sleep(100);
Expand Down Expand Up @@ -57,3 +57,20 @@ export async function assertDefinitions(docUri: Uri, position: Position, expecte
assertThat(definitions[i].targetSelectionRange, expectedDefinitions[i].targetSelectionRange);
}
}

export async function assertCompletions(
docUri: Uri,
position: Position,
expectedCompletionList: CompletionItem[],
triggerChar?: string,
): Promise<void> {
const actualCompletionList = await triggerCompletion(docUri, position, triggerChar);

assertThat(actualCompletionList.items.length, greaterThanOrEqualTo(expectedCompletionList.length));
expectedCompletionList.forEach((expectedItem, i) => {
const actualItem = actualCompletionList.items[i];
assertThat(actualItem.label, expectedItem.label);
assertThat(actualItem.kind, expectedItem.kind);
assertThat(actualItem.insertText, expectedItem.insertText ?? expectedItem.label);
});
}
83 changes: 38 additions & 45 deletions e2e/src/completion.spec.ts
Original file line number Diff line number Diff line change
@@ -1,59 +1,54 @@
import * as vscode from 'vscode';
import { activateAndWait, getDocUri, replaceText, testCompletion } from './helper';
import { assertCompletions } from './asserts';
import { activateAndWait, getDocUri, replaceText } from './helper';

suite('Should do completion', () => {
test('Should suggest table columns', async () => {
const docUri = getDocUri('simple_select.sql');
await activateAndWait(docUri);
await testCompletion(docUri, new vscode.Position(0, 8), {
items: [
{ label: 'date', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'time', kind: vscode.CompletionItemKind.Value },
],
});
await assertCompletions(docUri, new vscode.Position(0, 8), [
{ label: 'date', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'time', kind: vscode.CompletionItemKind.Value },
]);
});

test('Should suggest columns for both tables', async () => {
const docUri = getDocUri('join_tables.sql');
await activateAndWait(docUri);
await testCompletion(docUri, new vscode.Position(0, 8), {
items: [
{ label: 'test_table1.date', kind: vscode.CompletionItemKind.Value },
{ label: 'test_table1.id', kind: vscode.CompletionItemKind.Value },
{ label: 'test_table1.name', kind: vscode.CompletionItemKind.Value },
{ label: 'test_table1.time', kind: vscode.CompletionItemKind.Value },
await assertCompletions(docUri, new vscode.Position(0, 8), [
{ label: 'test_table1.date', kind: vscode.CompletionItemKind.Value },
{ label: 'test_table1.id', kind: vscode.CompletionItemKind.Value },
{ label: 'test_table1.name', kind: vscode.CompletionItemKind.Value },
{ label: 'test_table1.time', kind: vscode.CompletionItemKind.Value },

{ label: 'users.division', kind: vscode.CompletionItemKind.Value },
{ label: 'users.email', kind: vscode.CompletionItemKind.Value },
{ label: 'users.id', kind: vscode.CompletionItemKind.Value },
{ label: 'users.name', kind: vscode.CompletionItemKind.Value },
{ label: 'users.phone', kind: vscode.CompletionItemKind.Value },
{ label: 'users.profile_id', kind: vscode.CompletionItemKind.Value },
{ label: 'users.role', kind: vscode.CompletionItemKind.Value },
],
});
{ label: 'users.division', kind: vscode.CompletionItemKind.Value },
{ label: 'users.email', kind: vscode.CompletionItemKind.Value },
{ label: 'users.id', kind: vscode.CompletionItemKind.Value },
{ label: 'users.name', kind: vscode.CompletionItemKind.Value },
{ label: 'users.phone', kind: vscode.CompletionItemKind.Value },
{ label: 'users.profile_id', kind: vscode.CompletionItemKind.Value },
{ label: 'users.role', kind: vscode.CompletionItemKind.Value },
]);
});

test('Should suggest columns for table name after press .', async () => {
const docUri = getDocUri('join_tables.sql');
await activateAndWait(docUri);
await replaceText('*', 'users.');
await testCompletion(
await assertCompletions(
docUri,
new vscode.Position(0, 13),
{
items: [
{ label: 'division', kind: vscode.CompletionItemKind.Value },
{ label: 'email', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'phone', kind: vscode.CompletionItemKind.Value },
{ label: 'profile_id', kind: vscode.CompletionItemKind.Value },
{ label: 'role', kind: vscode.CompletionItemKind.Value },
],
},
[
{ label: 'division', kind: vscode.CompletionItemKind.Value },
{ label: 'email', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'phone', kind: vscode.CompletionItemKind.Value },
{ label: 'profile_id', kind: vscode.CompletionItemKind.Value },
{ label: 'role', kind: vscode.CompletionItemKind.Value },
],
'.',
);
});
Expand All @@ -63,17 +58,15 @@ suite('Should do completion', () => {
await activateAndWait(docUri);
await replaceText('*', 't.');

await testCompletion(
await assertCompletions(
docUri,
new vscode.Position(0, 9),
{
items: [
{ label: 'date', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'time', kind: vscode.CompletionItemKind.Value },
],
},
[
{ label: 'date', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'time', kind: vscode.CompletionItemKind.Value },
],
'.',
);
});
Expand Down
39 changes: 39 additions & 0 deletions e2e/src/completion/completion_macros.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { assertThat, defined } from 'hamjest';
import { CompletionItem, CompletionItemKind, Position } from 'vscode';
import { assertCompletions } from '../asserts';
import { activateAndWait, getCustomDocUri, triggerCompletion } from '../helper';

suite('Should suggest macros completions', () => {
const PROJECT_FILE_NAME = 'postgres/models/active_users_orders_count.sql';

const MACROS_COMPLETIONS = [
['extract_first_name', 'extract_first_name'],
['extract_last_name', 'extract_last_name'],
];

test('Should suggest macros', async () => {
// arrange
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);

// act
const actualCompletionList = await triggerCompletion(docUri, new Position(0, 15), 'e');

// assert
const expectedCompletions = getMacrosCompletionList();
expectedCompletions.forEach(c => {
const actualCompletion = actualCompletionList.items.find(a => a.label === c.label && a.insertText === c.insertText);
assertThat(actualCompletion, defined());
});
});

test('Should suggest macros from package', async () => {
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);
await assertCompletions(docUri, new Position(0, 89), getMacrosCompletionList());
});

function getMacrosCompletionList(): CompletionItem[] {
return MACROS_COMPLETIONS.map<CompletionItem>(c => ({ label: c[0], insertText: c[1], kind: CompletionItemKind.Value }));
}
});
Original file line number Diff line number Diff line change
@@ -1,30 +1,38 @@
import { assertThat, contains, not } from 'hamjest';
import * as vscode from 'vscode';
import { CompletionItem } from 'vscode';
import { activateAndWait, getCustomDocUri, setTestContent, testCompletion, triggerCompletion } from './helper';
import { CompletionItem, CompletionItemKind } from 'vscode';
import { assertCompletions } from '../asserts';
import { activateAndWait, getCustomDocUri, getTextInQuotesIfNeeded, setTestContent, triggerCompletion } from '../helper';

suite('Should do completion inside jinjas expression', () => {
suite('Should suggest model completions', () => {
const PROJECT_FILE_NAME = 'completion-jinja/models/completion_jinja.sql';

const MODELS_COMPLETIONS = [
['(my_new_project) completion_jinja', 'completion_jinja'],
['(my_new_project) join_ref', 'join_ref'],
['(my_new_project) test_table1', 'test_table1'],
['(my_new_project) users', 'users'],
];

test('Should suggest models for ref function by pressing "("', async () => {
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);
await setTestContent('select * from {{ref(');
await testCompletion(docUri, new vscode.Position(0, 20), getCompletionList(true), '(');
await assertCompletions(docUri, new vscode.Position(0, 20), getCompletionList(true), '(');
});

test('Should suggest models for ref function', async () => {
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);
await setTestContent('select * from {{ref(');
await testCompletion(docUri, new vscode.Position(0, 20), getCompletionList(true));
await assertCompletions(docUri, new vscode.Position(0, 20), getCompletionList(true));
});

test("Should suggest models for ref function by pressing ' ", async () => {
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);
await setTestContent(`select * from {{ref('`);
await testCompletion(docUri, new vscode.Position(0, 21), getCompletionList(false), "'");
await assertCompletions(docUri, new vscode.Position(0, 21), getCompletionList(false), "'");
});

test('Should not suggest models outside jinja', async () => {
Expand All @@ -39,15 +47,15 @@ suite('Should do completion inside jinjas expression', () => {
// assert
actualCompletionList.items.forEach(i => i.label instanceof String);
const actualLabels = actualCompletionList.items.map(i => i.label as string);
getCompletionList(false).items.forEach(i => assertThat(actualLabels, not(contains(i.label as string))));
getCompletionList(true).items.forEach(i => assertThat(actualLabels, not(contains(i.label as string))));
getCompletionList(false).forEach(i => assertThat(actualLabels, not(contains(i.label as string))));
getCompletionList(true).forEach(i => assertThat(actualLabels, not(contains(i.label as string))));
});

function getCompletionList(withQuotes: boolean): { items: vscode.CompletionItem[] } {
return { items: getLabels().map<CompletionItem>(l => ({ label: withQuotes ? `'${l}'` : l, kind: vscode.CompletionItemKind.Value })) };
}

function getLabels(): string[] {
return ['completion_jinja', 'join_ref', 'test_table1', 'users'];
function getCompletionList(withQuotes: boolean): CompletionItem[] {
return MODELS_COMPLETIONS.map<CompletionItem>(c => ({
label: c[0],
insertText: getTextInQuotesIfNeeded(c[1], withQuotes),
kind: CompletionItemKind.Value,
}));
}
});
33 changes: 33 additions & 0 deletions e2e/src/completion/completion_sources.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { CompletionItem, CompletionItemKind, Position } from 'vscode';
import { assertCompletions } from '../asserts';
import { activateAndWait, getCustomDocUri, getTextInQuotesIfNeeded } from '../helper';

suite('Should suggest sources completions', () => {
const PROJECT_FILE_NAME = 'postgres/models/active_users.sql';

const SOURCES_COMPLETIONS: [string, string][] = [['(dbt_postgres_test) users_orders', 'users_orders']];
const TABLES_COMPLETIONS: [string, string][] = [
['orders', 'orders'],
['users', 'users'],
];

test('Should suggest sources', async () => {
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);
await assertCompletions(docUri, new Position(1, 16), getSourcesCompletionList(SOURCES_COMPLETIONS, false));
});

test('Should suggest source tables', async () => {
const docUri = getCustomDocUri(PROJECT_FILE_NAME);
await activateAndWait(docUri);
await assertCompletions(docUri, new Position(1, 32), getSourcesCompletionList(TABLES_COMPLETIONS, false));
});

function getSourcesCompletionList(completions: [string, string][], withQuotes: boolean): CompletionItem[] {
return completions.map<CompletionItem>(c => ({
label: c[0],
insertText: getTextInQuotesIfNeeded(c[1], withQuotes),
kind: CompletionItemKind.Value,
}));
}
});
25 changes: 12 additions & 13 deletions e2e/src/completion_alias.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as vscode from 'vscode';
import { activateAndWait, getCustomDocUri, testCompletion } from './helper';
import { assertCompletions } from './asserts';
import { activateAndWait, getCustomDocUri } from './helper';

suite('Should suggest completions after ref aliases', () => {
const PROJECT_FILE_NAME = 'completion-jinja/models/join_ref.sql';
Expand All @@ -18,20 +19,18 @@ suite('Should suggest completions after ref aliases', () => {
await activateAndWait(docUri);

// act
await testCompletion(
await assertCompletions(
docUri,
position,
{
items: [
{ label: 'division', kind: vscode.CompletionItemKind.Value },
{ label: 'email', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'phone', kind: vscode.CompletionItemKind.Value },
{ label: 'profile_id', kind: vscode.CompletionItemKind.Value },
{ label: 'role', kind: vscode.CompletionItemKind.Value },
],
},
[
{ label: 'division', kind: vscode.CompletionItemKind.Value },
{ label: 'email', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'phone', kind: vscode.CompletionItemKind.Value },
{ label: 'profile_id', kind: vscode.CompletionItemKind.Value },
{ label: 'role', kind: vscode.CompletionItemKind.Value },
],
'.',
);
}
Expand Down
17 changes: 8 additions & 9 deletions e2e/src/completion_dbt.spec.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
import * as vscode from 'vscode';
import { activateAndWait, getDocUri, testCompletion } from './helper';
import { assertCompletions } from './asserts';
import { activateAndWait, getDocUri } from './helper';

suite('Should do completion with jinjas in query', () => {
test('Should suggest table columns', async () => {
const docUri = getDocUri('simple_select_dbt.sql');
await activateAndWait(docUri);

await testCompletion(docUri, new vscode.Position(0, 8), {
items: [
{ label: 'date', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'time', kind: vscode.CompletionItemKind.Value },
],
});
await assertCompletions(docUri, new vscode.Position(0, 8), [
{ label: 'date', kind: vscode.CompletionItemKind.Value },
{ label: 'id', kind: vscode.CompletionItemKind.Value },
{ label: 'name', kind: vscode.CompletionItemKind.Value },
{ label: 'time', kind: vscode.CompletionItemKind.Value },
]);
});
});
Loading

0 comments on commit 7e858ea

Please sign in to comment.