diff --git a/examples/requirements/test/validator.test.ts b/examples/requirements/test/validator.test.ts index 7b32aa7c6..e523bdb6a 100644 --- a/examples/requirements/test/validator.test.ts +++ b/examples/requirements/test/validator.test.ts @@ -13,13 +13,13 @@ import { NodeFileSystem } from 'langium/node'; describe('A requirement identifier and a test identifier shall contain a number.', () => { test('T001_good_case', async () => { const services = createRequirementsAndTestsLangServices(NodeFileSystem); - const [mainDoc,allDocs] = await extractDocuments( + const [mainDoc, allDocs] = await extractDocuments( path.join(__dirname, 'files', 'good', 'requirements.req'), services.requirements ); expect((mainDoc.diagnostics ?? [])).toEqual([]); expect(allDocs.length).toEqual(3); - allDocs.forEach(doc=>{ + allDocs.forEach(doc => { expect((doc.diagnostics ?? [])).toEqual([]); }); }); @@ -35,7 +35,7 @@ describe('A requirement identifier shall contain a number.', () => { expect(mainDoc.diagnostics ?? []).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Requirement name ReqIdABC_reqID should container a number'), - range: expect.objectContaining({start:expect.objectContaining({line: 2})}) // zero based + range: expect.objectContaining({ start: expect.objectContaining({ line: 2 }) }) // zero based }) ])); @@ -45,17 +45,17 @@ describe('A requirement identifier shall contain a number.', () => { describe('A test identifier shall contain a number.', () => { test('T003_badTstId: bad case', async () => { const services = createRequirementsAndTestsLangServices(NodeFileSystem); - const [,allDocs] = await extractDocuments( + const [, allDocs] = await extractDocuments( path.join(__dirname, 'files', 'bad1', 'requirements.req'), services.requirements ); - const doc = allDocs.find(doc=>/tests_part1.tst/.test(doc.uri.fsPath)); + const doc = allDocs.find(doc => /tests_part1.tst/.test(doc.uri.fsPath)); expect(doc).toBeDefined(); if (!doc) throw new Error('impossible'); expect(doc.diagnostics ?? []).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Test name TA should container a number.'), - range: expect.objectContaining({start:expect.objectContaining({line: 1})}) // zero based + range: expect.objectContaining({ start: expect.objectContaining({ line: 1 }) }) // zero based }) ])); }); @@ -71,7 +71,7 @@ describe('A requirement shall be covered by at least one test.', () => { expect(mainDoc.diagnostics ?? []).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Requirement ReqId004_unicorn not covered by a test.'), - range: expect.objectContaining({start:expect.objectContaining({line: 4})}) // zero based + range: expect.objectContaining({ start: expect.objectContaining({ line: 4 }) }) // zero based }) ])); }); @@ -80,28 +80,32 @@ describe('A requirement shall be covered by at least one test.', () => { describe('A referenced environment in a test must be found in one of the referenced requirements.', () => { test('referenced environment test', async () => { const services = createRequirementsAndTestsLangServices(NodeFileSystem); - const [,allDocs] = await extractDocuments( + const [, allDocs] = await extractDocuments( path.join(__dirname, 'files', 'bad2', 'requirements.req'), services.requirements ); - const doc = allDocs.find(doc=>/tests_part1.tst/.test(doc.uri.fsPath)); + const doc = allDocs.find(doc => /tests_part1.tst/.test(doc.uri.fsPath)); expect(doc).toBeDefined(); if (!doc) throw new Error('impossible'); expect((doc.diagnostics ?? [])).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Test T002_badReqId references environment Linux_x86 which is used in any referenced requirement.'), - range: expect.objectContaining({start:expect.objectContaining({ - line: 3, - character: 65 - })}) // zero based + range: expect.objectContaining({ + start: expect.objectContaining({ + line: 3, + character: 65 + }) + }) // zero based }) ])); expect((doc.diagnostics ?? [])).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Test T004_cov references environment Linux_x86 which is used in any referenced requirement.'), - range: expect.objectContaining({start:expect.objectContaining({ - line: 5 - })}) // zero based + range: expect.objectContaining({ + start: expect.objectContaining({ + line: 5 + }) + }) // zero based }) ])); diff --git a/packages/langium/src/default-module.ts b/packages/langium/src/default-module.ts index b96c1a097..0a637b648 100644 --- a/packages/langium/src/default-module.ts +++ b/packages/langium/src/default-module.ts @@ -35,6 +35,7 @@ import { LangiumParserErrorMessageProvider } from './parser/langium-parser.js'; import { DefaultAsyncParser } from './parser/async-parser.js'; import { DefaultWorkspaceLock } from './workspace/workspace-lock.js'; import { DefaultHydrator } from './serializer/hydrator.js'; +import { DefaultEnvironment } from './workspace/environment.js'; /** * Context required for creating the default language-specific dependency injection module. @@ -117,7 +118,8 @@ export function createDefaultSharedCoreModule(context: DefaultSharedCoreModuleCo WorkspaceManager: (services) => new DefaultWorkspaceManager(services), FileSystemProvider: (services) => context.fileSystemProvider(services), WorkspaceLock: () => new DefaultWorkspaceLock(), - ConfigurationProvider: (services) => new DefaultConfigurationProvider(services) + ConfigurationProvider: (services) => new DefaultConfigurationProvider(services), + Environment: () => new DefaultEnvironment() } }; } diff --git a/packages/langium/src/documentation/comment-provider.ts b/packages/langium/src/documentation/comment-provider.ts index 1d9c58d15..06b425d4f 100644 --- a/packages/langium/src/documentation/comment-provider.ts +++ b/packages/langium/src/documentation/comment-provider.ts @@ -28,9 +28,12 @@ export class DefaultCommentProvider implements CommentProvider { this.grammarConfig = () => services.parser.GrammarConfig; } getComment(node: AstNode): string | undefined { - if(isAstNodeWithComment(node)) { + if (isAstNodeWithComment(node)) { return node.$comment; + } else if (node.$segments && 'comment' in node.$segments) { + return node.$segments.comment; + } else { + return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text; } - return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text; } } diff --git a/packages/langium/src/lsp/call-hierarchy-provider.ts b/packages/langium/src/lsp/call-hierarchy-provider.ts index 3c509ee43..ef75d65ff 100644 --- a/packages/langium/src/lsp/call-hierarchy-provider.ts +++ b/packages/langium/src/lsp/call-hierarchy-provider.ts @@ -54,12 +54,12 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv return undefined; } - const declarationNode = this.references.findDeclarationNode(targetNode); + const declarationNode = this.references.findDeclaration(targetNode); if (!declarationNode) { return undefined; } - return this.getCallHierarchyItems(declarationNode.astNode, document); + return this.getCallHierarchyItems(declarationNode, document); } protected getCallHierarchyItems(targetNode: AstNode, document: LangiumDocument): CallHierarchyItem[] | undefined { diff --git a/packages/langium/src/lsp/definition-provider.ts b/packages/langium/src/lsp/definition-provider.ts index 603bc0ffd..4d5adefee 100644 --- a/packages/langium/src/lsp/definition-provider.ts +++ b/packages/langium/src/lsp/definition-provider.ts @@ -10,7 +10,7 @@ import type { GrammarConfig } from '../languages/grammar-config.js'; import type { NameProvider } from '../references/name-provider.js'; import type { References } from '../references/references.js'; import type { LangiumServices } from './lsp-services.js'; -import type { CstNode } from '../syntax-tree.js'; +import type { AstNode, CstNode } from '../syntax-tree.js'; import type { MaybePromise } from '../utils/promise-utils.js'; import type { LangiumDocument } from '../workspace/documents.js'; import { LocationLink } from 'vscode-languageserver'; @@ -37,7 +37,7 @@ export interface DefinitionProvider { export interface GoToLink { source: CstNode - target: CstNode + target: AstNode targetDocument: LangiumDocument } @@ -67,21 +67,25 @@ export class DefaultDefinitionProvider implements DefinitionProvider { protected collectLocationLinks(sourceCstNode: CstNode, _params: DefinitionParams): MaybePromise { const goToLink = this.findLink(sourceCstNode); - if (goToLink) { - return [LocationLink.create( - goToLink.targetDocument.textDocument.uri, - (goToLink.target.astNode.$cstNode ?? goToLink.target).range, - goToLink.target.range, - goToLink.source.range - )]; + if (goToLink && goToLink.target.$segments) { + const name = this.nameProvider.getNameProperty(goToLink.target); + if (name) { + const nameSegment = goToLink.target.$segments.properties.get(name); + return nameSegment.map(segment => LocationLink.create( + goToLink.targetDocument.textDocument.uri, + goToLink.target.$segments!.full.range, + segment.range, + goToLink.source.range + )); + } } return undefined; } protected findLink(source: CstNode): GoToLink | undefined { - const target = this.references.findDeclarationNode(source); - if (target?.astNode) { - const targetDocument = getDocument(target.astNode); + const target = this.references.findDeclaration(source); + if (target) { + const targetDocument = getDocument(target); if (target && targetDocument) { return { source, target, targetDocument }; } diff --git a/packages/langium/src/lsp/document-update-handler.ts b/packages/langium/src/lsp/document-update-handler.ts index e158cc9f0..d9ad4b9cd 100644 --- a/packages/langium/src/lsp/document-update-handler.ts +++ b/packages/langium/src/lsp/document-update-handler.ts @@ -4,17 +4,18 @@ * terms of the MIT License, which is available in the project root. ******************************************************************************/ -import type { TextDocumentWillSaveEvent, DidChangeWatchedFilesParams, DidChangeWatchedFilesRegistrationOptions, TextDocumentChangeEvent, TextEdit } from 'vscode-languageserver'; +import type { TextDocumentWillSaveEvent, DidChangeWatchedFilesParams, DidChangeWatchedFilesRegistrationOptions, TextDocumentChangeEvent, TextEdit, Connection } from 'vscode-languageserver'; import { DidChangeWatchedFilesNotification, FileChangeType } from 'vscode-languageserver'; import { stream } from '../utils/stream.js'; import { URI } from '../utils/uri-utils.js'; import type { DocumentBuilder } from '../workspace/document-builder.js'; -import type { TextDocument } from '../workspace/documents.js'; +import type { LangiumDocuments, TextDocument } from '../workspace/documents.js'; import type { WorkspaceLock } from '../workspace/workspace-lock.js'; import type { LangiumSharedServices } from './lsp-services.js'; import type { WorkspaceManager } from '../workspace/workspace-manager.js'; import type { ServiceRegistry } from '../service-registry.js'; import type { MaybePromise } from '../utils/promise-utils.js'; +import { discardCst } from '../utils/cst-utils.js'; /** * Shared service for handling text document changes and watching relevant files. @@ -71,6 +72,8 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { protected readonly workspaceManager: WorkspaceManager; protected readonly documentBuilder: DocumentBuilder; protected readonly workspaceLock: WorkspaceLock; + protected readonly documents: LangiumDocuments; + protected readonly connection: Connection | undefined; protected readonly serviceRegistry: ServiceRegistry; constructor(services: LangiumSharedServices) { @@ -78,6 +81,8 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { this.documentBuilder = services.workspace.DocumentBuilder; this.workspaceLock = services.workspace.WorkspaceLock; this.serviceRegistry = services.ServiceRegistry; + this.documents = services.workspace.LangiumDocuments; + this.connection = services.lsp.Connection; let canRegisterFileWatcher = false; services.lsp.LanguageServer.onInitialize(params => { @@ -98,7 +103,6 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { .distinct() .toArray(); if (fileExtensions.length > 0) { - const connection = services.lsp.Connection; const options: DidChangeWatchedFilesRegistrationOptions = { watchers: [{ globPattern: fileExtensions.length === 1 @@ -106,7 +110,7 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { : `**/*.{${fileExtensions.join(',')}}` }] }; - connection?.client.register(DidChangeWatchedFilesNotification.type, options); + this.connection?.client.register(DidChangeWatchedFilesNotification.type, options); } } @@ -141,4 +145,18 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { .toArray(); this.fireDocumentUpdate(changedUris, deletedUris); } + + didCloseDocument(event: TextDocumentChangeEvent): void { + const document = this.documents.getDocument(URI.parse(event.document.uri)); + if (document) { + // Preserve memory by discarding the CST of the document + // Whenever the user reopens the document, the CST will be rebuilt + discardCst(document.parseResult.value); + } + // Discard the diagnostics for the closed document + this.connection?.sendDiagnostics({ + uri: event.document.uri, + diagnostics: [] + }); + } } diff --git a/packages/langium/src/lsp/language-server.ts b/packages/langium/src/lsp/language-server.ts index 76cee87ab..3cc5af199 100644 --- a/packages/langium/src/lsp/language-server.ts +++ b/packages/langium/src/lsp/language-server.ts @@ -198,6 +198,7 @@ export class DefaultLanguageServer implements LanguageServer { } protected fireInitializeOnDefaultServices(params: InitializeParams): void { + this.services.workspace.Environment.initialize(params); this.services.workspace.ConfigurationProvider.initialize(params); this.services.workspace.WorkspaceManager.initialize(params); } diff --git a/packages/langium/src/lsp/type-hierarchy-provider.ts b/packages/langium/src/lsp/type-hierarchy-provider.ts index a9eec8308..f53a6d228 100644 --- a/packages/langium/src/lsp/type-hierarchy-provider.ts +++ b/packages/langium/src/lsp/type-hierarchy-provider.ts @@ -58,12 +58,12 @@ export abstract class AbstractTypeHierarchyProvider implements TypeHierarchyProv return undefined; } - const declarationNode = this.references.findDeclarationNode(targetNode); + const declarationNode = this.references.findDeclaration(targetNode); if (!declarationNode) { return undefined; } - return this.getTypeHierarchyItems(declarationNode.astNode, document); + return this.getTypeHierarchyItems(declarationNode, document); } protected getTypeHierarchyItems(targetNode: AstNode, document: LangiumDocument): TypeHierarchyItem[] | undefined { diff --git a/packages/langium/src/parser/async-parser.ts b/packages/langium/src/parser/async-parser.ts index 0473aa9b0..533b7faaf 100644 --- a/packages/langium/src/parser/async-parser.ts +++ b/packages/langium/src/parser/async-parser.ts @@ -7,7 +7,7 @@ import type { CancellationToken } from '../utils/cancellation.js'; import type { LangiumCoreServices } from '../services.js'; import type { AstNode } from '../syntax-tree.js'; -import type { LangiumParser, ParseResult } from './langium-parser.js'; +import type { LangiumParser, ParseResult, ParserOptions } from './langium-parser.js'; import type { Hydrator } from '../serializer/hydrator.js'; import type { Event } from '../utils/event.js'; import { Deferred, OperationCancelled } from '../utils/promise-utils.js'; @@ -30,7 +30,7 @@ export interface AsyncParser { * * @throws `OperationCancelled` if the parsing process is cancelled. */ - parse(text: string, cancelToken: CancellationToken): Promise>; + parse(text: string, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise>; } /** @@ -47,8 +47,8 @@ export class DefaultAsyncParser implements AsyncParser { this.syncParser = services.parser.LangiumParser; } - parse(text: string, _cancelToken: CancellationToken): Promise> { - return Promise.resolve(this.syncParser.parse(text)); + parse(text: string, options: ParserOptions | undefined, _cancelToken: CancellationToken): Promise> { + return Promise.resolve(this.syncParser.parse(text, options)); } } @@ -89,7 +89,7 @@ export abstract class AbstractThreadedAsyncParser implements AsyncParser { } } - async parse(text: string, cancelToken: CancellationToken): Promise> { + async parse(text: string, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise> { const worker = await this.acquireParserWorker(cancelToken); const deferred = new Deferred>(); let timeout: NodeJS.Timeout | undefined; @@ -101,7 +101,7 @@ export abstract class AbstractThreadedAsyncParser implements AsyncParser { this.terminateWorker(worker); }, this.terminationDelay); }); - worker.parse(text).then(result => { + worker.parse(text, options).then(result => { const hydrated = this.hydrator.hydrate(result); deferred.resolve(hydrated); }).catch(err => { @@ -194,13 +194,13 @@ export class ParserWorker { this.onReadyEmitter.fire(); } - parse(text: string): Promise { + parse(text: string, options: ParserOptions | undefined): Promise { if (this._parsing) { throw new Error('Parser worker is busy'); } this._parsing = true; this.deferred = new Deferred(); - this.sendMessage(text); + this.sendMessage([text, options]); return this.deferred.promise; } } diff --git a/packages/langium/src/parser/cst-node-builder.ts b/packages/langium/src/parser/cst-node-builder.ts index 72c89a160..3ec0f4b9d 100644 --- a/packages/langium/src/parser/cst-node-builder.ts +++ b/packages/langium/src/parser/cst-node-builder.ts @@ -87,20 +87,20 @@ export class CstNodeBuilder { } } - construct(item: { $type: string | symbol | undefined, $cstNode: CstNode }): void { + construct(item: { $type: string | symbol | undefined, $cstNode: CstNode }): CstNode { const current: CstNode = this.current; // The specified item could be a datatype ($type is symbol) or a fragment ($type is undefined) // Only if the $type is a string, we actually assign the element if (typeof item.$type === 'string') { this.current.astNode = item; } - item.$cstNode = current; const node = this.nodeStack.pop(); // Empty composite nodes are not valid // Simply remove the node from the tree if (node?.content.length === 0) { this.removeNode(node); } + return current; } } diff --git a/packages/langium/src/parser/langium-parser.ts b/packages/langium/src/parser/langium-parser.ts index 03a28b207..6d1e6277e 100644 --- a/packages/langium/src/parser/langium-parser.ts +++ b/packages/langium/src/parser/langium-parser.ts @@ -20,6 +20,9 @@ import { getExplicitRuleType, isDataTypeRule } from '../utils/grammar-utils.js'; import { assignMandatoryProperties, getContainerOfType, linkContentToContainer } from '../utils/ast-utils.js'; import { CstNodeBuilder } from './cst-node-builder.js'; import type { LexingReport } from './token-builder.js'; +import { toDocumentSegment } from '../utils/cst-utils.js'; +import type { CommentProvider } from '../documentation/comment-provider.js'; +import { MultiMap } from '../utils/collections.js'; export type ParseResult = { value: T, @@ -99,10 +102,6 @@ export interface BaseParser { * Executes a grammar action that modifies the currently active AST node */ action($type: string, action: Action): void; - /** - * Finishes construction of the current AST node. Only used by the AST parser. - */ - construct(): unknown; /** * Whether the parser is currently actually in use or in "recording mode". * Recording mode is activated once when the parser is analyzing itself. @@ -126,6 +125,7 @@ const withRuleSuffix = (name: string): string => name.endsWith(ruleSuffix) ? nam export abstract class AbstractLangiumParser implements BaseParser { protected readonly lexer: Lexer; + protected readonly commentProvider: CommentProvider; protected readonly wrapper: ChevrotainWrapper; protected _unorderedGroups: Map = new Map(); @@ -141,6 +141,7 @@ export abstract class AbstractLangiumParser implements BaseParser { skipValidations: production, errorMessageProvider: services.parser.ParserErrorMessageProvider }); + this.commentProvider = services.documentation.CommentProvider; } alternatives(idx: number, choices: Array>): void { @@ -163,7 +164,6 @@ export abstract class AbstractLangiumParser implements BaseParser { abstract consume(idx: number, tokenType: TokenType, feature: AbstractElement): void; abstract subrule(idx: number, rule: RuleResult, fragment: boolean, feature: AbstractElement, args: Args): void; abstract action($type: string, action: Action): void; - abstract construct(): unknown; getRule(name: string): RuleResult | undefined { return this.allRules.get(name); @@ -186,8 +186,14 @@ export abstract class AbstractLangiumParser implements BaseParser { } } +export enum CstParserMode { + Retain, + Discard +} + export interface ParserOptions { - rule?: string + rule?: string; + cst?: CstParserMode; } export class LangiumParser extends AbstractLangiumParser { @@ -198,6 +204,7 @@ export class LangiumParser extends AbstractLangiumParser { private lexerResult?: LexerResult; private stack: any[] = []; private assignmentMap = new Map(); + private currentMode: CstParserMode = CstParserMode.Retain; private get current(): any { return this.stack[this.stack.length - 1]; @@ -232,6 +239,7 @@ export class LangiumParser extends AbstractLangiumParser { } parse(input: string, options: ParserOptions = {}): ParseResult { + this.currentMode = options.cst ?? CstParserMode.Retain; this.nodeBuilder.buildRootNode(input); const lexerResult = this.lexerResult = this.lexer.tokenize(input); this.wrapper.input = lexerResult.tokens; @@ -240,7 +248,6 @@ export class LangiumParser extends AbstractLangiumParser { throw new Error(options.rule ? `No rule found with name '${options.rule}'` : 'No main rule available.'); } const result = ruleMethod.call(this.wrapper, {}); - this.nodeBuilder.addHiddenNodes(lexerResult.hidden); this.unorderedGroups.clear(); this.lexerResult = undefined; return { @@ -256,7 +263,7 @@ export class LangiumParser extends AbstractLangiumParser { // Only create a new AST node in case the calling rule is not a fragment rule const createNode = !this.isRecording() && $type !== undefined; if (createNode) { - const node: any = { $type }; + const node: any = { $type, $segments: { properties: new MultiMap() } }; this.stack.push(node); if ($type === DatatypeSymbol) { node.value = ''; @@ -269,7 +276,7 @@ export class LangiumParser extends AbstractLangiumParser { result = undefined; } if (result === undefined && createNode) { - result = this.construct(); + result = this.construct()[0]; } return result; }; @@ -362,33 +369,41 @@ export class LangiumParser extends AbstractLangiumParser { if (!this.isRecording()) { let last = this.current; if (action.feature && action.operator) { - last = this.construct(); - this.nodeBuilder.removeNode(last.$cstNode); + const [constructed, cstNode] = this.construct(); + last = constructed; const node = this.nodeBuilder.buildCompositeNode(action); - node.content.push(last.$cstNode); - const newItem = { $type }; + this.nodeBuilder.removeNode(cstNode); + node.content.push(cstNode); + const newItem = { $type, $segments: { properties: new MultiMap() } }; this.stack.push(newItem); - this.assign(action.operator, action.feature, last, last.$cstNode, false); + this.assign(action.operator, action.feature, last, cstNode, false); } else { last.$type = $type; } } } - construct(): unknown { + construct(): [unknown, CstNode] { if (this.isRecording()) { - return undefined; + return [undefined, undefined!]; } const obj = this.current; linkContentToContainer(obj); - this.nodeBuilder.construct(obj); + const cstNode = this.nodeBuilder.construct(obj); this.stack.pop(); if (isDataTypeNode(obj)) { - return this.converter.convert(obj.value, obj.$cstNode); + return [this.converter.convert(obj.value, cstNode), cstNode]; } else { assignMandatoryProperties(this.astReflection, obj); } - return obj; + obj.$cstNode = cstNode; + delete obj.$segments.comment; + obj.$segments.comment = this.commentProvider.getComment(obj); + obj.$segments.full = toDocumentSegment(cstNode); + if (this.currentMode === CstParserMode.Discard) { + obj.$cstNode = undefined; + } + return [obj, cstNode]; } private getAssignment(feature: AbstractElement): AssignmentElement { @@ -406,24 +421,29 @@ export class LangiumParser extends AbstractLangiumParser { const obj = this.current; let item: unknown; if (isCrossRef && typeof value === 'string') { - item = this.linker.buildReference(obj, feature, cstNode, value); + item = this.linker.buildReference(obj, feature, this.currentMode === CstParserMode.Retain ? cstNode : undefined, value); } else { item = value; } + const segment = toDocumentSegment(cstNode); switch (operator) { case '=': { obj[feature] = item; + obj.$segments.properties.add(feature, segment); break; } case '?=': { obj[feature] = true; + obj.$segments.properties.add(feature, segment); break; } case '+=': { if (!Array.isArray(obj[feature])) { obj[feature] = []; + obj.$segments.properties[feature] = []; } obj[feature].push(item); + obj.$segments.properties.add(feature, segment); } } } @@ -544,11 +564,6 @@ export class LangiumCompletionParser extends AbstractLangiumParser { // NOOP } - construct(): unknown { - // NOOP - return undefined; - } - parse(input: string): CompletionParserResult { this.resetState(); const tokens = this.lexer.tokenize(input, { mode: 'partial' }); diff --git a/packages/langium/src/references/name-provider.ts b/packages/langium/src/references/name-provider.ts index 209fd4da6..3d4604c35 100644 --- a/packages/langium/src/references/name-provider.ts +++ b/packages/langium/src/references/name-provider.ts @@ -21,9 +21,16 @@ export function isNamed(node: AstNode): node is NamedAstNode { export interface NameProvider { /** * Returns the `name` of a given AstNode. - * @param node Specified `AstNode` whose name node shall be retrieved. + * @param node Specified `AstNode` whose name shall be retrieved. */ getName(node: AstNode): string | undefined; + + /** + * Returns the property name that is used to store the name of an AstNode. + * @param node The AstNode for which the name property shall be retrieved. + */ + getNameProperty(node: AstNode): string | undefined; + /** * Returns the `CstNode` which contains the parsed value of the `name` assignment. * @param node Specified `AstNode` whose name node shall be retrieved. @@ -39,7 +46,11 @@ export class DefaultNameProvider implements NameProvider { return undefined; } + getNameProperty(_node: AstNode): string | undefined { + return 'name'; + } + getNameNode(node: AstNode): CstNode | undefined { - return findNodeForProperty(node.$cstNode, 'name'); + return findNodeForProperty(node.$cstNode, this.getNameProperty(node)); } } diff --git a/packages/langium/src/references/references.ts b/packages/langium/src/references/references.ts index 34c9089eb..06c303c54 100644 --- a/packages/langium/src/references/references.ts +++ b/packages/langium/src/references/references.ts @@ -15,7 +15,7 @@ import type { URI } from '../utils/uri-utils.js'; import { findAssignment } from '../utils/grammar-utils.js'; import { isReference } from '../syntax-tree.js'; import { getDocument } from '../utils/ast-utils.js'; -import { isChildNode, toDocumentSegment } from '../utils/cst-utils.js'; +import { isChildNode } from '../utils/cst-utils.js'; import { stream } from '../utils/stream.js'; import { UriUtils } from '../utils/uri-utils.js'; @@ -25,8 +25,8 @@ import { UriUtils } from '../utils/uri-utils.js'; export interface References { /** - * If the CstNode is a reference node the target CstNode will be returned. - * If the CstNode is a significant node of the CstNode this CstNode will be returned. + * If the CstNode is a reference node the target AstNode will be returned. + * If the CstNode is a significant node of the AstNode this AstNode will be returned. * * @param sourceCstNode CstNode that points to a AstNode */ @@ -37,6 +37,7 @@ export interface References { * If the CstNode is a significant node of the CstNode this CstNode will be returned. * * @param sourceCstNode CstNode that points to a AstNode + * @deprecated Since 4.0.0. Use {@link findDeclaration} instead. If the CST node of the referenced element has been discarded, this method will return `undefined`. */ findDeclarationNode(sourceCstNode: CstNode): CstNode | undefined; @@ -49,10 +50,6 @@ export interface References { } export interface FindReferencesOptions { - /** - * @deprecated Since v1.2.0. Please use `documentUri` instead. - */ - onlyLocal?: boolean; /** * When set, the `findReferences` method will only return references/declarations from the specified document. */ @@ -130,18 +127,21 @@ export class DefaultReferences implements References { } protected getReferenceToSelf(targetNode: AstNode): ReferenceDescription | undefined { - const nameNode = this.nameProvider.getNameNode(targetNode); - if (nameNode) { - const doc = getDocument(targetNode); - const path = this.nodeLocator.getAstNodePath(targetNode); - return { - sourceUri: doc.uri, - sourcePath: path, - targetUri: doc.uri, - targetPath: path, - segment: toDocumentSegment(nameNode), - local: true - }; + const nameProperty = this.nameProvider.getNameProperty(targetNode); + if (nameProperty && targetNode.$segments) { + const nameSegment = targetNode.$segments.properties.get(nameProperty)[0]; + if (nameSegment) { + const doc = getDocument(targetNode); + const path = this.nodeLocator.getAstNodePath(targetNode); + return { + sourceUri: doc.uri, + sourcePath: path, + targetUri: doc.uri, + targetPath: path, + segment: nameSegment, + local: true + }; + } } return undefined; } diff --git a/packages/langium/src/serializer/hydrator.ts b/packages/langium/src/serializer/hydrator.ts index 0f9477c70..3b7ec19b0 100644 --- a/packages/langium/src/serializer/hydrator.ts +++ b/packages/langium/src/serializer/hydrator.ts @@ -303,6 +303,9 @@ export class DefaultHydrator implements Hydrator { if (this.grammarElementIdMap.size === 0) { this.createGrammarElementIdMap(); } + if (!node) { + return undefined; + } return this.grammarElementIdMap.get(node); } diff --git a/packages/langium/src/serializer/json-serializer.ts b/packages/langium/src/serializer/json-serializer.ts index e21968776..5dd2f80b6 100644 --- a/packages/langium/src/serializer/json-serializer.ts +++ b/packages/langium/src/serializer/json-serializer.ts @@ -109,7 +109,7 @@ function isIntermediateReference(obj: unknown): obj is IntermediateReference { export class DefaultJsonSerializer implements JsonSerializer { /** The set of AstNode properties to be ignored by the serializer. */ - ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode']); + ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode', '$segments']); /** The document that is currently processed by the serializer; this is used by the replacer function. */ protected currentDocument: LangiumDocument | undefined; diff --git a/packages/langium/src/services.ts b/packages/langium/src/services.ts index a4760c42d..8f9856783 100644 --- a/packages/langium/src/services.ts +++ b/packages/langium/src/services.ts @@ -37,6 +37,7 @@ import type { IndexManager } from './workspace/index-manager.js'; import type { WorkspaceLock } from './workspace/workspace-lock.js'; import type { Hydrator } from './serializer/hydrator.js'; import type { WorkspaceManager } from './workspace/workspace-manager.js'; +import type { Environment } from './workspace/environment.js'; /** * The services generated by `langium-cli` for a specific language. These are derived from the @@ -112,6 +113,7 @@ export type LangiumGeneratedSharedCoreServices = { export type LangiumDefaultSharedCoreServices = { readonly ServiceRegistry: ServiceRegistry readonly workspace: { + readonly Environment: Environment readonly ConfigurationProvider: ConfigurationProvider readonly DocumentBuilder: DocumentBuilder readonly FileSystemProvider: FileSystemProvider diff --git a/packages/langium/src/syntax-tree.ts b/packages/langium/src/syntax-tree.ts index cbdc8bbbd..58306fcae 100644 --- a/packages/langium/src/syntax-tree.ts +++ b/packages/langium/src/syntax-tree.ts @@ -8,6 +8,7 @@ import type { TokenType } from 'chevrotain'; import type { URI } from './utils/uri-utils.js'; import type { AbstractElement } from './languages/generated/ast.js'; import type { DocumentSegment, LangiumDocument } from './workspace/documents.js'; +import type { MultiMap } from './utils/collections.js'; /** * A node in the Abstract Syntax Tree (AST). @@ -23,6 +24,8 @@ export interface AstNode { readonly $containerIndex?: number; /** The Concrete Syntax Tree (CST) node of the text range from which this node was parsed. */ readonly $cstNode?: CstNode; + /** Cache for storing segments (ranges) to respond to LSP requests */ + readonly $segments?: AstNodeSegments; /** The document containing the AST; only the root node has a direct reference to the document. */ readonly $document?: LangiumDocument; } @@ -42,6 +45,12 @@ type SpecificNodeProperties = keyof Omit = SpecificNodeProperties extends never ? string : SpecificNodeProperties +export interface AstNodeSegments { + readonly full: DocumentSegment; + readonly comment?: string; + readonly properties: MultiMap; +} + /** * A cross-reference in the AST. Cross-references may or may not be successfully resolved. */ diff --git a/packages/langium/src/utils/cst-utils.ts b/packages/langium/src/utils/cst-utils.ts index 54e2374a1..807a68209 100644 --- a/packages/langium/src/utils/cst-utils.ts +++ b/packages/langium/src/utils/cst-utils.ts @@ -6,11 +6,12 @@ import type { IToken } from '@chevrotain/types'; import type { Range } from 'vscode-languageserver-types'; -import type { CstNode, CompositeCstNode, LeafCstNode } from '../syntax-tree.js'; +import type { CstNode, CompositeCstNode, LeafCstNode, AstNode, Mutable, Reference } from '../syntax-tree.js'; import type { DocumentSegment } from '../workspace/documents.js'; import type { Stream, TreeStream } from './stream.js'; import { isCompositeCstNode, isLeafCstNode, isRootCstNode } from '../syntax-tree.js'; import { TreeStreamImpl } from './stream.js'; +import { streamAst, streamReferences } from './ast-utils.js'; /** * Create a stream of all CST nodes that are directly and indirectly contained in the given root node, @@ -339,3 +340,12 @@ interface ParentLink { parent: CompositeCstNode index: number } + +export function discardCst(node: AstNode): void { + streamAst(node).forEach(n => { + (n as Mutable).$cstNode = undefined; + streamReferences(n).forEach(r => { + (r.reference as Mutable>).$refNode = undefined; + }); + }); +} diff --git a/packages/langium/src/validation/document-validator.ts b/packages/langium/src/validation/document-validator.ts index fb3754411..4c866f4e4 100644 --- a/packages/langium/src/validation/document-validator.ts +++ b/packages/langium/src/validation/document-validator.ts @@ -61,6 +61,9 @@ export class DefaultDocumentValidator implements DocumentValidator { async validateDocument(document: LangiumDocument, options: ValidationOptions = {}, cancelToken = CancellationToken.None): Promise { const parseResult = document.parseResult; + if (!parseResult.value.$cstNode) { + return []; + } const diagnostics: Diagnostic[] = []; await interruptAndCheck(cancelToken); diff --git a/packages/langium/src/workspace/ast-descriptions.ts b/packages/langium/src/workspace/ast-descriptions.ts index 46a3743d8..88af62777 100644 --- a/packages/langium/src/workspace/ast-descriptions.ts +++ b/packages/langium/src/workspace/ast-descriptions.ts @@ -13,7 +13,6 @@ import type { DocumentSegment, LangiumDocument } from './documents.js'; import { CancellationToken } from '../utils/cancellation.js'; import { isLinkingError } from '../syntax-tree.js'; import { getDocument, streamAst, streamReferences } from '../utils/ast-utils.js'; -import { toDocumentSegment } from '../utils/cst-utils.js'; import { interruptAndCheck } from '../utils/promise-utils.js'; import { UriUtils } from '../utils/uri-utils.js'; @@ -53,15 +52,15 @@ export class DefaultAstNodeDescriptionProvider implements AstNodeDescriptionProv if (!name) { throw new Error(`Node at path ${path} has no name.`); } - let nameNodeSegment: DocumentSegment | undefined; - const nameSegmentGetter = () => nameNodeSegment ??= toDocumentSegment(this.nameProvider.getNameNode(node) ?? node.$cstNode); + const nameProperty = this.nameProvider.getNameProperty(node); + const nameSegment: DocumentSegment | undefined = nameProperty + ? node.$segments?.properties.get(nameProperty)[0] + : undefined; return { node, name, - get nameSegment() { - return nameSegmentGetter(); - }, - selectionSegment: toDocumentSegment(node.$cstNode), + nameSegment, + selectionSegment: node.$segments?.full, type: node.$type, documentUri: doc.uri, path @@ -131,8 +130,8 @@ export class DefaultReferenceDescriptionProvider implements ReferenceDescription protected createDescription(refInfo: ReferenceInfo): ReferenceDescription | undefined { const targetNodeDescr = refInfo.reference.$nodeDescription; - const refCstNode = refInfo.reference.$refNode; - if (!targetNodeDescr || !refCstNode) { + const refSegment = refInfo.container.$segments?.properties.get(refInfo.property)[refInfo.index ?? 0]; + if (!targetNodeDescr || !refSegment) { return undefined; } const docUri = getDocument(refInfo.container).uri; @@ -141,7 +140,7 @@ export class DefaultReferenceDescriptionProvider implements ReferenceDescription sourcePath: this.nodeLocator.getAstNodePath(refInfo.container), targetUri: targetNodeDescr.documentUri, targetPath: targetNodeDescr.path, - segment: toDocumentSegment(refCstNode), + segment: refSegment, local: UriUtils.equals(targetNodeDescr.documentUri, docUri) }; } diff --git a/packages/langium/src/workspace/documents.ts b/packages/langium/src/workspace/documents.ts index a94dc3563..2cb40c693 100644 --- a/packages/langium/src/workspace/documents.ts +++ b/packages/langium/src/workspace/documents.ts @@ -15,7 +15,7 @@ export { TextDocument } from 'vscode-languageserver-textdocument'; import type { Diagnostic, Range } from 'vscode-languageserver-types'; import type { FileSystemProvider } from './file-system-provider.js'; -import type { ParseResult, ParserOptions } from '../parser/langium-parser.js'; +import { CstParserMode, type ParseResult, type ParserOptions } from '../parser/langium-parser.js'; import type { ServiceRegistry } from '../service-registry.js'; import type { LangiumSharedCoreServices } from '../services.js'; import type { AstNode, AstNodeDescription, Mutable, Reference } from '../syntax-tree.js'; @@ -130,7 +130,7 @@ export interface LangiumDocumentFactory { /** * Create a Langium document from a `TextDocument` asynchronously. This action can be cancelled if a cancellable parser implementation has been provided. */ - fromTextDocument(textDocument: TextDocument, uri: URI | undefined, cancellationToken: CancellationToken): Promise>; + fromTextDocument(textDocument: TextDocument, uri: URI | undefined, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; /** * Create an Langium document from an in-memory string. @@ -139,7 +139,7 @@ export interface LangiumDocumentFactory { /** * Create a Langium document from an in-memory string asynchronously. This action can be cancelled if a cancellable parser implementation has been provided. */ - fromString(text: string, uri: URI, cancellationToken: CancellationToken): Promise>; + fromString(text: string, uri: URI, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; /** * Create an Langium document from a model that has been constructed in memory. @@ -149,7 +149,7 @@ export interface LangiumDocumentFactory { /** * Create an Langium document from a specified `URI`. The factory will use the `FileSystemAccess` service to read the file. */ - fromUri(uri: URI, cancellationToken?: CancellationToken): Promise>; + fromUri(uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): Promise>; /** * Update the given document after changes in the corresponding textual representation. @@ -173,29 +173,29 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { this.fileSystemProvider = services.workspace.FileSystemProvider; } - async fromUri(uri: URI, cancellationToken = CancellationToken.None): Promise> { + async fromUri(uri: URI, options?: ParserOptions | undefined, cancellationToken = CancellationToken.None): Promise> { const content = await this.fileSystemProvider.readFile(uri); - return this.createAsync(uri, content, cancellationToken); + return this.createAsync(uri, content, options, cancellationToken); } fromTextDocument(textDocument: TextDocument, uri?: URI, options?: ParserOptions): LangiumDocument; - fromTextDocument(textDocument: TextDocument, uri: URI | undefined, cancellationToken: CancellationToken): Promise>; - fromTextDocument(textDocument: TextDocument, uri?: URI, token?: CancellationToken | ParserOptions): LangiumDocument | Promise> { + fromTextDocument(textDocument: TextDocument, uri: URI | undefined, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; + fromTextDocument(textDocument: TextDocument, uri?: URI, options?: ParserOptions, cancellationToken?: CancellationToken): LangiumDocument | Promise> { uri = uri ?? URI.parse(textDocument.uri); - if (CancellationToken.is(token)) { - return this.createAsync(uri, textDocument, token); + if (cancellationToken) { + return this.createAsync(uri, textDocument, options, cancellationToken); } else { - return this.create(uri, textDocument, token); + return this.create(uri, textDocument, options); } } fromString(text: string, uri: URI, options?: ParserOptions): LangiumDocument; - fromString(text: string, uri: URI, cancellationToken: CancellationToken): Promise>; - fromString(text: string, uri: URI, token?: CancellationToken | ParserOptions): LangiumDocument | Promise> { - if (CancellationToken.is(token)) { - return this.createAsync(uri, text, token); + fromString(text: string, uri: URI, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; + fromString(text: string, uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): LangiumDocument | Promise> { + if (cancellationToken) { + return this.createAsync(uri, text, options, cancellationToken); } else { - return this.create(uri, text, token); + return this.create(uri, text, options); } } @@ -218,12 +218,12 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { } } - protected async createAsync(uri: URI, content: string | TextDocument, cancelToken: CancellationToken): Promise> { + protected async createAsync(uri: URI, content: string | TextDocument, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise> { if (typeof content === 'string') { - const parseResult = await this.parseAsync(uri, content, cancelToken); + const parseResult = await this.parseAsync(uri, content, options, cancelToken); return this.createLangiumDocument(parseResult, uri, undefined, content); } else { - const parseResult = await this.parseAsync(uri, content.getText(), cancelToken); + const parseResult = await this.parseAsync(uri, content.getText(), options, cancelToken); return this.createLangiumDocument(parseResult, uri, content); } } @@ -293,7 +293,10 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { // Some of these documents can be pretty large, so parsing them again can be quite expensive. // Therefore, we only parse if the text has actually changed. if (oldText !== text) { - document.parseResult = await this.parseAsync(document.uri, text, cancellationToken); + const options: ParserOptions = { + cst: textDocument ? CstParserMode.Retain : CstParserMode.Discard + }; + document.parseResult = await this.parseAsync(document.uri, text, options, cancellationToken); (document.parseResult.value as Mutable).$document = document; } document.state = DocumentState.Parsed; @@ -305,9 +308,9 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { return services.parser.LangiumParser.parse(text, options); } - protected parseAsync(uri: URI, text: string, cancellationToken: CancellationToken): Promise> { + protected parseAsync(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise> { const services = this.serviceRegistry.getServices(uri); - return services.parser.AsyncParser.parse(text, cancellationToken); + return services.parser.AsyncParser.parse(text, options, cancellationToken); } protected createTextDocumentGetter(uri: URI, text?: string): () => TextDocument { @@ -346,7 +349,7 @@ export interface LangiumDocuments { * Retrieve the document with the given URI. If not present, a new one will be created using the file system access. * The new document will be added to the list of documents managed under this service. */ - getOrCreateDocument(uri: URI, cancellationToken?: CancellationToken): Promise; + getOrCreateDocument(uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): Promise; /** * Creates a new document with the given URI and text content. @@ -354,7 +357,7 @@ export interface LangiumDocuments { * * @throws an error if a document with the same URI is already present. */ - createDocument(uri: URI, text: string): LangiumDocument; + createDocument(uri: URI, text: string, options?: ParserOptions): LangiumDocument; /** * Creates a new document with the given URI and text content asynchronously. @@ -363,7 +366,7 @@ export interface LangiumDocuments { * * @throws an error if a document with the same URI is already present. */ - createDocument(uri: URI, text: string, cancellationToken: CancellationToken): Promise; + createDocument(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise; /** * Returns `true` if a document with the given URI is managed under this service. @@ -418,26 +421,26 @@ export class DefaultLangiumDocuments implements LangiumDocuments { return this.documentMap.get(uriString); } - async getOrCreateDocument(uri: URI, cancellationToken?: CancellationToken): Promise { + async getOrCreateDocument(uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): Promise { let document = this.getDocument(uri); if (document) { return document; } - document = await this.langiumDocumentFactory.fromUri(uri, cancellationToken); + document = await this.langiumDocumentFactory.fromUri(uri, options, cancellationToken); this.addDocument(document); return document; } - createDocument(uri: URI, text: string): LangiumDocument; - createDocument(uri: URI, text: string, cancellationToken: CancellationToken): Promise; - createDocument(uri: URI, text: string, cancellationToken?: CancellationToken): LangiumDocument | Promise { + createDocument(uri: URI, text: string, options?: ParserOptions): LangiumDocument; + createDocument(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise; + createDocument(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken?: CancellationToken): LangiumDocument | Promise { if (cancellationToken) { - return this.langiumDocumentFactory.fromString(text, uri, cancellationToken).then(document => { + return this.langiumDocumentFactory.fromString(text, uri, options, cancellationToken).then(document => { this.addDocument(document); return document; }); } else { - const document = this.langiumDocumentFactory.fromString(text, uri); + const document = this.langiumDocumentFactory.fromString(text, uri, options); this.addDocument(document); return document; } diff --git a/packages/langium/src/workspace/environment.ts b/packages/langium/src/workspace/environment.ts new file mode 100644 index 000000000..667f0e6e4 --- /dev/null +++ b/packages/langium/src/workspace/environment.ts @@ -0,0 +1,51 @@ +/****************************************************************************** + * Copyright 2024 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +import type { InitializeParams, InitializedParams } from 'vscode-languageserver-protocol'; + +export interface EnvironmentInfo { + readonly isLanguageServer: boolean; + readonly locale: string; +} + +export interface Environment extends EnvironmentInfo { + initialize(params: InitializeParams): void; + initialized(params: InitializedParams): void; + update(newEnvironment: Partial): void; +} + +export class DefaultEnvironment implements Environment { + + private _isLanguageServer: boolean = false; + private _locale: string = 'en'; + + get isLanguageServer(): boolean { + return this._isLanguageServer; + } + + get locale(): string { + return this._locale; + } + + initialize(params: InitializeParams): void { + this.update({ + isLanguageServer: true, + locale: params.locale + }); + } + + initialized(_params: InitializedParams): void { + } + + update(newEnvironment: Partial): void { + if (typeof newEnvironment.isLanguageServer === 'boolean') { + this._isLanguageServer = newEnvironment.isLanguageServer; + } + if (typeof newEnvironment.locale === 'string') { + this._locale = newEnvironment.locale; + } + } +} diff --git a/packages/langium/src/workspace/workspace-manager.ts b/packages/langium/src/workspace/workspace-manager.ts index cdc9c7c4a..02edcefec 100644 --- a/packages/langium/src/workspace/workspace-manager.ts +++ b/packages/langium/src/workspace/workspace-manager.ts @@ -15,6 +15,8 @@ import type { BuildOptions, DocumentBuilder } from './document-builder.js'; import type { LangiumDocument, LangiumDocuments } from './documents.js'; import type { FileSystemNode, FileSystemProvider } from './file-system-provider.js'; import type { WorkspaceLock } from './workspace-lock.js'; +import { CstParserMode } from '../parser/langium-parser.js'; +import type { Environment } from './environment.js'; // export type WorkspaceFolder from 'vscode-languageserver-types' for convenience, // is supposed to avoid confusion as 'WorkspaceFolder' might accidentally be imported via 'vscode-languageclient' @@ -75,6 +77,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { protected readonly langiumDocuments: LangiumDocuments; protected readonly documentBuilder: DocumentBuilder; protected readonly fileSystemProvider: FileSystemProvider; + protected readonly environment: Environment; protected readonly mutex: WorkspaceLock; protected readonly _ready = new Deferred(); protected folders?: WorkspaceFolder[]; @@ -85,6 +88,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { this.documentBuilder = services.workspace.DocumentBuilder; this.fileSystemProvider = services.workspace.FileSystemProvider; this.mutex = services.workspace.WorkspaceLock; + this.environment = services.workspace.Environment; } get ready(): Promise { @@ -167,13 +171,19 @@ export class DefaultWorkspaceManager implements WorkspaceManager { if (entry.isDirectory) { await this.traverseFolder(workspaceFolder, entry.uri, fileExtensions, collector); } else if (entry.isFile) { - const document = await this.langiumDocuments.getOrCreateDocument(entry.uri); + const document = await this.langiumDocuments.getOrCreateDocument(entry.uri, { + cst: this.getCstParserMode(entry.uri) + }); collector(document); } } })); } + protected getCstParserMode(_uri: URI): CstParserMode { + return this.environment.isLanguageServer ? CstParserMode.Discard : CstParserMode.Retain; + } + /** * Determine whether the given folder entry shall be included while indexing the workspace. */ diff --git a/packages/langium/test/documentation/comment-provider.test.ts b/packages/langium/test/documentation/comment-provider.test.ts index f01ace249..f91e0dcc2 100644 --- a/packages/langium/test/documentation/comment-provider.test.ts +++ b/packages/langium/test/documentation/comment-provider.test.ts @@ -6,21 +6,25 @@ import { describe, expect, test } from 'vitest'; import { parseHelper } from 'langium/test'; -import { AstUtils, EmptyFileSystem, GrammarAST } from 'langium'; +import { AstUtils, CstParserMode, EmptyFileSystem, GrammarAST } from 'langium'; import { createLangiumGrammarServices } from 'langium/grammar'; const services = createLangiumGrammarServices(EmptyFileSystem).grammar; const parse = parseHelper(services); -describe('Comment provider', () => { - test('Get a comment', async () => { +describe.each([CstParserMode.Discard, CstParserMode.Retain])('Comment provider', parserMode => { + test(`Get a comment with parser mode ${parserMode === 0 ? 'retain' : 'discard'}`, async () => { const ast = (await parse(` grammar Test /** Rule */ entry Rule: 'rule' num=INT; /** INT */ terminal INT: /\\d+/; - `)).parseResult.value; + `, { + parserOptions: { + cst: parserMode + } + })).parseResult.value; expect(ast).toBeDefined(); const grammarComment = services.documentation.CommentProvider.getComment(ast); diff --git a/packages/langium/test/parser/worker-thread-async-parser.test.ts b/packages/langium/test/parser/worker-thread-async-parser.test.ts index 96f396832..7b45fe0fe 100644 --- a/packages/langium/test/parser/worker-thread-async-parser.test.ts +++ b/packages/langium/test/parser/worker-thread-async-parser.test.ts @@ -32,7 +32,7 @@ describe('WorkerThreadAsyncParser', () => { asyncParser.setThreadCount(4); const promises: Array>> = []; for (let i = 0; i < 16; i++) { - promises.push(asyncParser.parse(file, CancellationToken.None)); + promises.push(asyncParser.parse(file, undefined, CancellationToken.None)); } const result = await Promise.all(promises); for (const parseResult of result) { @@ -50,7 +50,7 @@ describe('WorkerThreadAsyncParser', () => { setTimeout(() => cancellationTokenSource.cancel(), 50); const start = performance.now(); try { - await asyncParser.parse(file, cancellationTokenSource.token); + await asyncParser.parse(file, undefined, cancellationTokenSource.token); fail('Parsing should have been cancelled'); } catch (err) { expect(isOperationCancelled(err)).toBe(true); @@ -68,20 +68,20 @@ describe('WorkerThreadAsyncParser', () => { const cancellationTokenSource = startCancelableOperation(); setTimeout(() => cancellationTokenSource.cancel(), 50); try { - await asyncParser.parse(file, cancellationTokenSource.token); + await asyncParser.parse(file, undefined, cancellationTokenSource.token); fail('Parsing should have been cancelled'); } catch (err) { expect(isOperationCancelled(err)).toBe(true); } // Calling this method should recreate the worker and parse the file correctly - const result = await asyncParser.parse(createLargeFile(10), CancellationToken.None); + const result = await asyncParser.parse(createLargeFile(10), undefined, CancellationToken.None); expect(result.value.name).toBe('Test'); }); test('async parsing yields correct CST', async () => { const services = getServices(); const file = createLargeFile(10); - const result = await services.parser.AsyncParser.parse(file, CancellationToken.None); + const result = await services.parser.AsyncParser.parse(file, undefined, CancellationToken.None); const index = file.indexOf('TestRule'); // Assert that the CST can be found at all from the root node // This indicates that the CST is correctly linked to itself @@ -104,7 +104,7 @@ describe('WorkerThreadAsyncParser', () => { test('parser errors are correctly transmitted', async () => { const services = getServices(); const file = 'grammar Test Rule: name="Hello" // missing semicolon'; - const result = await services.parser.AsyncParser.parse(file, CancellationToken.None); + const result = await services.parser.AsyncParser.parse(file, undefined, CancellationToken.None); expect(result.parserErrors).toHaveLength(1); expect(result.parserErrors[0].name).toBe('MismatchedTokenException'); expect(result.parserErrors[0]).toHaveProperty('previousToken'); diff --git a/packages/langium/test/parser/worker-thread.js b/packages/langium/test/parser/worker-thread.js index 5922a8459..e9a95a748 100644 --- a/packages/langium/test/parser/worker-thread.js +++ b/packages/langium/test/parser/worker-thread.js @@ -12,8 +12,8 @@ const services = createLangiumGrammarServices(EmptyFileSystem).grammar; const parser = services.parser.LangiumParser; const hydrator = services.serializer.Hydrator; -parentPort.on('message', text => { - const result = parser.parse(text); +parentPort.on('message', ([text, options]) => { + const result = parser.parse(text, options); const dehydrated = hydrator.dehydrate(result); parentPort.postMessage(dehydrated); });