From e1c45e6cd47d9950e7bec965a7cd15ba334ef827 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Mon, 8 Jan 2024 16:22:56 +0100 Subject: [PATCH 1/3] Generate grammar comments --- .../src/language-server/domain-model.langium | 3 +++ .../src/language-server/generated/grammar.ts | 3 ++- .../langium-cli/src/generator/grammar-serializer.ts | 1 + packages/langium/src/grammar/langium-grammar.langium | 11 ++++++----- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/domainmodel/src/language-server/domain-model.langium b/examples/domainmodel/src/language-server/domain-model.langium index b1cc17e3a..12809a003 100644 --- a/examples/domainmodel/src/language-server/domain-model.langium +++ b/examples/domainmodel/src/language-server/domain-model.langium @@ -22,6 +22,9 @@ Entity: (features+=Feature)* '}'; +/** + * This comment should be generated into the grammar! + */ Feature: (many?='many')? name=ID ':' type=[Type:QualifiedName]; diff --git a/examples/domainmodel/src/language-server/generated/grammar.ts b/examples/domainmodel/src/language-server/generated/grammar.ts index 0560815e5..8e6896d1d 100644 --- a/examples/domainmodel/src/language-server/generated/grammar.ts +++ b/examples/domainmodel/src/language-server/generated/grammar.ts @@ -315,7 +315,8 @@ export const DomainModelGrammar = (): Grammar => loadedDomainModelGrammar ?? (lo "fragment": false, "hiddenTokens": [], "parameters": [], - "wildcard": false + "wildcard": false, + "$comment": "/**\\r\\n * This comment should be generated into the grammar!\\r\\n */" }, { "$type": "ParserRule", diff --git a/packages/langium-cli/src/generator/grammar-serializer.ts b/packages/langium-cli/src/generator/grammar-serializer.ts index 856126533..e94b5c010 100644 --- a/packages/langium-cli/src/generator/grammar-serializer.ts +++ b/packages/langium-cli/src/generator/grammar-serializer.ts @@ -36,6 +36,7 @@ export function serializeGrammar(services: LangiumServices, grammars: Grammar[], }; const serializedGrammar = services.serializer.JsonSerializer.serialize(grammar, { space: production ? undefined : 2, + comments: true, uriConverter }); // The json serializer returns strings with \n line delimiter by default diff --git a/packages/langium/src/grammar/langium-grammar.langium b/packages/langium/src/grammar/langium-grammar.langium index 1a5abf1c7..b964d8636 100644 --- a/packages/langium/src/grammar/langium-grammar.langium +++ b/packages/langium/src/grammar/langium-grammar.langium @@ -1,8 +1,9 @@ -/****************************************************************************** - * Copyright 2021 TypeFox GmbH - * This program and the accompanying materials are made available under the - * terms of the MIT License, which is available in the project root. - ******************************************************************************/ +// ****************************************************************************** +// Copyright 2021 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. +// ***************************************************************************** + grammar LangiumGrammar entry Grammar: From d181da3127a514627e09b784c1adc9baab778c83 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Tue, 9 Jan 2024 14:08:15 +0100 Subject: [PATCH 2/3] Fix Posix vs Windows issue --- .../domainmodel/src/language-server/domain-model.langium | 4 ++-- .../domainmodel/src/language-server/generated/grammar.ts | 5 +++-- packages/langium/src/serializer/json-serializer.ts | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/domainmodel/src/language-server/domain-model.langium b/examples/domainmodel/src/language-server/domain-model.langium index 12809a003..4aaa252db 100644 --- a/examples/domainmodel/src/language-server/domain-model.langium +++ b/examples/domainmodel/src/language-server/domain-model.langium @@ -23,10 +23,10 @@ Entity: '}'; /** - * This comment should be generated into the grammar! + * This is a parser rule */ Feature: - (many?='many')? name=ID ':' type=[Type:QualifiedName]; + (many?='many')? name=ID ':' /** This is an assignment */ type=[Type:QualifiedName]; QualifiedName returns string: ID ('.' ID)*; diff --git a/examples/domainmodel/src/language-server/generated/grammar.ts b/examples/domainmodel/src/language-server/generated/grammar.ts index 8e6896d1d..1c2408411 100644 --- a/examples/domainmodel/src/language-server/generated/grammar.ts +++ b/examples/domainmodel/src/language-server/generated/grammar.ts @@ -306,7 +306,8 @@ export const DomainModelGrammar = (): Grammar => loadedDomainModelGrammar ?? (lo "arguments": [] }, "deprecatedSyntax": false - } + }, + "$comment": "/** This is an assignment */" } ] }, @@ -316,7 +317,7 @@ export const DomainModelGrammar = (): Grammar => loadedDomainModelGrammar ?? (lo "hiddenTokens": [], "parameters": [], "wildcard": false, - "$comment": "/**\\r\\n * This comment should be generated into the grammar!\\r\\n */" + "$comment": "/**\\n * This is a parser rule\\n */" }, { "$type": "ParserRule", diff --git a/packages/langium/src/serializer/json-serializer.ts b/packages/langium/src/serializer/json-serializer.ts index 068f94df7..4e4867a22 100644 --- a/packages/langium/src/serializer/json-serializer.ts +++ b/packages/langium/src/serializer/json-serializer.ts @@ -188,7 +188,10 @@ export class DefaultJsonSerializer implements JsonSerializer { } if (comments) { astNode ??= { ...value }; - (astNode as AstNodeWithComment).$comment = this.commentProvider.getComment(value); + const comment = this.commentProvider.getComment(value); + if (comment) { + (astNode as AstNodeWithComment).$comment = comment.replace(/\r/g, ''); + } } return astNode ?? value; } else { From fe9c09745cadf2315230aa8748322f5a7bafff8c Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Mon, 22 Jan 2024 14:04:06 +0100 Subject: [PATCH 3/3] Add integration test case --- .../src/language-server/domain-model.langium | 5 +- .../src/language-server/generated/grammar.ts | 6 +-- .../test/generator/grammar-serializer.test.ts | 48 +++++++++++++++++++ 3 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 packages/langium-cli/test/generator/grammar-serializer.test.ts diff --git a/examples/domainmodel/src/language-server/domain-model.langium b/examples/domainmodel/src/language-server/domain-model.langium index 4aaa252db..b1cc17e3a 100644 --- a/examples/domainmodel/src/language-server/domain-model.langium +++ b/examples/domainmodel/src/language-server/domain-model.langium @@ -22,11 +22,8 @@ Entity: (features+=Feature)* '}'; -/** - * This is a parser rule - */ Feature: - (many?='many')? name=ID ':' /** This is an assignment */ type=[Type:QualifiedName]; + (many?='many')? name=ID ':' type=[Type:QualifiedName]; QualifiedName returns string: ID ('.' ID)*; diff --git a/examples/domainmodel/src/language-server/generated/grammar.ts b/examples/domainmodel/src/language-server/generated/grammar.ts index 1c2408411..0560815e5 100644 --- a/examples/domainmodel/src/language-server/generated/grammar.ts +++ b/examples/domainmodel/src/language-server/generated/grammar.ts @@ -306,8 +306,7 @@ export const DomainModelGrammar = (): Grammar => loadedDomainModelGrammar ?? (lo "arguments": [] }, "deprecatedSyntax": false - }, - "$comment": "/** This is an assignment */" + } } ] }, @@ -316,8 +315,7 @@ export const DomainModelGrammar = (): Grammar => loadedDomainModelGrammar ?? (lo "fragment": false, "hiddenTokens": [], "parameters": [], - "wildcard": false, - "$comment": "/**\\n * This is a parser rule\\n */" + "wildcard": false }, { "$type": "ParserRule", diff --git a/packages/langium-cli/test/generator/grammar-serializer.test.ts b/packages/langium-cli/test/generator/grammar-serializer.test.ts new file mode 100644 index 000000000..fcbee7134 --- /dev/null +++ b/packages/langium-cli/test/generator/grammar-serializer.test.ts @@ -0,0 +1,48 @@ +/****************************************************************************** + * Copyright 2021 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +import type { LangiumConfig } from '../../src/package.js'; +import { EmptyFileSystem, URI, type Grammar } from 'langium'; +import { describe, expect, test } from 'vitest'; +import { RelativePath } from '../../src/package.js'; +import { serializeGrammar } from '../../src/generator/grammar-serializer.js'; +import { createLangiumGrammarServices } from 'langium/grammar'; +import { expandToString } from 'langium/generate'; + +const grammarServices = createLangiumGrammarServices(EmptyFileSystem); + +describe('Grammar serializer', () => { + test('should include comments of AST elements', async () => { + // arrange + const config: LangiumConfig = { + projectName: 'Magic', + languages: [], + [RelativePath]: '/path/to/magic', + }; + + const grammarText = expandToString` + grammar Test + + /** + * This is the entry rule + */ + entry Model: /** This is the name assignment */ name='test'; + `; + + const document = grammarServices.shared.workspace.LangiumDocumentFactory.fromString(grammarText, URI.file('test.langium')); + grammarServices.shared.workspace.LangiumDocuments.addDocument(document); + await grammarServices.shared.workspace.DocumentBuilder.build([document]); + const grammar = document.parseResult.value; + + // act + const moduleString = serializeGrammar(grammarServices.grammar, [grammar], config); + + // assert + expect(moduleString).toMatch('"$comment": "/** This is the name assignment */"'); + expect(moduleString).toMatch('"$comment": "/**\\\\n * This is the entry rule\\\\n */"'); + }); + +});