Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert semantic token types to TextMate scopes #21

Merged
merged 4 commits into from
Oct 4, 2024
Merged

Convert semantic token types to TextMate scopes #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a192204
Move conversion of legacy semantic token types into general converter
sungshik Oct 4, 2024
3648bfc
Add conversion of current semantic token types to TextMate scopes
sungshik Oct 4, 2024
80349e5
Update tests
sungshik Oct 4, 2024
5c21d9d
Split preprocessor in separate functions
sungshik Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 1 addition & 25 deletions rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,7 @@ private Grammar getRscGrammar() {
: p[attributes = attributes + \tag("category"(category))];

return visit (rsc) {

// The following mapping is based on:
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
// With updates based on:
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
case \tag("category"("Normal")) => \tag("category"("source"))
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
case \tag("category"("Identifier")) => \tag("category"("variable"))
case \tag("category"("Variable")) => \tag("category"("variable"))
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
case \tag("category"("Comment")) => \tag("category"("comment"))
case \tag("category"("Todo")) => \tag("category"("comment"))
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
sungshik marked this conversation as resolved.
Show resolved Hide resolved

// With additional hot-patching as discussed:
// Temporarily hot-patch Rascal's own grammar as discussed here:
// - https://github.com/SWAT-engineering/rascal-textmate/pull/6
case p: prod(label("integer", sort("Literal")), _, _) => setCategory(p, "constant.numeric")
case p: prod(label("real", sort("Literal")), _, _) => setCategory(p, "constant.numeric")
Expand Down
72 changes: 67 additions & 5 deletions rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,75 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration =
}

RscGrammar preprocess(RscGrammar rsc) {
// Replace occurrences of singleton ranges with just the corresponding
// literal. This makes it easier to identify delimiters.
return visit (rsc) {
rsc = replaceSingletonRanges(rsc);
rsc = replaceCurrentSemanticTokenTypes(rsc);
rsc = replaceLegacySemanticTokenTypes(rsc);
return rsc;
}

// Replace occurrences of singleton ranges with just the corresponding literal.
// This makes it easier to identify delimiters.
private RscGrammar replaceSingletonRanges(RscGrammar rsc)
= visit (rsc) {
case \char-class([range(char, char)]) => d
when d := \lit("<stringChar(char)>"), isDelimiter(d)
}
}
};

// Replace current semantic token types with TextMate scopes based on:
// - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571
private RscGrammar replaceCurrentSemanticTokenTypes(RscGrammar rsc)
= visit (rsc) {
case \tag("category"("comment")) => \tag("category"("comment"))
case \tag("category"("string")) => \tag("category"("string"))
case \tag("category"("keyword")) => \tag("category"("keyword.control"))
case \tag("category"("number")) => \tag("category"("constant.numeric"))
case \tag("category"("regexp")) => \tag("category"("constant.regexp"))
case \tag("category"("operator")) => \tag("category"("keyword.operator"))
case \tag("category"("namespace")) => \tag("category"("entity.name.namespace"))
case \tag("category"("type")) => \tag("category"("support.type")) // Alternative: support.type
case \tag("category"("struct")) => \tag("category"("entity.name.type.struct"))
case \tag("category"("class")) => \tag("category"("entity.name.type.class")) // Alternative: support.class
case \tag("category"("interface")) => \tag("category"("entity.name.type.interface"))
case \tag("category"("enum")) => \tag("category"("entity.name.type.enum"))
case \tag("category"("typeParameter")) => \tag("category"("entity.name.type.parameter"))
case \tag("category"("function")) => \tag("category"("entity.name.function")) // Alternative: support.function
case \tag("category"("method")) => \tag("category"("entity.name.function.member")) // Alternative: support.function
case \tag("category"("macro")) => \tag("category"("entity.name.function.preprocessor"))
case \tag("category"("variable")) => \tag("category"("variable.other.readwrite")) // Alternative: entity.name.variable
case \tag("category"("parameter")) => \tag("category"("variable.parameter"))
case \tag("category"("property")) => \tag("category"("variable.other.property"))
case \tag("category"("enumMember")) => \tag("category"("variable.other.enummember"))
case \tag("category"("event")) => \tag("category"("variable.other.event"))
case \tag("category"("decorator")) => \tag("category"("entity.name.decorator")) // Alternative: entity.name.function
// Note: Categories types `member` and `label` are deprecated/undefined
// and therefore excluded from this mapping
};

// Replace legacy semantic token types with TextMate scopes based on:
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
// With updates based on:
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
private RscGrammar replaceLegacySemanticTokenTypes(RscGrammar rsc)
= visit (rsc) {
case \tag("category"("Normal")) => \tag("category"("source"))
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
case \tag("category"("Identifier")) => \tag("category"("variable"))
case \tag("category"("Variable")) => \tag("category"("variable"))
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
case \tag("category"("Comment")) => \tag("category"("comment"))
case \tag("category"("Todo")) => \tag("category"("comment"))
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
};

@synoposis{
Analyzes Rascal grammar `rsc`. Returns a list of productions, in the form of
Expand Down
4 changes: 2 additions & 2 deletions rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ Grammar rsc = preprocess(grammar(#Program));

list[ConversionUnit] units = [
sungshik marked this conversation as resolved.
Show resolved Hide resolved
unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("+"),lit("||"),lit(":="),lit("\""),lit(";")})],{}), false, false, <nothing(),nothing()>, <nothing(),nothing()>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("Comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("Comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("do"),lit("declare"),lit("fi"),lit("else"),lit("end"),lit("od"),lit("nil-type"),lit("begin"),lit("natural"),lit("then"),lit("if"),lit("while"),lit("string")})],{\tag("category"("keyword.control"))}), false, false, <nothing(),nothing()>, <nothing(),nothing()>)
];

Expand Down
6 changes: 3 additions & 3 deletions rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
# SYNTAX TEST "Pico"

%% foo bar
# ^^^^^^^^^^ Comment
# ^^^^^^^^^^ comment

%% foo % bar
# ^^^^^^^^^^^^ Comment
# ^^^^^^^^^^^^ comment

%% do
# ^^^^^ Comment
# ^^^^^ comment

do
# ^^ keyword.control
Loading