Fixes to lexer, parser, highlighting settings, file type icon

monogon-dev · Feb 16, 2021 · 9b60676 · 9b60676
1 parent 12fba0f
commit 9b60676
Show file tree

Hide file tree

Showing 46 changed files with 914 additions and 410 deletions.
diff --git a/README.md b/README.md
@@ -2,8 +2,23 @@
 
 **CUE Language** support for the IntelliJ platform.
 
+## Usage
+The CUE plugin is compatible with **any JetBrains IDE 2020.3**. Only IntelliJ-based IDEs are supported, i.e. it's incompatible with ReSharper.
+
+### Features
+**Please note that this plugin is still in a very early state.**
+- Parsing
+- Basic syntax highlighting
+
+### Bug Reports & Feature Requests
+Please report your issues at [github.com/nexantic/intellij-cue](https://github.com/nexantic/intellij-cue).
+
 ## Development
+
+- Java JDK 11 is required
+
 ### IDE
+
 Development is best in IntelliJ IDEA.
 
 The following plugins are required for development:
@@ -12,18 +27,46 @@ The following plugins are required for development:
 - Gradle
 - Kotlin, for Gradle build file support
 
+### Building
+
+After a build, the plugin is available as a ZIP file at `build/districutions/`.
+
+Building with tests:
+
+```bash
+./gradlew clean build
+```
+
+Building without tests:
+
+```bash
+./gradlew clean build -x test
+```
+
+### Executing
+
+You could build the plugin (see above) and install it into your IDE of choice. Alternatively, you can run the plugin in a sandbox:
+
+```bash
+./gradlew runIde
+```
+
 ### Lexer
+
 The lexer is generated by JFlex. The definition is at `src/grammar/cue.flex`.
 
 The following command regenerates the lexer:
+
 ```bash
 ./gradlew generateLexer
 ```
+
 ### Parser
-The parser is generated with JetBrains' GrammarKit. GrammarKit is a plugin for IntelliJ IDEA.
-The definition is at `src/grammar/cue.bnf`.
 
-To update the parser and all related classes, open the `cue.bnf` file in your IDE and choose `Generate Parser` in the context menu of the editor.
+The parser is generated with JetBrains' GrammarKit. GrammarKit is a plugin for IntelliJ IDEA. The definition is at `src/grammar/cue.bnf`.
+
+To update the parser and all related classes, open the `cue.bnf` file in your IDE and choose `Generate Parser` in the context menu of the
+editor.
 
 ## Useful Link
 

diff --git a/src/grammar/cue.bnf b/src/grammar/cue.bnf
@@ -5,13 +5,12 @@ Please not the following important differences:
 - CUE has ":" to start a rule, GrammarKit has "::="
 - in CUE's grammar {a} means 0 or more "a", but in GrammarKit {} it's just a grouping element.
   GrammarKit's most-similar equivalent: {}*
-- GrammarKit doesn't support left-recursive rules
+- GrammarKit doesn't support left-recursive rules without adjustment to the type hierarchy
  */
 
 {
-    generate=[java="8"]
+    generate=[java="11"]
     consumeTokenMethod(".*")="consumeTokenFast"
-    generateTokenAccessors=false
 
     parserClass="dev.monogon.cue.lang.parser.CueParser"
     parserUtilClass="dev.monogon.cue.lang.parser.CueParserUtil"
@@ -42,12 +41,16 @@ Please not the following important differences:
         REL_OP      = "REL_OP"
         ADD_OP      = "ADD_OP"
         MUL_OP      = "MUL_OP"
-        PIPE        = "|"
-        AMP         = "&"
-        PIPE_PIPE   = "||"
-        AMP_AMP     = "&&"
-        EQ_EQ       = "=="
-        OPERATOR    = "OPERATOR"
+        OP_DISJUNCTION  = "|"
+        OP_UNIFICATION  = "&"
+        OP_OR       = "||"
+        OP_AND      = "&&"
+        OP_EQ       = "=="
+
+        QMARK       = "?"
+        EXCL        = "!"
+        DOT         = "."
+        BOTTOM      = "_|_"
 
         // we're using _END tokens for string terminating tokens,
         // because CueCommaInsertingLexer needs to know where string literals end
@@ -105,16 +108,17 @@ private string_lit ::= simple_string_lit
                    | "#" string_lit "#"
 
 // https://cuelang.org/docs/references/spec/#structs
-StructLit       ::= "{" { Declaration "," }* "}" {extends=Literal}
+// fixme change of grammar rules
+StructLit       ::= "{" [ Declaration {"," Declaration}* ","? ] "}" {extends=Literal pin=1}
 Declaration     ::= Field | Ellipsis | Embedding | LetClause | attribute
-Ellipsis        ::= "..." [ Expression ] {extends=Declaration}
+Ellipsis        ::= "..." [ Expression ] // use by both Declaration and ListLit
 Embedding       ::= Comprehension | AliasExpr {extends=Declaration}
 Field           ::= Label ":" { Label ":" }* Expression { attribute }* {extends=Declaration}
 Label           ::= [ IDENTIFIER "=" ] LabelExpr
-LabelExpr       ::= LabelName [ "?" ] | "[" AliasExpr "]"
-private LabelName       ::= IDENTIFIER | simple_string_lit
+LabelExpr       ::= LabelName ["?"] | "[" AliasExpr "]"
+private LabelName ::= <<struct_label>> | simple_string_lit
 
-attribute       ::= "@" IDENTIFIER "(" attr_tokens ")"
+attribute       ::= "@" IDENTIFIER "(" attr_tokens ")" {pin=1}
 attr_tokens     ::= { <<attr_token>> // fixme psi element for attr_token?
                       | "(" attr_tokens ")"
                       | "[" attr_tokens "]"
@@ -125,7 +129,8 @@ attr_tokens     ::= { <<attr_token>> // fixme psi element for attr_token?
 AliasExpr  ::= Expression | IDENTIFIER "=" Expression { extends=Expression }
 
 // fixme added closing ] at the end, bug in grammar
-ListLit       ::= "[" [ ElementList [ "," [ Ellipsis ] ] [ "," ] ] "]" {extends=Literal}
+ListLit       ::= "[" [ListLitInner | Ellipsis] "]" {extends=Literal pin=1}
+private ListLitInner ::= ElementList ["," Ellipsis] [","]
 ElementList   ::= Embedding { "," Embedding }*
 
 // https://cuelang.org/docs/references/spec/#expressions
@@ -142,21 +147,26 @@ QualifiedIdent ::= PackageName "." IDENTIFIER {extends=Operand}
 PrimaryExpr ::= Operand {Selector | Index | Slice | Arguments}* {extends=Expression}
 
 Selector       ::= "." (IDENTIFIER | simple_string_lit) {extends=PrimaryExpr}
-Index          ::= "[" Expression "]" {extends=PrimaryExpr}
+Index          ::= "[" Expression "]" {extends=PrimaryExpr pin=1}
 Argument       ::= Expression {extends=PrimaryExpr}
-Arguments      ::= "(" [ ( Argument { "," Argument }* ) [ "," ] ] ")" {extends=PrimaryExpr}
+Arguments      ::= "(" [ ( Argument { "," Argument }* ) [ "," ] ] ")" {extends=PrimaryExpr pin=1}
 // fixme Slice is missing
 
 // https://cuelang.org/docs/references/spec/#operators
 Expression ::= UnaryExpr | BinaryExpr // fixme extra root?
 UnaryExpr  ::= PrimaryExpr | unary_op UnaryExpr { extends=Expression }
 BinaryExpr ::= Expression binary_op Expression { extends=Expression }
 
-private binary_op  ::= PIPE | AMP | PIPE_PIPE | AMP_AMP | EQ_EQ | rel_op | add_op | mul_op //"|" | "&" | "||" | "&&" | "==" | rel_op | add_op | mul_op
-private rel_op     ::= REL_OP //"!=" | "<" | "<=" | ">" | ">=" | "=~" | "!~"
-private add_op     ::= ADD_OP //"+" | "-"
-private mul_op     ::= MUL_OP //"*" | "/" | "div" | "mod" | "quo" | "rem"
-private unary_op   ::= ADD_OP | "!" | "*" | rel_op // "+" | "-" | "!" | "*" | rel_op
+//"|" | "&" | "||" | "&&" | "==" | rel_op | add_op | mul_op
+private binary_op  ::= OP_DISJUNCTION | OP_UNIFICATION | OP_OR | OP_AND | OP_EQ | rel_op | add_op | mul_op
+//"!=" | "<" | "<=" | ">" | ">=" | "=~" | "!~"
+private rel_op     ::= REL_OP
+//"+" | "-"
+private add_op     ::= ADD_OP
+//"*" | "/" | "div" | "mod" | "quo" | "rem"
+private mul_op     ::= MUL_OP
+// "+" | "-" | "!" | "*" | rel_op
+private unary_op   ::= ADD_OP | EXCL | "*" | rel_op
 
 // https://cuelang.org/docs/references/spec/#comprehensions
 Comprehension       ::= Clauses StructLit

diff --git a/src/grammar/cue.flex b/src/grammar/cue.flex
@@ -48,9 +48,10 @@ import static dev.monogon.cue.lang.CueTokenTypes.*;
 %type IElementType
 
 // White space, formed from spaces (U+0020), horizontal tabs (U+0009), carriage returns (U+000D), and newlines (U+000A)
-// IntelliJ: tokenizing newlines, which are whitespace, as separate tokens to simplify out CueCommaInsertingLexer
+// IntelliJ: tokenizing newlines, which are whitespace, as separate tokens to simplify our CueCommaInsertingLexer
 WHITE_SPACE=[ \t\r]
 WHITE_SPACE_NEWLINE=[\n]
+WHITE_SPACE_ANY=[ \t\r\n]
 
 // https://cuelang.org/docs/references/spec/#characters
 newline        = \n /* the Unicode code point U+000A */
@@ -75,7 +76,7 @@ identifier  = ("#" | "_#")? {letter} {letter_digit}*
 // https://cuelang.org/docs/references/spec/#letters-and-digits
 decimal_lit = [1-9] ("_"? {decimal_digit})*
 decimals    = {decimal_digit} ("_"? {decimal_digit})*
-si_lit      =   {decimals} ("." {decimals})? {multiplier}
+si_lit      = {decimals} ("." {decimals})? {multiplier}
               | "." {decimals} {multiplier}
 binary_lit  = "0b" {binary_digit} ("_"? {binary_digit})*
 hex_lit     = "0" [xX] {hex_digit} ("_"? {hex_digit})*
@@ -89,23 +90,6 @@ float_lit = {decimals} "." {decimals}? {exponent}?
             | "." {decimals} {exponent}?
 exponent  = [eE] [+-]? {decimals}
 
-// https://cuelang.org/docs/references/spec/#null
-null_lit = "null"
-
-// fixme undefined in spec
-bool_lit = "true" | "false"
-
-// https://cuelang.org/docs/references/spec/#keywords
-keyword_preamble        = "package" | "import"
-keyword_comprehensions  = "for" | "in" | "if" | "let"
-//keyword_arithmetic      = "div" | "mod" | "quo" | "rem"
-
-// https://cuelang.org/docs/references/spec/#operators-and-punctuation
-// most operators are matched below
-operators   = "?" | "!"
-            | "_|_"
-            | "."
-
 // https://cuelang.org/docs/references/spec/#string-and-byte-sequence-literals
 escaped_char     = "\\" "#"* [abfnrtv/\\'\"]
 byte_value       = {octal_byte_value} | {hex_byte_value}
@@ -153,24 +137,25 @@ interpolation_end = ")"
 }
 
 <YYINITIAL, EXPRESSION> {
-    {keyword_preamble} |
-    {keyword_comprehensions}
-                    { return KEYWORD; } // for now, a single token
+    "package" | "import"
+    | "for" | "in" | "if" | "let"
+                     { return KEYWORD; } // for now, one token for all, https://cuelang.org/docs/references/spec/#keywords
+    "null"           { return NULL_LIT; } // https://cuelang.org/docs/references/spec/#null
+    "true" | "false" { return BOOL_LIT; } // fixme currently undefined in spec
 
-    {null_lit}       { return NULL_LIT; }
-    {bool_lit}       { return BOOL_LIT; }
+    "_|_"            { return BOTTOM; }
 
-// operator tokens
+    // operator tokens
     "{"     { return LEFT_CURLY; }
     "}"     { return RIGHT_CURLY; }
     ":"     { return COLON; }
     "..."   { return ELLIPSIS_TOKEN; }
     ","     { return COMMA; }
     "="     { return EQ; }
     "?"     { return QMARK; }
+    "@"     { return AT; }
     "["     { return LEFT_BRACKET; }
     "]"     { return RIGHT_BRACKET; }
-    "@"     { return AT; }
     "("     { return LEFT_PAREN; }
     ")"     { return RIGHT_PAREN; }
 
@@ -180,31 +165,36 @@ interpolation_end = ")"
             { return ADD_OP; }
     "*" | "/" | "div" | "mod" | "quo" | "rem"
             { return MUL_OP; }
-    "|"     { return PIPE; }
-    "&"     { return AMP; }
-    "||"    { return PIPE_PIPE; }
-    "&&"    { return AMP_AMP; }
-    "=="    { return EQ_EQ; }
-    {operators}     { return OPERATOR; } // for now, a single token
-// end of operators
+    "|"     { return OP_DISJUNCTION; }
+    "&"     { return OP_UNIFICATION; }
+    "||"    { return OP_OR; }
+    "&&"    { return OP_AND; }
+    "=="    { return OP_EQ; }
+    // end of operators
+
+    "?"     { return QMARK; }
+    "!"     { return EXCL; }
+    "."     { return DOT; }
+
     {identifier}    { return IDENTIFIER; }
 
     {float_lit}     { return FLOAT_LIT; }
     {decimal_lit}
-     | {si_lit}
-     | {octal_lit}
-     | {binary_lit}
-     | {hex_lit}     { return INT_LIT; }
+    | {si_lit}
+    | {octal_lit}
+    | {binary_lit}
+    | {hex_lit}     { return INT_LIT; }
+    "0" // fixme temporary change, "0" seems to be missing in the lang spec
+                    { return INT_LIT; }
 
     "\""                   { pushState(STRING_LITERAL); return DOUBLE_QUOTE; }
-    "'"                    { pushState(BYTE_LITERAL); return SINGLE_QUOTE; }
     "\"\"\"" / {newline}   { pushState(STRING_MULTILINE); return MULTILINE_STRING_START; }
+    "'"                    { pushState(BYTE_LITERAL); return SINGLE_QUOTE; }
     "'''" / {newline}      { pushState(BYTES_MULTILINE); return MULTILINE_BYTES_START; }
 
     "//" {unicode_char}*   { return COMMENT; }
 }
 
-// all states
 {WHITE_SPACE_NEWLINE}   { return WHITE_SPACE_NEWLINE; }
 {WHITE_SPACE}           { return WHITE_SPACE; }
 [^]                     { return BAD_CHARACTER; }
diff --git a/src/main/java-gen/dev/monogon/cue/lang/CueTypes.java b/src/main/java-gen/dev/monogon/cue/lang/CueTypes.java