From c93291cdaa7f2605d0d5d19346017f56bef28611 Mon Sep 17 00:00:00 2001 From: wrapperup Date: Wed, 6 Mar 2024 18:47:47 -0500 Subject: [PATCH] simplify grammar --- corpus/main.txt | 37 ++- grammar.js | 39 ++- src/grammar.json | 151 +++++------ src/node-types.json | 2 +- src/parser.c | 600 ++++++++++++++++---------------------------- src/scanner.c | 18 +- 6 files changed, 328 insertions(+), 519 deletions(-) diff --git a/corpus/main.txt b/corpus/main.txt index 4fa37a3..8a96d16 100644 --- a/corpus/main.txt +++ b/corpus/main.txt @@ -32,7 +32,6 @@ Variables (keyword)) (content)) - ==================== Comments ==================== @@ -185,3 +184,39 @@ Expressions (tag (keyword)) (content)) + +==================== +Expressions 2 +==================== + +{{ if test }} +{{}} +{{ }} +{{> javascriptCode() }} +{{-> javascriptCode() }} +{{> javascriptCode() -}} + +--- + +(template + (content) + (tag + (keyword) + (code)) + (content) + (tag) + (content) + (tag) + (content) + (tag + (keyword) + (code)) + (content) + (tag + (keyword) + (code)) + (content) + (tag + (keyword) + (code)) + (content)) diff --git a/grammar.js b/grammar.js index 6e0e959..962cde6 100644 --- a/grammar.js +++ b/grammar.js @@ -2,41 +2,38 @@ module.exports = grammar({ name: 'vento', externals: $ => [ $.code, - $.keyword, ], rules: { template: $ => repeat(choice( $.tag, - alias($.comment_tag, $.tag), $.content )), content: $ => prec.right(/[^(\{\{)]+/), - keyword: $ => /[a-zA-Z0-9\(\)\.]+/, - close_keyword: $ => /\/[a-zA-Z]+/, - - codeSnippet: $ => /[^\/][\s]+/, + tag: $ => seq( + choice("{{", "{{-"), + optional($._expression), + optional($.filter), + choice("}}", "-}}") + ), _expression: $ => choice( - // "Solo keywords" aren't a valid expression, - // but we need to handle them here to avoid - // a conflict with the the sequence of a keyword - // and a code block. - alias(choice($.keyword, $.codeSnippet), $.code), + // "Solo keywords" are just code blocks + alias($.keyword, $.code), alias($.close_keyword, $.keyword), seq( - prec.left(1, $.keyword), + $.keyword, $.code ), + $.comment, ), - tag: $ => seq( - choice("{{", "{{-"), - $._expression, - optional($.filter), - choice("}}", "-}}") - ), + // General rule for keyword tags + // It just tries to match the first word in a tag block, + // plus any other special characters that might be present + keyword: $ => /[a-zA-Z0-9\(\)\.>]+/, + close_keyword: $ => /\/[a-zA-Z]+/, filter: $ => repeat1(seq( "|>", @@ -44,11 +41,5 @@ module.exports = grammar({ )), comment: $ => /#[^#]+#/, - - comment_tag: $ => seq( - "{{", - $.comment, - "}}" - ), } }); diff --git a/src/grammar.json b/src/grammar.json index 91d37e7..2e927b9 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -10,15 +10,6 @@ "type": "SYMBOL", "name": "tag" }, - { - "type": "ALIAS", - "content": { - "type": "SYMBOL", - "name": "comment_tag" - }, - "named": true, - "value": "tag" - }, { "type": "SYMBOL", "name": "content" @@ -34,67 +25,6 @@ "value": "[^(\\{\\{)]+" } }, - "keyword": { - "type": "PATTERN", - "value": "[a-zA-Z0-9\\(\\)\\.]+" - }, - "close_keyword": { - "type": "PATTERN", - "value": "\\/[a-zA-Z]+" - }, - "codeSnippet": { - "type": "PATTERN", - "value": "[^\\/][\\s]+" - }, - "_expression": { - "type": "CHOICE", - "members": [ - { - "type": "ALIAS", - "content": { - "type": "CHOICE", - "members": [ - { - "type": "SYMBOL", - "name": "keyword" - }, - { - "type": "SYMBOL", - "name": "codeSnippet" - } - ] - }, - "named": true, - "value": "code" - }, - { - "type": "ALIAS", - "content": { - "type": "SYMBOL", - "name": "close_keyword" - }, - "named": true, - "value": "keyword" - }, - { - "type": "SEQ", - "members": [ - { - "type": "PREC_LEFT", - "value": 1, - "content": { - "type": "SYMBOL", - "name": "keyword" - } - }, - { - "type": "SYMBOL", - "name": "code" - } - ] - } - ] - }, "tag": { "type": "SEQ", "members": [ @@ -112,8 +42,16 @@ ] }, { - "type": "SYMBOL", - "name": "_expression" + "type": "CHOICE", + "members": [ + { + "type": "SYMBOL", + "name": "_expression" + }, + { + "type": "BLANK" + } + ] }, { "type": "CHOICE", @@ -142,6 +80,54 @@ } ] }, + "_expression": { + "type": "CHOICE", + "members": [ + { + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "keyword" + }, + "named": true, + "value": "code" + }, + { + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "close_keyword" + }, + "named": true, + "value": "keyword" + }, + { + "type": "SEQ", + "members": [ + { + "type": "SYMBOL", + "name": "keyword" + }, + { + "type": "SYMBOL", + "name": "code" + } + ] + }, + { + "type": "SYMBOL", + "name": "comment" + } + ] + }, + "keyword": { + "type": "PATTERN", + "value": "[a-zA-Z0-9\\(\\)\\.>]+" + }, + "close_keyword": { + "type": "PATTERN", + "value": "\\/[a-zA-Z]+" + }, "filter": { "type": "REPEAT1", "content": { @@ -161,23 +147,6 @@ "comment": { "type": "PATTERN", "value": "#[^#]+#" - }, - "comment_tag": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "{{" - }, - { - "type": "SYMBOL", - "name": "comment" - }, - { - "type": "STRING", - "value": "}}" - } - ] } }, "extras": [ @@ -192,10 +161,6 @@ { "type": "SYMBOL", "name": "code" - }, - { - "type": "SYMBOL", - "name": "keyword" } ], "inline": [], diff --git a/src/node-types.json b/src/node-types.json index 06542df..0a8ff31 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -25,7 +25,7 @@ "fields": {}, "children": { "multiple": true, - "required": true, + "required": false, "types": [ { "type": "code", diff --git a/src/parser.c b/src/parser.c index 4486364..66ce914 100644 --- a/src/parser.c +++ b/src/parser.c @@ -6,57 +6,53 @@ #endif #define LANGUAGE_VERSION 14 -#define STATE_COUNT 21 +#define STATE_COUNT 19 #define LARGE_STATE_COUNT 2 -#define SYMBOL_COUNT 20 +#define SYMBOL_COUNT 18 #define ALIAS_COUNT 0 -#define TOKEN_COUNT 12 -#define EXTERNAL_TOKEN_COUNT 2 +#define TOKEN_COUNT 11 +#define EXTERNAL_TOKEN_COUNT 1 #define FIELD_COUNT 0 #define MAX_ALIAS_SEQUENCE_LENGTH 4 #define PRODUCTION_ID_COUNT 2 enum { aux_sym_content_token1 = 1, - sym_keyword = 2, - sym_close_keyword = 3, - sym_codeSnippet = 4, - anon_sym_LBRACE_LBRACE = 5, - anon_sym_LBRACE_LBRACE_DASH = 6, - anon_sym_RBRACE_RBRACE = 7, - anon_sym_DASH_RBRACE_RBRACE = 8, - anon_sym_PIPE_GT = 9, - sym_comment = 10, - sym_code = 11, - sym_template = 12, - sym_content = 13, + anon_sym_LBRACE_LBRACE = 2, + anon_sym_LBRACE_LBRACE_DASH = 3, + anon_sym_RBRACE_RBRACE = 4, + anon_sym_DASH_RBRACE_RBRACE = 5, + sym_keyword = 6, + sym_close_keyword = 7, + anon_sym_PIPE_GT = 8, + sym_comment = 9, + sym_code = 10, + sym_template = 11, + sym_content = 12, + sym_tag = 13, sym__expression = 14, - sym_tag = 15, - sym_filter = 16, - sym_comment_tag = 17, - aux_sym_template_repeat1 = 18, - aux_sym_filter_repeat1 = 19, + sym_filter = 15, + aux_sym_template_repeat1 = 16, + aux_sym_filter_repeat1 = 17, }; static const char * const ts_symbol_names[] = { [ts_builtin_sym_end] = "end", [aux_sym_content_token1] = "content_token1", - [sym_keyword] = "keyword", - [sym_close_keyword] = "keyword", - [sym_codeSnippet] = "code", [anon_sym_LBRACE_LBRACE] = "{{", [anon_sym_LBRACE_LBRACE_DASH] = "{{-", [anon_sym_RBRACE_RBRACE] = "}}", [anon_sym_DASH_RBRACE_RBRACE] = "-}}", + [sym_keyword] = "keyword", + [sym_close_keyword] = "keyword", [anon_sym_PIPE_GT] = "|>", [sym_comment] = "comment", [sym_code] = "code", [sym_template] = "template", [sym_content] = "content", - [sym__expression] = "_expression", [sym_tag] = "tag", + [sym__expression] = "_expression", [sym_filter] = "filter", - [sym_comment_tag] = "tag", [aux_sym_template_repeat1] = "template_repeat1", [aux_sym_filter_repeat1] = "filter_repeat1", }; @@ -64,22 +60,20 @@ static const char * const ts_symbol_names[] = { static const TSSymbol ts_symbol_map[] = { [ts_builtin_sym_end] = ts_builtin_sym_end, [aux_sym_content_token1] = aux_sym_content_token1, - [sym_keyword] = sym_keyword, - [sym_close_keyword] = sym_keyword, - [sym_codeSnippet] = sym_code, [anon_sym_LBRACE_LBRACE] = anon_sym_LBRACE_LBRACE, [anon_sym_LBRACE_LBRACE_DASH] = anon_sym_LBRACE_LBRACE_DASH, [anon_sym_RBRACE_RBRACE] = anon_sym_RBRACE_RBRACE, [anon_sym_DASH_RBRACE_RBRACE] = anon_sym_DASH_RBRACE_RBRACE, + [sym_keyword] = sym_keyword, + [sym_close_keyword] = sym_keyword, [anon_sym_PIPE_GT] = anon_sym_PIPE_GT, [sym_comment] = sym_comment, [sym_code] = sym_code, [sym_template] = sym_template, [sym_content] = sym_content, - [sym__expression] = sym__expression, [sym_tag] = sym_tag, + [sym__expression] = sym__expression, [sym_filter] = sym_filter, - [sym_comment_tag] = sym_tag, [aux_sym_template_repeat1] = aux_sym_template_repeat1, [aux_sym_filter_repeat1] = aux_sym_filter_repeat1, }; @@ -93,18 +87,6 @@ static const TSSymbolMetadata ts_symbol_metadata[] = { .visible = false, .named = false, }, - [sym_keyword] = { - .visible = true, - .named = true, - }, - [sym_close_keyword] = { - .visible = true, - .named = true, - }, - [sym_codeSnippet] = { - .visible = true, - .named = true, - }, [anon_sym_LBRACE_LBRACE] = { .visible = true, .named = false, @@ -121,6 +103,14 @@ static const TSSymbolMetadata ts_symbol_metadata[] = { .visible = true, .named = false, }, + [sym_keyword] = { + .visible = true, + .named = true, + }, + [sym_close_keyword] = { + .visible = true, + .named = true, + }, [anon_sym_PIPE_GT] = { .visible = true, .named = false, @@ -141,19 +131,15 @@ static const TSSymbolMetadata ts_symbol_metadata[] = { .visible = true, .named = true, }, - [sym__expression] = { - .visible = false, - .named = true, - }, [sym_tag] = { .visible = true, .named = true, }, - [sym_filter] = { - .visible = true, + [sym__expression] = { + .visible = false, .named = true, }, - [sym_comment_tag] = { + [sym_filter] = { .visible = true, .named = true, }, @@ -198,8 +184,6 @@ static const TSStateId ts_primary_state_ids[STATE_COUNT] = { [16] = 16, [17] = 17, [18] = 18, - [19] = 19, - [20] = 20, }; static bool ts_lex(TSLexer *lexer, TSStateId state) { @@ -207,13 +191,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { eof = lexer->eof(lexer); switch (state) { case 0: - if (eof) ADVANCE(16); - if (lookahead == '#') ADVANCE(14); - if (lookahead == '-') ADVANCE(10); - if (lookahead == '/') ADVANCE(13); - if (lookahead == '{') ADVANCE(7); - if (lookahead == '|') ADVANCE(6); - if (lookahead == '}') ADVANCE(8); + if (eof) ADVANCE(10); + if (lookahead == '#') ADVANCE(8); + if (lookahead == '-') ADVANCE(6); + if (lookahead == '/') ADVANCE(7); + if (lookahead == '{') ADVANCE(3); + if (lookahead == '|') ADVANCE(2); + if (lookahead == '}') ADVANCE(4); if (lookahead == '\t' || lookahead == '\n' || lookahead == '\r' || @@ -221,227 +205,101 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == '(' || lookahead == ')' || ('.' <= lookahead && lookahead <= '9') || + lookahead == '>' || ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(20); + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(17); END_STATE(); case 1: - if (lookahead == '#') ADVANCE(31); + if (lookahead == '#') ADVANCE(20); if (lookahead != 0) ADVANCE(1); END_STATE(); case 2: - if (lookahead == '#') ADVANCE(11); - if (lookahead == '/') ADVANCE(13); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(23); - if (lookahead == '(' || - lookahead == ')' || - ('.' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(19); - if (lookahead != 0) ADVANCE(12); + if (lookahead == '>') ADVANCE(19); END_STATE(); case 3: - if (lookahead == '#') ADVANCE(11); - if (lookahead == '/') ADVANCE(13); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(2); - if (lookahead == '(' || - lookahead == ')' || - ('.' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(19); - if (lookahead != 0) ADVANCE(12); + if (lookahead == '{') ADVANCE(13); END_STATE(); case 4: - if (lookahead == '/') ADVANCE(13); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(25); - if (lookahead == '(' || - lookahead == ')' || - ('.' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(19); - if (lookahead != 0) ADVANCE(12); + if (lookahead == '}') ADVANCE(15); END_STATE(); case 5: - if (lookahead == '/') ADVANCE(13); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(4); - if (lookahead == '(' || - lookahead == ')' || - ('.' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(19); - if (lookahead != 0) ADVANCE(12); + if (lookahead == '}') ADVANCE(16); END_STATE(); case 6: - if (lookahead == '>') ADVANCE(30); + if (lookahead == '}') ADVANCE(5); END_STATE(); case 7: - if (lookahead == '{') ADVANCE(26); - END_STATE(); - case 8: - if (lookahead == '}') ADVANCE(28); - END_STATE(); - case 9: - if (lookahead == '}') ADVANCE(29); - END_STATE(); - case 10: - if (lookahead == '}') ADVANCE(9); - END_STATE(); - case 11: - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(22); - if (lookahead != 0 && - lookahead != '#') ADVANCE(1); - END_STATE(); - case 12: - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(24); - END_STATE(); - case 13: if (('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(21); + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(18); END_STATE(); - case 14: + case 8: if (lookahead != 0 && lookahead != '#') ADVANCE(1); END_STATE(); - case 15: - if (eof) ADVANCE(16); - if (lookahead == '{') ADVANCE(7); + case 9: + if (eof) ADVANCE(10); + if (lookahead == '{') ADVANCE(3); if (lookahead == '\t' || lookahead == '\n' || lookahead == '\r' || - lookahead == ' ') ADVANCE(17); + lookahead == ' ') ADVANCE(11); if (lookahead != 0 && lookahead != '(' && - lookahead != ')') ADVANCE(18); + lookahead != ')') ADVANCE(12); END_STATE(); - case 16: + case 10: ACCEPT_TOKEN(ts_builtin_sym_end); END_STATE(); - case 17: + case 11: ACCEPT_TOKEN(aux_sym_content_token1); if (lookahead == '\t' || lookahead == '\n' || lookahead == '\r' || - lookahead == ' ') ADVANCE(17); + lookahead == ' ') ADVANCE(11); if (lookahead != 0 && lookahead != '(' && lookahead != ')' && - lookahead != '{') ADVANCE(18); + lookahead != '{') ADVANCE(12); END_STATE(); - case 18: + case 12: ACCEPT_TOKEN(aux_sym_content_token1); if (lookahead != 0 && lookahead != '(' && lookahead != ')' && - lookahead != '{') ADVANCE(18); + lookahead != '{') ADVANCE(12); END_STATE(); - case 19: - ACCEPT_TOKEN(sym_keyword); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(24); - if (lookahead == '(' || - lookahead == ')' || - lookahead == '.' || - ('0' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(20); - END_STATE(); - case 20: - ACCEPT_TOKEN(sym_keyword); - if (lookahead == '(' || - lookahead == ')' || - lookahead == '.' || - ('0' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(20); - END_STATE(); - case 21: - ACCEPT_TOKEN(sym_close_keyword); - if (('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(21); + case 13: + ACCEPT_TOKEN(anon_sym_LBRACE_LBRACE); + if (lookahead == '-') ADVANCE(14); END_STATE(); - case 22: - ACCEPT_TOKEN(sym_codeSnippet); - if (lookahead == '#') ADVANCE(31); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(22); - if (lookahead != 0) ADVANCE(1); + case 14: + ACCEPT_TOKEN(anon_sym_LBRACE_LBRACE_DASH); END_STATE(); - case 23: - ACCEPT_TOKEN(sym_codeSnippet); - if (lookahead == '#') ADVANCE(11); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(23); - if (lookahead == '(' || - lookahead == ')' || - lookahead == '.' || - ('0' <= lookahead && lookahead <= '9') || - ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(19); - if (lookahead != 0 && - lookahead != '/') ADVANCE(12); + case 15: + ACCEPT_TOKEN(anon_sym_RBRACE_RBRACE); END_STATE(); - case 24: - ACCEPT_TOKEN(sym_codeSnippet); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(24); + case 16: + ACCEPT_TOKEN(anon_sym_DASH_RBRACE_RBRACE); END_STATE(); - case 25: - ACCEPT_TOKEN(sym_codeSnippet); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') ADVANCE(25); + case 17: + ACCEPT_TOKEN(sym_keyword); if (lookahead == '(' || lookahead == ')' || lookahead == '.' || ('0' <= lookahead && lookahead <= '9') || + lookahead == '>' || ('A' <= lookahead && lookahead <= 'Z') || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(19); - if (lookahead != 0 && - lookahead != '/') ADVANCE(12); - END_STATE(); - case 26: - ACCEPT_TOKEN(anon_sym_LBRACE_LBRACE); - if (lookahead == '-') ADVANCE(27); + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(17); END_STATE(); - case 27: - ACCEPT_TOKEN(anon_sym_LBRACE_LBRACE_DASH); - END_STATE(); - case 28: - ACCEPT_TOKEN(anon_sym_RBRACE_RBRACE); - END_STATE(); - case 29: - ACCEPT_TOKEN(anon_sym_DASH_RBRACE_RBRACE); + case 18: + ACCEPT_TOKEN(sym_close_keyword); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(18); END_STATE(); - case 30: + case 19: ACCEPT_TOKEN(anon_sym_PIPE_GT); END_STATE(); - case 31: + case 20: ACCEPT_TOKEN(sym_comment); END_STATE(); default: @@ -451,276 +309,246 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { static const TSLexMode ts_lex_modes[STATE_COUNT] = { [0] = {.lex_state = 0, .external_lex_state = 1}, - [1] = {.lex_state = 15}, - [2] = {.lex_state = 15}, - [3] = {.lex_state = 15}, - [4] = {.lex_state = 3, .external_lex_state = 2}, + [1] = {.lex_state = 9}, + [2] = {.lex_state = 0}, + [3] = {.lex_state = 9}, + [4] = {.lex_state = 9}, [5] = {.lex_state = 0}, - [6] = {.lex_state = 15}, - [7] = {.lex_state = 15}, - [8] = {.lex_state = 5, .external_lex_state = 2}, - [9] = {.lex_state = 0, .external_lex_state = 3}, - [10] = {.lex_state = 0}, - [11] = {.lex_state = 15}, - [12] = {.lex_state = 15}, + [6] = {.lex_state = 9}, + [7] = {.lex_state = 9}, + [8] = {.lex_state = 0, .external_lex_state = 1}, + [9] = {.lex_state = 0}, + [10] = {.lex_state = 9}, + [11] = {.lex_state = 0}, + [12] = {.lex_state = 9}, [13] = {.lex_state = 0}, [14] = {.lex_state = 0}, [15] = {.lex_state = 0}, [16] = {.lex_state = 0}, [17] = {.lex_state = 0}, - [18] = {.lex_state = 0}, - [19] = {.lex_state = 0, .external_lex_state = 3}, - [20] = {.lex_state = 0}, + [18] = {.lex_state = 0, .external_lex_state = 1}, }; enum { ts_external_token_code = 0, - ts_external_token_keyword = 1, }; static const TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = { [ts_external_token_code] = sym_code, - [ts_external_token_keyword] = sym_keyword, }; -static const bool ts_external_scanner_states[4][EXTERNAL_TOKEN_COUNT] = { +static const bool ts_external_scanner_states[2][EXTERNAL_TOKEN_COUNT] = { [1] = { [ts_external_token_code] = true, - [ts_external_token_keyword] = true, - }, - [2] = { - [ts_external_token_keyword] = true, - }, - [3] = { - [ts_external_token_code] = true, }, }; static const uint16_t ts_parse_table[LARGE_STATE_COUNT][SYMBOL_COUNT] = { [0] = { [ts_builtin_sym_end] = ACTIONS(1), - [sym_keyword] = ACTIONS(1), - [sym_close_keyword] = ACTIONS(1), [anon_sym_LBRACE_LBRACE] = ACTIONS(1), [anon_sym_LBRACE_LBRACE_DASH] = ACTIONS(1), [anon_sym_RBRACE_RBRACE] = ACTIONS(1), [anon_sym_DASH_RBRACE_RBRACE] = ACTIONS(1), + [sym_keyword] = ACTIONS(1), + [sym_close_keyword] = ACTIONS(1), [anon_sym_PIPE_GT] = ACTIONS(1), [sym_comment] = ACTIONS(1), [sym_code] = ACTIONS(1), }, [1] = { - [sym_template] = STATE(18), - [sym_content] = STATE(2), - [sym_tag] = STATE(2), - [sym_comment_tag] = STATE(2), - [aux_sym_template_repeat1] = STATE(2), + [sym_template] = STATE(17), + [sym_content] = STATE(3), + [sym_tag] = STATE(3), + [aux_sym_template_repeat1] = STATE(3), [ts_builtin_sym_end] = ACTIONS(3), [aux_sym_content_token1] = ACTIONS(5), [anon_sym_LBRACE_LBRACE] = ACTIONS(7), - [anon_sym_LBRACE_LBRACE_DASH] = ACTIONS(9), + [anon_sym_LBRACE_LBRACE_DASH] = ACTIONS(7), }, }; static const uint16_t ts_small_parse_table[] = { - [0] = 5, + [0] = 7, + ACTIONS(11), 1, + sym_keyword, + ACTIONS(15), 1, + anon_sym_PIPE_GT, + STATE(5), 1, + sym__expression, + STATE(9), 1, + aux_sym_filter_repeat1, + STATE(15), 1, + sym_filter, + ACTIONS(9), 2, + anon_sym_RBRACE_RBRACE, + anon_sym_DASH_RBRACE_RBRACE, + ACTIONS(13), 2, + sym_close_keyword, + sym_comment, + [24] = 4, ACTIONS(5), 1, aux_sym_content_token1, - ACTIONS(7), 1, + ACTIONS(17), 1, + ts_builtin_sym_end, + ACTIONS(7), 2, anon_sym_LBRACE_LBRACE, - ACTIONS(9), 1, anon_sym_LBRACE_LBRACE_DASH, - ACTIONS(11), 1, - ts_builtin_sym_end, - STATE(3), 4, + STATE(4), 3, sym_content, sym_tag, - sym_comment_tag, aux_sym_template_repeat1, - [19] = 5, - ACTIONS(13), 1, + [40] = 4, + ACTIONS(19), 1, ts_builtin_sym_end, - ACTIONS(15), 1, + ACTIONS(21), 1, aux_sym_content_token1, - ACTIONS(18), 1, + ACTIONS(24), 2, anon_sym_LBRACE_LBRACE, - ACTIONS(21), 1, anon_sym_LBRACE_LBRACE_DASH, - STATE(3), 4, + STATE(4), 3, sym_content, sym_tag, - sym_comment_tag, aux_sym_template_repeat1, - [38] = 5, - ACTIONS(24), 1, - sym_keyword, - ACTIONS(26), 1, - sym_close_keyword, - ACTIONS(28), 1, - sym_codeSnippet, - ACTIONS(30), 1, - sym_comment, - STATE(5), 1, - sym__expression, - [54] = 4, - ACTIONS(34), 1, + [56] = 4, + ACTIONS(15), 1, anon_sym_PIPE_GT, - STATE(13), 1, + STATE(9), 1, aux_sym_filter_repeat1, - STATE(17), 1, + STATE(16), 1, sym_filter, - ACTIONS(32), 2, + ACTIONS(27), 2, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, - [68] = 2, - ACTIONS(36), 2, + [70] = 2, + ACTIONS(29), 2, ts_builtin_sym_end, aux_sym_content_token1, - ACTIONS(38), 2, + ACTIONS(31), 2, anon_sym_LBRACE_LBRACE, anon_sym_LBRACE_LBRACE_DASH, - [77] = 2, - ACTIONS(40), 2, + [79] = 2, + ACTIONS(33), 2, ts_builtin_sym_end, aux_sym_content_token1, - ACTIONS(42), 2, + ACTIONS(35), 2, anon_sym_LBRACE_LBRACE, anon_sym_LBRACE_LBRACE_DASH, - [86] = 4, - ACTIONS(24), 1, - sym_keyword, - ACTIONS(26), 1, - sym_close_keyword, - ACTIONS(44), 1, - sym_codeSnippet, - STATE(5), 1, - sym__expression, - [99] = 2, - ACTIONS(48), 1, + [88] = 2, + ACTIONS(39), 1, sym_code, - ACTIONS(46), 3, + ACTIONS(37), 3, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, anon_sym_PIPE_GT, - [108] = 3, - ACTIONS(52), 1, + [97] = 3, + ACTIONS(15), 1, anon_sym_PIPE_GT, - STATE(10), 1, + STATE(11), 1, aux_sym_filter_repeat1, - ACTIONS(50), 2, + ACTIONS(41), 2, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, - [119] = 2, - ACTIONS(55), 2, + [108] = 2, + ACTIONS(43), 2, ts_builtin_sym_end, aux_sym_content_token1, - ACTIONS(57), 2, + ACTIONS(45), 2, anon_sym_LBRACE_LBRACE, anon_sym_LBRACE_LBRACE_DASH, + [117] = 3, + ACTIONS(49), 1, + anon_sym_PIPE_GT, + STATE(11), 1, + aux_sym_filter_repeat1, + ACTIONS(47), 2, + anon_sym_RBRACE_RBRACE, + anon_sym_DASH_RBRACE_RBRACE, [128] = 2, - ACTIONS(59), 2, + ACTIONS(52), 2, ts_builtin_sym_end, aux_sym_content_token1, - ACTIONS(61), 2, + ACTIONS(54), 2, anon_sym_LBRACE_LBRACE, anon_sym_LBRACE_LBRACE_DASH, - [137] = 3, - ACTIONS(34), 1, - anon_sym_PIPE_GT, - STATE(10), 1, - aux_sym_filter_repeat1, - ACTIONS(63), 2, - anon_sym_RBRACE_RBRACE, - anon_sym_DASH_RBRACE_RBRACE, - [148] = 1, - ACTIONS(46), 3, + [137] = 1, + ACTIONS(56), 3, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, anon_sym_PIPE_GT, - [154] = 1, - ACTIONS(65), 3, + [143] = 1, + ACTIONS(47), 3, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, anon_sym_PIPE_GT, - [160] = 1, - ACTIONS(50), 3, + [149] = 1, + ACTIONS(27), 2, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, - anon_sym_PIPE_GT, - [166] = 1, - ACTIONS(67), 2, + [154] = 1, + ACTIONS(58), 2, anon_sym_RBRACE_RBRACE, anon_sym_DASH_RBRACE_RBRACE, - [171] = 1, - ACTIONS(69), 1, + [159] = 1, + ACTIONS(60), 1, ts_builtin_sym_end, - [175] = 1, - ACTIONS(71), 1, + [163] = 1, + ACTIONS(62), 1, sym_code, - [179] = 1, - ACTIONS(73), 1, - anon_sym_RBRACE_RBRACE, }; static const uint32_t ts_small_parse_table_map[] = { [SMALL_STATE(2)] = 0, - [SMALL_STATE(3)] = 19, - [SMALL_STATE(4)] = 38, - [SMALL_STATE(5)] = 54, - [SMALL_STATE(6)] = 68, - [SMALL_STATE(7)] = 77, - [SMALL_STATE(8)] = 86, - [SMALL_STATE(9)] = 99, + [SMALL_STATE(3)] = 24, + [SMALL_STATE(4)] = 40, + [SMALL_STATE(5)] = 56, + [SMALL_STATE(6)] = 70, + [SMALL_STATE(7)] = 79, + [SMALL_STATE(8)] = 88, + [SMALL_STATE(9)] = 97, [SMALL_STATE(10)] = 108, - [SMALL_STATE(11)] = 119, + [SMALL_STATE(11)] = 117, [SMALL_STATE(12)] = 128, [SMALL_STATE(13)] = 137, - [SMALL_STATE(14)] = 148, - [SMALL_STATE(15)] = 154, - [SMALL_STATE(16)] = 160, - [SMALL_STATE(17)] = 166, - [SMALL_STATE(18)] = 171, - [SMALL_STATE(19)] = 175, - [SMALL_STATE(20)] = 179, + [SMALL_STATE(14)] = 143, + [SMALL_STATE(15)] = 149, + [SMALL_STATE(16)] = 154, + [SMALL_STATE(17)] = 159, + [SMALL_STATE(18)] = 163, }; static const TSParseActionEntry ts_parse_actions[] = { [0] = {.entry = {.count = 0, .reusable = false}}, [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), [3] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_template, 0), - [5] = {.entry = {.count = 1, .reusable = true}}, SHIFT(7), - [7] = {.entry = {.count = 1, .reusable = false}}, SHIFT(4), - [9] = {.entry = {.count = 1, .reusable = false}}, SHIFT(8), - [11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_template, 1), - [13] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_template_repeat1, 2), - [15] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_template_repeat1, 2), SHIFT_REPEAT(7), - [18] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_template_repeat1, 2), SHIFT_REPEAT(4), - [21] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_template_repeat1, 2), SHIFT_REPEAT(8), - [24] = {.entry = {.count = 1, .reusable = false}}, SHIFT(9), - [26] = {.entry = {.count = 1, .reusable = false}}, SHIFT(5), - [28] = {.entry = {.count = 1, .reusable = false}}, SHIFT(14), - [30] = {.entry = {.count = 1, .reusable = false}}, SHIFT(20), - [32] = {.entry = {.count = 1, .reusable = true}}, SHIFT(12), - [34] = {.entry = {.count = 1, .reusable = true}}, SHIFT(19), - [36] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_tag, 4), - [38] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_tag, 4), - [40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_content, 1), - [42] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_content, 1), - [44] = {.entry = {.count = 1, .reusable = true}}, SHIFT(14), - [46] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__expression, 1, .production_id = 1), - [48] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15), - [50] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_filter_repeat1, 2), - [52] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_filter_repeat1, 2), SHIFT_REPEAT(19), - [55] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_comment_tag, 3), - [57] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_comment_tag, 3), - [59] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_tag, 3), - [61] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_tag, 3), - [63] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_filter, 1), - [65] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__expression, 2), - [67] = {.entry = {.count = 1, .reusable = true}}, SHIFT(6), - [69] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), - [71] = {.entry = {.count = 1, .reusable = true}}, SHIFT(16), - [73] = {.entry = {.count = 1, .reusable = true}}, SHIFT(11), + [5] = {.entry = {.count = 1, .reusable = true}}, SHIFT(6), + [7] = {.entry = {.count = 1, .reusable = false}}, SHIFT(2), + [9] = {.entry = {.count = 1, .reusable = true}}, SHIFT(7), + [11] = {.entry = {.count = 1, .reusable = true}}, SHIFT(8), + [13] = {.entry = {.count = 1, .reusable = true}}, SHIFT(5), + [15] = {.entry = {.count = 1, .reusable = true}}, SHIFT(18), + [17] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_template, 1), + [19] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_template_repeat1, 2), + [21] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_template_repeat1, 2), SHIFT_REPEAT(6), + [24] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_template_repeat1, 2), SHIFT_REPEAT(2), + [27] = {.entry = {.count = 1, .reusable = true}}, SHIFT(10), + [29] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_content, 1), + [31] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_content, 1), + [33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_tag, 2), + [35] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_tag, 2), + [37] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__expression, 1, .production_id = 1), + [39] = {.entry = {.count = 1, .reusable = true}}, SHIFT(13), + [41] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_filter, 1), + [43] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_tag, 3), + [45] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_tag, 3), + [47] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_filter_repeat1, 2), + [49] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_filter_repeat1, 2), SHIFT_REPEAT(18), + [52] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_tag, 4), + [54] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_tag, 4), + [56] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__expression, 2), + [58] = {.entry = {.count = 1, .reusable = true}}, SHIFT(12), + [60] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), + [62] = {.entry = {.count = 1, .reusable = true}}, SHIFT(14), }; #ifdef __cplusplus diff --git a/src/scanner.c b/src/scanner.c index 2d7a1e1..a3d0b2f 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -43,19 +43,6 @@ bool tree_sitter_vento_external_scanner_scan( skip_whitespace(lexer); if (valid_symbols[CODE]) { - // if (iswalpha(lexer->lookahead)) { - // while (iswalpha(lexer->lookahead)) { - // advance(lexer); - // } - // - // skip_whitespace(lexer); - // - // if (lexer->lookahead != '}') { - // return false; - // } - // } - // - // lexer->mark_end(lexer); if (lexer->lookahead == '\0' || lexer->lookahead == '/') { return false; @@ -82,8 +69,11 @@ bool tree_sitter_vento_external_scanner_scan( depth++; } else if (lexer->lookahead == '}') { - lexer->mark_end(lexer); advance(lexer); + + if (depth > 1) { + lexer->mark_end(lexer); + } depth--; } else if (lexer->lookahead == '|') {