Grammar: Fix a bunch of issues related to symbol fragmentation and al…

…lowed format specifiers (#14)

Related to open-goal/jak-project#3535

grammar.js

@@ -67,11 +67,11 @@ const INTEGER =
 
 // TODO - does OG support negative hex/binary?
 const NUMBER =
-  token(prec(10, seq(optional(/[+-]/),
+  token(seq(optional(/[+-]/),
     choice(HEX_NUMBER,
       BINARY_NUMBER,
       FLOAT,
-      INTEGER))));
+      INTEGER)));
 
 const NULL =
   token('none');
@@ -117,16 +117,19 @@ const CHARACTER =
 // \u205f => <medium mathematical space>
 // \u3000 => <ideographic space>
 const SYMBOL_HEAD =
-  /[^\f\n\r\t \/()\[\]{}"@~^;`\\,:#'0-9\u000B\u001C\u001D\u001E\u001F\u2028\u2029\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2008\u2009\u200a\u205f\u3000]/;
+  /[^\f\n\r\t ()\[\]{}"@~^;`\\,:'0-9\u000B\u001C\u001D\u001E\u001F\u2028\u2029\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2008\u2009\u200a\u205f\u3000]/;
 
 const SYMBOL_BODY =
   choice(SYMBOL_HEAD,
-    /[:#'0-9]/);
+    /[:#~'0-9\/]/);
 
 const SYMBOL =
   token(seq(SYMBOL_HEAD,
     repeat(SYMBOL_BODY)));
 
+// Define a rule specifically for symbols starting with digits followed by non-digit characters
+const DIGIT_SYMBOL = token(seq(repeat1(DIGIT), SYMBOL_BODY, repeat(SYMBOL_BODY)));
+
 module.exports = grammar({
   name: 'opengoal',
 
@@ -205,7 +208,7 @@ module.exports = grammar({
       seq(field('numberOfArgs', $._format_token), '*'),
       '?',
       "Newline",
-      seq(repeat(choice($._format_token, ',')), /[$mrRbBdDgGxXeEoOsStTfHhJjKkLlNnVwWyYzZ]/),
+      seq(repeat(choice($._format_token, ',')), /[$AmMrRbBdDgGxXeEoOsStTfFHhJjKkLlNnVwWyYzZ]/),
     ),
     format_specifier: $ =>
       prec.left(seq(
@@ -237,7 +240,9 @@ module.exports = grammar({
       BOOLEAN,
 
     sym_lit: $ =>
-      seq(choice($._sym_unqualified)),
+      seq(choice($._sym_unqualified, $._digit_sym)),
+
+    _digit_sym: $ => field('name', alias(DIGIT_SYMBOL, $.sym_name)),
 
     _sym_unqualified: $ =>
       field('name', alias(choice("/", SYMBOL),

package.json

@@ -6,7 +6,7 @@
   "types": "bindings/node",
   "scripts": {
     "gen": "yarn tree-sitter generate",
-    "test": "yarn tree-sitter test",
+    "test": "yarn tree-sitter generate && yarn tree-sitter test",
     "parse-sample": "yarn gen && yarn tree-sitter parse test/samples/sample.gc",
     "parse-test": "yarn gen && yarn tree-sitter parse test/samples/test.gc",
     "install": "node-gyp-build",