Lex via Moo

package.json

@@ -5,6 +5,7 @@
     "packages/*"
   ],
   "devDependencies": {
+    "@types/moo": "^0.5",
     "@types/node": "^20",
     "@types/yargs": "^17",
     "@vscode/vsce": "^2",

packages/rose/package.json

@@ -19,6 +19,7 @@
     "src"
   ],
   "dependencies": {
+    "moo": "^0.5",
     "yargs": "^17"
   },
   "scripts": {

packages/rose/src/lex.test.ts

@@ -0,0 +1,46 @@
+import { expect, test } from "vitest";
+import { lexer } from "./lex.js";
+
+const tokens = (
+  source: string,
+): { type: string | undefined; text: string }[] => {
+  const lex = lexer();
+  lex.reset(source);
+  return [...lex].map(({ type, text }) => ({ type, text }));
+};
+
+test("comment", () => {
+  expect(
+    tokens(`# one comment
+
+# another comment`),
+  ).toEqual([
+    { type: "comment", text: "# one comment" },
+    { type: "space", text: "\n\n" },
+    { type: "comment", text: "# another comment" },
+  ]);
+});
+
+test("string", () => {
+  expect(tokens('"one string" "another string"')).toEqual([
+    { type: "str", text: '"one string"' },
+    { type: "space", text: " " },
+    { type: "str", text: '"another string"' },
+  ]);
+});
+
+test("integer", () => {
+  expect(tokens("42")).toEqual([{ type: "num", text: "42" }]);
+});
+
+test("float", () => {
+  expect(tokens("42.0")).toEqual([{ type: "num", text: "42.0" }]);
+});
+
+test("number followed by letter", () => {
+  expect(() => tokens("1a")).toThrow("invalid syntax");
+});
+
+test("letter followed by number", () => {
+  expect(tokens("a1")).toEqual([{ type: "id", text: "a1" }]);
+});

packages/rose/src/lex.ts

@@ -0,0 +1,19 @@
+import moo from "moo";
+
+export const lexer = () =>
+  moo.compile({
+    op: /[^\s#\w"\(\)\[\]\{\},\.]+/,
+    space: { match: /\s+/, lineBreaks: true },
+    comment: /#.*?$/,
+    str: { match: /".*?"/, lineBreaks: true },
+    num: /\d+(?:\.\d+)?\b/,
+    id: /(?!\d)\w+/,
+    lparen: "(",
+    rparen: ")",
+    lbracket: "[",
+    rbracket: "]",
+    lbrace: "{",
+    rbrace: "}",
+    comma: ",",
+    dot: ".",
+  });

packages/vscode/syntaxes/rose.tmLanguage.json

@@ -50,12 +50,13 @@
     "literals": {
       "patterns": [
         {
-          "name": "constant.numeric.rose",
-          "match": "\\b[0-9]+(\\.[0-9]+)?\\b"
+          "name": "string.quoted.double.rose",
+          "begin": "\"",
+          "end": "\""
         },
         {
-          "name": "string.quoted.double.rose",
-          "match": "\"([^\"\\\\]|\\\\.)*\""
+          "name": "constant.numeric.rose",
+          "match": "\\b\\d+(\\.\\d+)?\\b"
         },
         {
           "name": "constant.language.rose",
@@ -91,7 +92,7 @@
       "patterns": [
         {
           "name": "keyword.operator.rose",
-          "match": "[^\\s#\\w\\d\"\\(\\)\\[\\]\\{\\},\\.]+"
+          "match": "[^\\s#\\w\"\\(\\)\\[\\]\\{\\},\\.]+"
         }
       ]
     }