feat: reintroduce raw-strings (#117)

[Discussion with tree-sitter
devs](tree-sitter/tree-sitter#3878 (comment))

This will break the current `nvim-treesitter` config. They have to
update to also use `scanner.c`. People using that config might want to
use "our" way to install ts-nu.

After merge, it might be necessary to run `:TSUpdate nu` at least once.
I don't know about treesitter's caching, so let's just hope that
nobody's setup breaks.

With that scanner, we can also create custom logic for unquoted strings.

.github/workflows/main.yml

@@ -5,7 +5,7 @@ on:
       - main
 jobs:
   main:
-    name: install
+    name: setup, lint, test
 
     runs-on: ${{ matrix.os }}
     strategy:

Package.swift

@@ -35,7 +35,7 @@ let package = Package(
                 ],
                 sources: [
                     "src/parser.c",
-                    // NOTE: if your language has an external scanner, add it here.
+                    "src/scanner.c",
                 ],
                 resources: [
                     .copy("queries")

binding.gyp

@@ -11,7 +11,7 @@
       "sources": [
         "bindings/node/binding.cc",
         "src/parser.c",
-        # NOTE: if your language has an external scanner, add it here.
+        "src/scanner.c",
       ],
       "conditions": [
         ["OS!='win'", {

bindings/go/binding.go

@@ -2,7 +2,7 @@ package tree_sitter_nu
 
 // #cgo CFLAGS: -std=c11 -fPIC
 // #include "../../src/parser.c"
-// // NOTE: if your language has an external scanner, add it here.
+// #include "../../src/scanner.c"
 import "C"
 
 import "unsafe"

bindings/rust/build.rs

@@ -16,11 +16,9 @@ fn main() {
     // If your language uses an external scanner written in C,
     // then include this block of code:
 
-    /*
     let scanner_path = src_dir.join("scanner.c");
     c_config.file(&scanner_path);
     println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
 
     c_config.compile("parser");
     println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());

grammar.js

@@ -19,8 +19,11 @@ module.exports = grammar({
     $._terminator,
   ],
 
-  // externals: $ => [
-  // ],
+  externals: ($) => [
+    $.raw_string_begin,
+    $.raw_string_content,
+    $.raw_string_end,
+  ],
 
   conflicts: ($) => [
     [$._binary_predicate_parenthesized],
@@ -981,7 +984,15 @@ module.exports = grammar({
       ),
 
     val_string: ($) =>
-      choice($._str_double_quotes, $._str_single_quotes, $._str_back_ticks),
+      choice(
+        $._str_double_quotes,
+        $._str_single_quotes,
+        $._str_back_ticks,
+        $._raw_str,
+      ),
+
+    _raw_str: ($) =>
+      seq($.raw_string_begin, $.raw_string_content, $.raw_string_end),
 
     _str_double_quotes: ($) =>
       seq(

installation/neovim.md

@@ -34,7 +34,7 @@ local parser_config = require("nvim-treesitter.parsers").get_parser_configs()
 parser_config.nu = {
   install_info = {
     url = "https://github.com/nushell/tree-sitter-nu",
-    files = { "src/parser.c" },
+    files = { "src/parser.c", "src/scanner.c" },
     branch = "main",
   },
   filetype = "nu",
@@ -68,6 +68,8 @@ mkdir $local
 http get ([$remote $file] | str join "/") | save --force ($local | path join $file)
 ```
 
+You need to run this snippet whenever the highlights change and `:TSUpdate nu` whenever there is a new version of the parser.
+
 [tree-sitter]: https://tree-sitter.github.io/tree-sitter/
 [nvim-treesitter]: https://github.com/nvim-treesitter/nvim-treesitter
 [nvim-treesitter/playground]: https://github.com/nvim-treesitter/playground

package.json

@@ -13,21 +13,21 @@
   "author": "The Nushell Contributors",
   "license": "MIT",
   "dependencies": {
-    "node-addon-api": "^8.0.0",
-    "node-gyp-build": "^4.8.0"
+    "node-addon-api": "^8.1.0",
+    "node-gyp-build": "^4.8.2"
   },
   "peerDependencies": {
-    "tree-sitter": "^0.21.0"
+    "tree-sitter": "^0.21.1"
   },
   "peerDependenciesMeta": {
     "tree_sitter": {
       "optional": true
     }
   },
   "devDependencies": {
-    "prettier": "3.2.5",
-    "tree-sitter-cli": "^0.22.5",
-    "prebuildify": "^6.0.0"
+    "prebuildify": "^6.0.0",
+    "prettier": "3.3.3",
+    "tree-sitter-cli": "^0.23.0"
   },
   "files": [
     "grammar.js",

plugin/init.lua

@@ -1,15 +1,17 @@
 vim.filetype.add({ extension = { nu = "nu" }, filename = { ["nurfile"] = "nu" } })
 
 vim.api.nvim_create_autocmd("FileType", {
-  pattern = "nu",
-  callback = function(event) vim.bo[event.buf].commentstring = "# %s" end,
+	pattern = "nu",
+	callback = function(event)
+		vim.bo[event.buf].commentstring = "# %s"
+	end,
 })
 
 require("nvim-treesitter.parsers").get_parser_configs().nu = {
-  install_info = {
-    url = "https://github.com/nushell/tree-sitter-nu",
-    files = { "src/parser.c" },
-    branch = "main",
-  },
-  filetype = "nu",
+	install_info = {
+		url = "https://github.com/nushell/tree-sitter-nu",
+		files = { "src/parser.c", "src/scanner.c" },
+		branch = "main",
+	},
+	filetype = "nu",
 }

setup.py

@@ -36,7 +36,7 @@ def get_tag(self):
             sources=[
                 "bindings/python/tree_sitter_nu/binding.c",
                 "src/parser.c",
-                # NOTE: if your language uses an external scanner, add it here.
+                "src/scanner.c",
             ],
             extra_compile_args=[
                 "-std=c11",
@@ -45,8 +45,9 @@ def get_tag(self):
                 "/utf-8",
             ],
             define_macros=[
-                ("Py_LIMITED_API", "0x03080000"),
-                ("PY_SSIZE_T_CLEAN", None)
+                ("Py_LIMITED_API", "0x03090000"),
+                ("PY_SSIZE_T_CLEAN", None),
+                ("TREE_SITTER_HIDE_SYMBOLS", None),
             ],
             include_dirs=["src"],
             py_limited_api=True,

src/grammar.json

@@ -14886,6 +14886,27 @@
         {
           "type": "SYMBOL",
           "name": "_str_back_ticks"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "_raw_str"
+        }
+      ]
+    },
+    "_raw_str": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "raw_string_begin"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "raw_string_content"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "raw_string_end"
         }
       ]
     },
@@ -17883,7 +17904,20 @@
     ]
   ],
   "precedences": [],
-  "externals": [],
+  "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "raw_string_begin"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "raw_string_content"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "raw_string_end"
+    }
+  ],
   "inline": [
     "_do_expression",
     "_flag_value",