[JavaScript] Support explicit resource management proposal (and rewri…

…te for loops in the process) (#3836)

[This feature](https://github.com/tc39/proposal-explicit-resource-management) is currently in Stage 3 of the TC39 process. In addition, it is part of [TypeScript 5.2](https://devblogs.microsoft.com/typescript/announcing-typescript-5-2/#using-declarations-and-explicit-resource-management), which was released last week.

The lookaheads are exact (unless I made a mistake). Since `using` is not a reserved word, the proposal spec has to restrict line endings between `using` and the variable name to ensure backward-compatibility.

As part of this PR, I rewrote `for` loop handling. JavaScript basically has two different kinds of for loops: C-style (e.g. `for (let i = 0; i < len; i++) {}`) and `in`/`of` (e.g. `for (const item of list) {}`). The existing implementation predates branching, so it tries to handle both cases simultaneously. This works surprisingly well in practice, but `using` would have made a mess of it. This PR uses branching to try to parse the loop condition as in/of, and falls back to C-style if that fails. It's more verbose, but it should be less confusing, and as a bonus it correctly handles the case where the binding of an `in`/`of` loop is an arbitrary left expression (why!?).

The only known bug is that `for (using of in obj) {}` is highlighted slightly wrong. Basically the only relevant syntactic difference between `in` and `of` loops is a dumb special case related to this. I could fix it, but that would likely require splitting the handling of `in` and `of` loops, which I think is not worth the tiny benefit.

The PR also incidentally removes a couple of unwanted `.js` suffixes in the tests.

JavaScript/JavaScript.sublime-syntax

@@ -447,6 +447,7 @@ contexts:
 
   declaration:
     - include: variable-declaration
+    - include: using-declaration
     - include: class
     - include: regular-function
 
@@ -568,6 +569,32 @@ contexts:
         - variable-binding-list-top
         - variable-binding-top
 
+  using-declaration:
+    - include: plain-using-declaration
+
+    - match: (?=await{{identifier_break}})
+      branch_point: await-using-declaration
+      branch:
+        - await-using-declaration
+        - expression-statement
+      pop: 1
+
+  plain-using-declaration:
+    - match: using{{identifier_break}}(?={{nothing}}{{non_reserved_identifier}})
+      scope: keyword.declaration.js
+      set:
+        - expect-semicolon
+        - variable-binding-list-top
+        - variable-binding-top
+
+  await-using-declaration:
+    - match: await{{identifier_break}}
+      scope: keyword.declaration.js
+      set:
+        - include: using-declaration
+        - match: (?=\S)
+          fail: await-using-declaration
+
   function-parameter-binding-pattern:
     - include: function-parameter-binding-name
     - include: function-parameter-binding-array-destructuring
@@ -849,29 +876,77 @@ contexts:
       pop: 1
 
   for-condition-contents:
-    # This could be either type of for loop.
+    - match: ''
+      branch_point: for-in-of
+      branch:
+        - for-in-of
+        - for-oldstyle
+      pop: 1
+
+  for-in-of:
+    - match: ''
+      set:
+        - expression
+        - for-in-of-word
+        - for-in-of-declaration
+
+  for-in-of-declaration:
     - match: (?:const|let|var){{identifier_break}}
       scope: keyword.declaration.js
+      set: for-in-of-binding
+
+    - match: |-
+        (?x:
+          (?! # [lookahead ≠ using of]
+            using{{identifier_break}}{{nothing}}
+            of{{identifier_break}}
+          )
+          (?= # await [no LineTerminator here] using [no LineTerminator here] ForBinding
+            (?:await{{identifier_break}}{{nothing}})?
+            using{{identifier_break}}{{nothing}}
+            {{non_reserved_identifier}}
+          )
+        )
+      scope: keyword.declaration.js
       set:
-        - - include: for-of-rest
-          - match: (?=\S)
-            set:
-              - for-oldstyle-rest
-              - variable-binding-list
-              - initializer
-        - variable-binding-pattern
+        - match: await{{identifier_break}}
+          scope: keyword.declaration.js
+        - match: using{{identifier_break}}
+          scope: keyword.declaration.js
+          set: for-in-of-binding
 
     - match: (?=\S)
       set:
-        - - include: for-of-rest
-          - match: (?=\S)
-            set: for-oldstyle-rest
-        - expression-end-no-in
+        - left-expression-end
         - expression-begin
 
-  for-of-rest:
+  for-in-of-binding:
+    - include: variable-binding-name
+    - include: variable-binding-array-destructuring
+    - include: variable-binding-object-destructuring
+    - match: (?=\S)
+      fail: for-in-of
+
+  for-in-of-word:
     - match: (?:of|in){{identifier_break}}
       scope: keyword.operator.word.js
+      pop: 1
+    - match: (?=\S)
+      fail: for-in-of
+
+  for-oldstyle:
+    - match: ''
+      set:
+        - for-oldstyle-rest
+        - for-oldstyle-first
+
+  for-oldstyle-first:
+    - match: (?:const|let|var){{identifier_break}}
+      scope: keyword.declaration.js
+      set:
+        - variable-binding-list-top
+        - variable-binding-top
+    - match: (?=\S)
       set: expression
 
   for-oldstyle-rest:
@@ -2398,7 +2473,7 @@ contexts:
       pop: 1
 
   support-property-ecma-symbol:
-    - match: (?:asyncIterator|hasInstance|isConcatSpreadable|iterator|match|replace|search|species|split|toPrimitive|toStringTag|unscopeables){{identifier_break}}
+    - match: (?:asyncDispose|asyncIterator|dispose|hasInstance|isConcatSpreadable|iterator|match|replace|search|species|split|toPrimitive|toStringTag|unscopeables){{identifier_break}}
       scope: support.constant.builtin.js
       pop: 1
     - match: (?:for|keyFor){{identifier_break}}

JavaScript/tests/syntax_test_js_bindings.js

@@ -26,7 +26,7 @@ const [ x, [a, b], z] = value;
 const [ x = 42, y = [a, b, c] ] = value;
 //    ^^^^^^^^^^^^^^^^^^^^^^^^^ meta.binding.destructuring.sequence
 //        ^ keyword.operator.assignment
-//          ^^ meta.binding.destructuring.sequence.js meta.number.integer.decimal.js constant.numeric.value.js
+//          ^^ meta.binding.destructuring.sequence meta.number.integer.decimal constant.numeric.value
 //                ^ keyword.operator.assignment
 //                  ^^^^^^^^^ meta.sequence
 //                   ^ variable.other.readwrite - meta.binding.name
@@ -184,7 +184,7 @@ let f = ([ x = 42, y = [a, b, c] ]) => {};
 //      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.function
 //       ^^^^^^^^^^^^^^^^^^^^^^^^^ meta.binding.destructuring.sequence
 //           ^ keyword.operator.assignment
-//             ^^ meta.binding.destructuring.sequence.js meta.number.integer.decimal.js constant.numeric.value.js
+//             ^^ meta.binding.destructuring.sequence meta.number.integer.decimal constant.numeric.value
 //                   ^ keyword.operator.assignment
 //                     ^^^^^^^^^ meta.sequence
 //                      ^ variable.other.readwrite - meta.binding.name
@@ -223,3 +223,63 @@ let f = (a, ...rest) => {};
 let f = (new) => {};
 //      ^^^^^^^^^^^ meta.function
 //       ^^^ invalid.illegal.identifier meta.binding.name variable.parameter.function
+
+using x = 0;
+//^^^ keyword.declaration
+//    ^ meta.binding.name variable.other.readwrite
+//      ^ keyword.operator.assignment
+//        ^ meta.number.integer.decimal constant.numeric.value
+//         ^ punctuation.terminator.statement
+
+using [ x ] = 0;
+//^^^ variable.other.readwrite
+//    ^^^^^ meta.brackets
+//    ^ punctuation.section.brackets.begin
+//      ^ variable.other.readwrite - meta.binding
+//        ^ punctuation.section.brackets.end
+//          ^ keyword.operator.assignment
+//            ^ meta.number.integer.decimal constant.numeric.value
+//             ^ punctuation.terminator.statement
+
+using
+//^^^ variable.other.readwrite
+x = 0;
+// <- variable.other.readwrite - meta.binding
+//^ keyword.operator.assignment
+//  ^ meta.number.integer.decimal constant.numeric.value
+//   ^ punctuation.terminator.statement
+
+
+await using x = 0;
+//^^^ keyword.declaration
+//    ^^^^^ keyword.declaration
+//          ^ meta.binding.name variable.other.readwrite
+//            ^ keyword.operator.assignment
+//              ^ meta.number.integer.decimal constant.numeric.value
+//               ^ punctuation.terminator.statement
+
+await
+//^^^ keyword.declaration
+using x = 0;
+//^^^ keyword.declaration
+//    ^ meta.binding.name variable.other.readwrite
+//      ^ keyword.operator.assignment
+//        ^ meta.number.integer.decimal constant.numeric.value
+//         ^ punctuation.terminator.statement
+
+await using;
+//^^^ keyword.control.flow.await
+//    ^^^^^ variable.other.readwrite
+//         ^ punctuation.terminator.statement
+
+await
+//^^^ keyword.control.flow.await
+using [ x ] = 0;
+//^^^ variable.other.readwrite
+
+await using in x;
+//^^^ keyword.control.flow.await
+//    ^^^^^ variable.other.readwrite
+//          ^^ keyword.operator
+//             ^ variable.other.readwrite
+//              ^ punctuation.terminator.statement

JavaScript/tests/syntax_test_js_control.js

@@ -163,10 +163,119 @@
 //      ^^^^^^^^^^^ meta.group
 //         ^^ keyword.operator.word
 
+    for (x.y.z of list) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//       ^ variable.other.readwrite
+//        ^ punctuation.accessor
+//         ^ meta.property.object
+//          ^ punctuation.accessor
+//           ^ meta.property.object
+//             ^^ keyword.operator.word
+//                ^^^^ variable.other.readwrite
+//                    ^ punctuation.section.group.end
+//                      ^^ meta.block
+
     for await (const x of list) {}
 //  ^^^ keyword.control.loop.for
 //      ^^^^^ keyword.control.flow.await
 
+    for ( using x of list ) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ keyword.declaration
+//              ^ meta.binding.name variable.other.readwrite
+//                ^^ keyword.operator.word
+//                   ^^^^ variable.other.readwrite
+//                        ^ punctuation.section.group.end
+//                          ^^ meta.block
+
+    for ( await using x of list ) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ keyword.declaration
+//              ^^^^^ keyword.declaration
+//        ^^^^^ keyword.declaration
+//              ^^^^^ keyword.declaration
+//                    ^ meta.binding.name variable.other.readwrite
+//                      ^^ keyword.operator.word
+//                         ^^^^ variable.other.readwrite
+//                              ^ punctuation.section.group.end
+//                                ^^ meta.block
+
+    for ( await using of of list ) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ keyword.declaration
+//              ^^^^^ keyword.declaration
+//                    ^^ meta.binding.name variable.other.readwrite
+//                       ^^ keyword.operator.word
+//                          ^^^^ variable.other.readwrite
+//                               ^ punctuation.section.group.end
+//                                 ^^ meta.block
+//                                 ^ punctuation.section.block.begin
+//                                  ^ punctuation.section.block.end
+
+    for ( using of list ) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ variable.other.readwrite
+//              ^^ keyword.operator.word
+//                 ^^^^ variable.other.readwrite
+//                      ^ punctuation.section.group.end
+//                        ^^ meta.block
+
+    for ( using [x] of list ) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ variable.other.readwrite
+//              ^^^ meta.brackets
+//              ^ punctuation.section.brackets.begin
+//               ^ variable.other.readwrite
+//                ^ punctuation.section.brackets.end
+//                  ^^ keyword.operator.word
+//                     ^^^^ variable.other.readwrite
+//                          ^ punctuation.section.group.end
+//                            ^^ meta.block
+
+    for ( await using ;;) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ keyword.control.flow.await
+//              ^^^^^ variable.other.readwrite
+//                    ^^ punctuation.separator.expression
+//                      ^ punctuation.section.group.end
+//                        ^^ meta.block
+//                        ^ punctuation.section.block.begin
+//                         ^ punctuation.section.block.end
+
+    for ( await using instanceof x ;;) {}
+//  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.for
+//  ^^^ keyword.control.loop.for
+//      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.group
+//      ^ punctuation.section.group.begin
+//        ^^^^^ keyword.control.flow.await
+//              ^^^^^ variable.other.readwrite
+//                    ^^^^^^^^^^ keyword.operator
+//                               ^ variable.other.readwrite
+//                                 ^^ punctuation.separator.expression
+//                                   ^ punctuation.section.group.end
+//                                     ^^ meta.block
+
 for
     42;
 //  ^^ constant.numeric - meta.for
@@ -279,7 +388,7 @@ with (undefined) {
 //^^^^^^^^^^ meta.with
 //    ^^^^^^^^^ constant.language.undefined
     return;
-//  ^^^^^^ meta.with.js meta.block.js keyword.control.flow.return
+//  ^^^^^^ meta.with meta.block keyword.control.flow.return
 }
 
 with // Incomplete statement
@@ -290,7 +399,7 @@ with(false){}/**/
 //           ^ - meta.with
 
 switch ($foo) {
-// <- meta.switch.js keyword.control.conditional.switch
+// <- meta.switch keyword.control.conditional.switch
 // ^^^^^^^^^^^^ meta.switch
 //^^^^ keyword.control.conditional.switch
 //      ^^^^ meta.group

JavaScript/tests/syntax_test_js_support_builtin.js

@@ -401,8 +401,12 @@ String.fromCodePoint;
 String.raw;
 //     ^^^ support.function.builtin
 
+Symbol.asyncDispose;
+//     ^^^^^^^^^^^^ support.constant.builtin
 Symbol.asyncIterator;
 //     ^^^^^^^^^^^^^ support.constant.builtin
+Symbol.dispose;
+//     ^^^^^^^ support.constant.builtin
 Symbol.hasInstance;
 //     ^^^^^^^^^^^ support.constant.builtin
 Symbol.isConcatSpreadable;