[Rust] Refactor path matching (#3854)

* [Rust] Scope accessor punctuation in attributes

* [Rust] Scope accessor punctuation in nested attribute call

* [Rust] Introduce `declarations` scope

* [Rust] Restructure definition file

Move some context to related ones and add section headers.

This commit does not claim to be an accurate grouping, but some grouping
for now is better than none.

Additionally, the `statements-block` context was replaced with a `block`
context that pops after matching, following the plural vs singular
naming convention.

* [Rust] Rewrite identifier matching

Rewrite type and identifier matching to fix problems with properly
highlighting function calls. Not all cases are implemented yet, but
this is a significant improvement over the previous situation.

- The generics meta scope doesn't cover the preceding identifier
  anymore.
- `meta.path` now spans the entire path, including the last segment.
- The explicit `types` context is used in a couple more newly introduced
  contexts, i.e. for variable type declarations and `type` variables.

A test file concerning identifier paths and function calls has been
added.

Some plural-vs-singular renaming has also been done, but not to all
contexts.

* [Rust] Add test for `union` as an identifier

* [Rust] Optimize some contexts according to review

Rust/tests/syntax_test_attributes.rs

@@ -144,4 +144,16 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Box<T> { }
 #[rustfmt::skip]
 //^^^^^^^^^^^^^^ meta.annotation
 //^^^^^^^^^^^^^ meta.path
+//       ^^ punctuation.accessor
 //         ^^^^ variable.annotation
+
+
+#[rustfmt::skip(rust::fmt::skip())]
+//             ^^^^^^^^^^^^^^^^^^^ meta.annotation meta.annotation.parameters meta.group
+//              ^^^^^^^^^^^^^^^ meta.path
+//                  ^^ punctuation.accessor
+//                       ^^ punctuation.accessor
+//                         ^^^^^^ meta.function-call
+//                         ^^^^ variable.function
+//                             ^ meta.group meta.group punctuation.section.group.begin
+//                              ^ meta.group meta.group punctuation.section.group.end

Rust/tests/syntax_test_closures.rs

@@ -163,7 +163,7 @@ let x = |a: i32, b: Foo, c: Option<i32>, d: extern "C" fn (), e: *const u8| {};
 //          ^^^ storage.type
 //               ^ variable.parameter
 //                       ^ variable.parameter
-//                          ^^^^^^ meta.generic support.type
+//                          ^^^^^^ support.type
 //                                 ^^^ meta.generic storage.type
 //                                       ^ variable.parameter
 //                                          ^^^^^^ keyword.other

Rust/tests/syntax_test_dyn.rs

@@ -33,13 +33,15 @@ type A1 = dyn::dyn;
 //        ^^^^^ meta.path -storage.type.trait
 //             ^^^ -storage.type.trait
 type A2 = dyn<dyn, dyn>;
-//        ^^^ meta.generic -storage.type.trait
+//        ^^^ -storage.type.trait
 //            ^^^ meta.generic -storage.type.trait
 //                 ^^^ meta.generic -storage.type.trait
 // This is incorrect.  `identifier` should not match on the keyword `as`.
 // However, avoiding keywords is a little complicated and slow.
 type A3 = dyn<<dyn as dyn>::dyn>;
-//        ^^^ meta.generic -storage.type.trait
-//             ^^^ meta.generic storage.type.trait
-//                    ^^^ meta.generic -storage.type.trait
-//                          ^^^ meta.generic -storage.type.trait
+//        ^^^ -storage.type.trait
+//           ^^^^^^^^^^^^^^^^^^^ meta.generic
+//            ^^^^^^^^^^^^ meta.generic meta.generic
+//             ^^^ storage.type.trait
+//                    ^^^ -storage.type.trait
+//                          ^^^ -storage.type.trait

Rust/tests/syntax_test_enum.rs

@@ -7,7 +7,7 @@ enum OperatingSystem
 {
 // <- meta.block meta.enum punctuation.section.block.begin
     Osx,
-//  ^^^ meta.enum storage.type.source
+//  ^^^ meta.enum storage.type.rust
     Windows,
     Linux,
     Bsd(String),
@@ -21,16 +21,21 @@ enum OperatingSystem
 // <- meta.block meta.enum punctuation.section.block.end
 
 let q = Message::Quit;
-//      ^^^^^^^^^ meta.path
-//      ^^^^^^^ storage.type.source
-//               ^^^^ storage.type.source
+//      ^^^^^^^^^^^^^ meta.path
+//      ^^^^^^^ storage.type.rust
+//             ^^ punctuation.accessor
+//               ^^^^ storage.type.rust
 //                   ^ punctuation.terminator
 let w = Message::WriteString("Some string".to_string());
-//               ^^^^^^^^^^^ storage.type.source
+//      ^^^^^^^^^^^^^^^^^^^^ meta.path
+//             ^^ punctuation.accessor
+//               ^^^^^^^^^^^ storage.type.rust
 //                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.group
 //                           ^^^^^^^^^^^^^ string.quoted.double
 //                                         ^^^^^^^^^ variable.function
 let m = Message::Move { x: 50, y: 200 };
+//      ^^^^^^^^^^^^^ meta.path
+//             ^^ punctuation.accessor
 //                    ^^^^^^^^^^^^^^^^^ meta.block
 //                         ^^ constant.numeric.integer.decimal
 //                                ^^^ constant.numeric.integer.decimal
@@ -45,7 +50,7 @@ enum Discriminant {
     V2,
 //  ^^ meta.enum entity.name.constant
     SomeValue = 123,
-//  ^^^^^^^^^ meta.enum storage.type.source
+//  ^^^^^^^^^ meta.enum storage.type.rust
 //              ^^^ meta.enum constant.numeric.integer.decimal
     V3 = (1<<4),
 //  ^^ meta.enum entity.name.constant

Rust/tests/syntax_test_functions.rs

@@ -153,12 +153,3 @@ fn sum((x, y): (i32, i32)) -> i32 {
 //                   ^^^ storage.type
 //                      ^ punctuation.section.group.end
 }
-
-input.parse::<SnailNum>()
-//           ^^^^^^^^^^ meta.generic.rust
-//           ^ punctuation.definition.generic.begin.rust
-//            ^^^^^^^^ storage.type.rust
-//                    ^ punctuation.definition.generic.end.rust
-//                     ^^ meta.group.rust
-//                     ^ punctuation.section.group.begin.rust
-//                      ^ punctuation.section.group.end.rust

Rust/tests/syntax_test_generics.rs

@@ -213,7 +213,10 @@ impl<T> From<AsRef<T>> for CliError<T> { }
 //                                 ^^^ meta.generic
 
 fn legal_dates_iter() -> Box<Iterator<Item = Date<UTC>>> {
-//                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.function.return-type meta.generic
+//                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.function.return-type
+//                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.generic
+//                                   ^^^^^^^^^^^^^^^^^^ meta.generic meta.generic
+//                                               ^^^^^ meta.generic meta.generic meta.generic
 //                                         ^ keyword.operator
     unimplemented!()
 }
@@ -229,7 +232,7 @@ fn numbers() -> impl Iterator<Item = u64> {
 
 fn collect_vec() {
     let _: Vec<(usize, usize)> = (0..10).enumerate().collect::<Vec<_>>();
-//         ^^^^^^^^^^^^^^^^^^^ meta.generic
+//            ^^^^^^^^^^^^^^^^ meta.generic
 //         ^^^ support.type
 //             ^ punctuation.section.group.begin
 //              ^^^^^ storage.type
@@ -248,7 +251,7 @@ fn collect_vec() {
 //                                                          ^^ punctuation.accessor
 //                                                             ^^^ support.type
 //                                                            ^^^^^^^^ meta.generic
-//                                                             ^^^^^^ meta.generic meta.generic
+//                                                                ^^^ meta.generic meta.generic
 //                                                                 ^ storage.type.inference.rust
     let _: Vec<(usize, usize)> = vec!();
 //                               ^^^^ support.macro

Rust/tests/syntax_test_identifiers.rs

@@ -0,0 +1,79 @@
+// SYNTAX TEST "Packages/Rust/Rust.sublime-syntax"
+
+let line = Cow::from(x);
+//         ^^^^^^^^^ meta.path
+//         ^^^ storage.type
+//            ^^ punctuation.accessor
+//              ^^^^ variable.function
+//                  ^^^ meta.group
+//                  ^ punctuation.section.group.begin
+
+
+buffer.rope.char();
+//^^^^^^^^^^^^^^ meta.path
+//    ^ punctuation.accessor.dot
+//         ^ punctuation.accessor.dot
+//          ^^^^ variable.function
+//              ^^ meta.group
+//                ^ punctuation.terminator
+
+
+buffer . rope . char ();
+//^^^^^^^^^^^^^^^^^^ meta.path
+//     ^ punctuation.accessor.dot
+//            ^ punctuation.accessor.dot
+//              ^^^^ variable.function
+//                   ^^ meta.group
+//                     ^ punctuation.terminator
+
+path . with_a_break s.path ();
+//^^^^^^^^^^^^^^^^^ meta.path
+//                 ^ -meta.path
+//                  ^^^^^^ meta.path
+
+env::current_dir()
+//^^^^^^^^^^^^^^ meta.path
+// ^^ punctuation.accessor
+//   ^^^^^^^^^^^ variable.function
+
+let file_bytes = fs::read(&path_buf)?;
+//               ^^^^^^^^ meta.path
+//                 ^^ punctuation.accessor
+//                   ^^^^ variable.function
+//                       ^^^^^^^^^^^ meta.group
+//                                  ^ keyword.operator
+//                                   ^ punctuation.terminator
+
+u8::try_from(), f64 :: from ()
+// <- meta.path storage.type
+ // <- meta.path storage.type
+//^^^^^^^^^^ meta.path
+//  ^^^^^^^^ variable.function
+//          ^^^^ -meta.path
+//            ^ punctuation.separator
+//              ^^^^^^^^^^^ meta.path
+//              ^^^ storage.type
+//                  ^^ punctuation.accessor
+//                     ^^^^ variable.function
+
+Vec::with_capacity()
+//^^^^^^^^^^^^^^^^ meta.path
+//^ support.type
+// ^^ punctuation.accessor
+
+for x in a.union(&b) {
+    //   ^^^^^^^ meta.path
+    //    ^ punctuation.accessor.dot
+    //     ^^^^^ variable.function
+    println!("{}", x);
+}
+
+// The following should eventually recognize `parse` as the function name
+input.parse::<SnailNum>()
+//           ^^^^^^^^^^ meta.generic.rust
+//           ^ punctuation.definition.generic.begin.rust
+//            ^^^^^^^^ storage.type.rust
+//                    ^ punctuation.definition.generic.end.rust
+//                     ^^ meta.group.rust
+//                     ^ punctuation.section.group.begin.rust
+//                      ^ punctuation.section.group.end.rust

Rust/tests/syntax_test_macros.rs

@@ -209,7 +209,7 @@ macro_rules! forward_ref_binop [
 //                        ^ keyword.operator
 //                         ^^ storage.modifier.lifetime
 //                            ^^ variable.other
-//                                ^^^ keyword.other
+//                                ^^^ entity.name.impl
 //                                    ^ keyword.operator
 //                                     ^^ storage.modifier.lifetime
 //                                        ^^ variable.other
@@ -379,7 +379,6 @@ macro_rules! designators {
 //                                              ^^^ string.quoted.single
 //                                                  ^ keyword.operator
 //                                                    ^^^^^^ storage.type.struct
-//                                                           ^^^^ keyword.other
 //                                                               ^ punctuation.section.block.end
 //                                                                 ^^ meta.macro keyword.operator
 //                                                                    ^^ meta.macro meta.macro.transcribers

Rust/tests/syntax_test_modules.rs

@@ -41,25 +41,24 @@ pub use self::trafile::*;
 
 use std::fmt;
 // <- keyword.other
-//  ^^^^^ meta.path
+//  ^^^^^^^^ meta.path
 //     ^^ punctuation.accessor
-//       ^^^ - meta.path
 //          ^ punctuation.terminator
 use foo::i32;
 //  ^^^^^ meta.path
 //     ^^ punctuation.accessor
-//       ^^^ - meta.path storage.type
+//       ^^^ - storage.type
 use foo::Bar;
 //  ^^^^^ meta.path
 //     ^^ punctuation.accessor
-//       ^^^ storage.type.source
+//       ^^^ storage.type.rust
 
 use foo::{Baz, QUX, quux};
 //  ^^^^^ meta.path
 //     ^^ punctuation.accessor.rust
 //       ^^^^^^^^^^^^^^^^ meta.block
 //       ^ punctuation.section.block.begin
-//        ^^^ storage.type.source
+//        ^^^ storage.type.rust
 //             ^^^ constant.other
 //                  ^^^^ meta.block
 //                      ^ punctuation.section.block.end
@@ -78,8 +77,8 @@ use std::{
     path::{Path, PathBuf},
 //  ^^^^^^ meta.block meta.path
 //        ^ meta.block meta.block punctuation.section.block.begin
-//         ^^^^ meta.block meta.block storage.type.source
-//               ^^^^^^^ meta.block meta.block storage.type.source
+//         ^^^^ meta.block meta.block storage.type.rust
+//               ^^^^^^^ meta.block meta.block storage.type.rust
 //                      ^ meta.block meta.block punctuation.section.block.end
    };
 // ^ meta.block punctuation.section.block.end

Rust/tests/syntax_test_raw.rs

@@ -14,7 +14,7 @@ fn main() {
 
     let r#local = r#Struct { r#struct: () };
 //      ^^^^^^^^ -keyword
-//                ^^^^^^^^ storage.type.source -keyword
+//                ^^^^^^^^ storage.type.rust -keyword
 //                           ^^^^^^^^ -keyword
     let r#async = r#foo(r#local);
 //      ^^^^^^^^ -keyword
@@ -27,13 +27,13 @@ fn main() {
 
     if let r#sub_pat @ Foo(_) = r#Foo(3) {}
 //         ^^^^^^^^^^ -keyword
-//                              ^^^^^ storage.type.source
+//                              ^^^^^ storage.type.rust
 
     match r#async {
 //        ^^^^^^^^ -keyword
         r#Foo | r#Bar => r#foo(),
-//      ^^^^^ storage.type.source -keyword
-//              ^^^^^ storage.type.source -keyword
+//      ^^^^^ storage.type.rust -keyword
+//              ^^^^^ storage.type.rust -keyword
 //                       ^^^^^ -keyword
     }
 }
@@ -77,11 +77,11 @@ impl r#Trait for r#Impl {
 //               ^^^^^^ meta.impl entity.name.impl -keyword
     type r#Type = r#u32;
 //       ^^^^^^ meta.impl meta.block entity.name.type
-//                ^^^^^ meta.impl meta.block -keyword -storage
+//                ^^^^^ meta.impl meta.block storage.type
     fn r#xxx(r#fjio: r#u32) {}
 //     ^^^^^ meta.impl meta.block meta.function entity.name.function -keyword
 //           ^^^^^^ meta.impl meta.block meta.function meta.function.parameters variable.parameter -keyword
-//                   ^^^^^ meta.impl meta.block meta.function meta.function.parameters -keyword -storage
+//                   ^^^^^ meta.impl meta.block meta.function meta.function.parameters storage.type
 }
 
 extern "C" {

Rust/tests/syntax_test_struct.rs

@@ -15,8 +15,8 @@ struct PrintableStruct(Box<i32>);
 //^^^^ storage.type.struct
 //     ^^^^^^^^^^^^^^^ entity.name.struct
 //                    ^ punctuation.section.group.begin
-//                     ^^^^^^^^ meta.generic
 //                     ^^^ support.type
+//                        ^^^^^ meta.generic
 //                        ^ punctuation.definition.generic.begin
 //                         ^^^ storage.type
 //                            ^ punctuation.definition.generic.end
@@ -107,18 +107,18 @@ let mut j = BasicStruct(10);
 //                      ^^ constant.numeric.integer.decimal
 
 let p = Point {x: 10.0, y: 20.0};
-//      ^^^^^ storage.type.source
+//      ^^^^^ storage.type.rust
 //            ^^^^^^^^^^^^^^^^^^ meta.block
 //            ^ punctuation.section.block.begin
 //              ^ punctuation.separator
 //                ^^^^ constant.numeric.float
 //                             ^ punctuation.section.block.end
 let n = NothingInMe {};
-//      ^^^^^^^^^^^ storage.type.source
+//      ^^^^^^^^^^^ storage.type.rust
 //                  ^^ meta.block
 let tp = TuplePoint { 0: 10.0, 1: 20.0 };
 //                    ^constant.numeric.integer.decimal
 //                             ^ constant.numeric.integer.decimal
 let p = Point { x, y };
-//      ^^^^^ storage.type.source
+//      ^^^^^ storage.type.rust
 //            ^^^^^^^^ meta.block

Rust/tests/syntax_test_traits.rs

@@ -85,7 +85,7 @@ impl<T: ?Sized> !marker::Sync for Rc<T> {}
 //       ^^^^^ support.type.rust
 //            ^ punctuation.definition.generic.end.rust
 //              ^ meta.impl.rust keyword.operator.rust meta.impl.opt-out.rust
-//               ^^^^^^^^ meta.impl.rust meta.path.rust
+//               ^^^^^^^^^^^^ meta.impl.rust meta.path.rust
 //                       ^^^^ meta.impl.rust support.type.rust
 //                            ^^^ meta.impl.rust keyword.other.rust
 //                                ^^ meta.impl.rust meta.generic.rust entity.name.impl.rust
@@ -139,3 +139,8 @@ trait Bar: for<'a> Foo<&'a ()> {}
 //             ^^ meta.where meta.generic storage.modifier.lifetime
 //                     ^ meta.where meta.generic keyword.operator
 //                      ^^ meta.where meta.generic storage.modifier.lifetime
+
+trait t {}
+//    ^ meta.trait entity.name.trait
+
+// <- - meta