Implement macro meta-variable expressions

compiler/rustc_expand/src/mbe.rs

@@ -6,17 +6,17 @@
 crate mod macro_check;
 crate mod macro_parser;
 crate mod macro_rules;
+crate mod metavar_expr;
 crate mod quoted;
 crate mod transcribe;
 
+use metavar_expr::MetaVarExpr;
 use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
 use rustc_ast::tokenstream::DelimSpan;
-
+use rustc_data_structures::sync::Lrc;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
-use rustc_data_structures::sync::Lrc;
-
 /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
 /// that the delimiter itself might be `NoDelim`.
 #[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
@@ -73,8 +73,8 @@ enum KleeneOp {
     ZeroOrOne,
 }
 
-/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
-/// are "first-class" token trees. Useful for parsing macros.
+/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, `$(...)`,
+/// and `${...}` are "first-class" token trees. Useful for parsing macros.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
 enum TokenTree {
     Token(Token),
@@ -85,6 +85,8 @@ enum TokenTree {
     MetaVar(Span, Ident),
     /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros.
     MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
+    /// A meta-variable expression inside `${...}`
+    MetaVarExpr(DelimSpan, MetaVarExpr),
 }
 
 impl TokenTree {
@@ -139,7 +141,9 @@ impl TokenTree {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
             | TokenTree::MetaVarDecl(span, _, _) => span,
-            TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(),
+            TokenTree::Delimited(span, _)
+            | TokenTree::MetaVarExpr(span, _)
+            | TokenTree::Sequence(span, _) => span.entire(),
         }
     }
 

compiler/rustc_expand/src/mbe/macro_check.rs

@@ -278,6 +278,9 @@ fn check_binders(
                 binders.insert(name, BinderInfo { span, ops: ops.into() });
             }
         }
+        // `MetaVarExpr` can not appear in the LHS of a macro arm, not even in a nested
+        // macro definition.
+        TokenTree::MetaVarExpr(..) => {}
         TokenTree::Delimited(_, ref del) => {
             for tt in &del.tts {
                 check_binders(sess, node_id, tt, macros, binders, ops, valid);
@@ -335,6 +338,7 @@ fn check_occurrences(
             let name = MacroRulesNormalizedIdent::new(name);
             check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name);
         }
+        TokenTree::MetaVarExpr(..) => {}
         TokenTree::Delimited(_, ref del) => {
             check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
         }

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -280,10 +280,11 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
     ms.iter().fold(0, |count, elt| {
         count
             + match *elt {
-                TokenTree::Sequence(_, ref seq) => seq.num_captures,
                 TokenTree::Delimited(_, ref delim) => count_names(&delim.tts),
                 TokenTree::MetaVar(..) => 0,
                 TokenTree::MetaVarDecl(..) => 1,
+                TokenTree::MetaVarExpr(..) => 0,
+                TokenTree::Sequence(_, ref seq) => seq.num_captures,
                 TokenTree::Token(..) => 0,
             }
     })
@@ -392,7 +393,8 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
                 }
                 Occupied(..) => return Err((sp, format!("duplicated bind name: {}", bind_name))),
             },
-            TokenTree::MetaVar(..) | TokenTree::Token(..) => (),
+            // FIXME(c410-f3r) MetaVar and MetaVarExpr should be handled instead of being ignored.
+            TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) | TokenTree::Token(..) => {}
         }
 
         Ok(())
@@ -603,7 +605,7 @@ fn inner_parse_loop<'root, 'tt>(
                 // rules. NOTE that this is not necessarily an error unless _all_ items in
                 // `cur_items` end up doing this. There may still be some other matchers that do
                 // end up working out.
-                TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
+                TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => {}
             }
         }
     }

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -584,7 +584,10 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
     use mbe::TokenTree;
     for tt in tts {
         match *tt {
-            TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (),
+            TokenTree::Token(..)
+            | TokenTree::MetaVar(..)
+            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarExpr(..) => (),
             TokenTree::Delimited(_, ref del) => {
                 if !check_lhs_no_empty_seq(sess, &del.tts) {
                     return false;
@@ -673,7 +676,10 @@ impl FirstSets {
             let mut first = TokenSet::empty();
             for tt in tts.iter().rev() {
                 match *tt {
-                    TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
+                    TokenTree::Token(..)
+                    | TokenTree::MetaVar(..)
+                    | TokenTree::MetaVarDecl(..)
+                    | TokenTree::MetaVarExpr(..) => {
                         first.replace_with(tt.clone());
                     }
                     TokenTree::Delimited(span, ref delimited) => {
@@ -735,7 +741,10 @@ impl FirstSets {
         for tt in tts.iter() {
             assert!(first.maybe_empty);
             match *tt {
-                TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
+                TokenTree::Token(..)
+                | TokenTree::MetaVar(..)
+                | TokenTree::MetaVarDecl(..)
+                | TokenTree::MetaVarExpr(..) => {
                     first.add_one(tt.clone());
                     return first;
                 }
@@ -911,7 +920,10 @@ fn check_matcher_core(
         // First, update `last` so that it corresponds to the set
         // of NT tokens that might end the sequence `... token`.
         match *token {
-            TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
+            TokenTree::Token(..)
+            | TokenTree::MetaVar(..)
+            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarExpr(..) => {
                 if token_can_be_followed_by_any(token) {
                     // don't need to track tokens that work with any,
                     last.replace_with_irrelevant();

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -0,0 +1,180 @@
+use std::fmt::Display;
+use std::str::FromStr;
+
+use rustc_ast::token::{self, Lit};
+use rustc_ast::tokenstream;
+use rustc_ast_pretty::pprust;
+use rustc_errors::{Applicability, PResult};
+use rustc_session::parse::ParseSess;
+
+use rustc_span::symbol::Ident;
+use rustc_span::Span;
+
+/// A meta-variable expression, for expansions based on properties of meta-variables.
+#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
+crate enum MetaVarExpr {
+    /// The number of repetitions of an identifier, optionally limited to a number
+    /// of outer-most repetition depths. If the depth limit is `None` then the depth is unlimited.
+    Count(Ident, Option<usize>),
+
+    /// Ignore a meta-variable for repetition without expansion.
+    Ignore(Ident),
+
+    /// The index of the repetition at a particular depth, where 0 is the inner-most
+    /// repetition. The `usize` is the depth.
+    Index(usize),
+
+    /// The length of the repetition at a particular depth, where 0 is the inner-most
+    /// repetition. The `usize` is the depth.
+    Length(usize),
+}
+
+impl MetaVarExpr {
+    /// Attempt to parse a meta-variable expression from a token stream.
+    crate fn parse<'sess>(
+        input: &tokenstream::TokenStream,
+        sess: &'sess ParseSess,
+    ) -> PResult<'sess, MetaVarExpr> {
+        let mut tts = input.trees();
+        match tts.next() {
+            Some(tokenstream::TokenTree::Token(token)) if let Some((ident, false)) = token.ident() => {
+                let Some(tokenstream::TokenTree::Delimited(_, token::Paren, args)) = tts.next() else {
+                    let msg = "meta-variable expression paramter must be wrapped in parentheses";
+                    return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
+                };
+                let mut iter = args.trees();
+                let rslt = match &*ident.as_str() {
+                    "count" => parse_count(&mut iter, sess, ident.span)?,
+                    "ignore" => MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?),
+                    "index" => MetaVarExpr::Index(parse_depth(&mut iter, sess, ident.span)?),
+                    "length" => MetaVarExpr::Length(parse_depth(&mut iter, sess, ident.span)?),
+                    _ => {
+                        let msg = "unrecognised meta-variable expression. Supported expressions \
+                        are count, ignore, index and length";
+                        return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
+                    }
+                };
+                if let Some(arg) = iter.next() {
+                    let msg = "unexpected meta-variable expression argument";
+                    return Err(sess.span_diagnostic.struct_span_err(arg.span(), msg));
+                }
+                Ok(rslt)
+            }
+            Some(tokenstream::TokenTree::Token(token)) => {
+                return Err(sess.span_diagnostic.struct_span_err(
+                    token.span,
+                    &format!(
+                        "expected meta-variable expression, found `{}`",
+                        pprust::token_to_string(&token),
+                    ),
+                ));
+            }
+            _ => return Err(sess.span_diagnostic.struct_err("expected meta-variable expression"))
+        }
+    }
+
+    crate fn ident(&self) -> Option<&Ident> {
+        match self {
+            MetaVarExpr::Count(ident, _) | MetaVarExpr::Ignore(ident) => Some(&ident),
+            MetaVarExpr::Index(..) | MetaVarExpr::Length(..) => None,
+        }
+    }
+}
+
+/// Tries to convert a literal to an arbitrary type
+fn convert_literal<T>(lit: Lit, sess: &ParseSess, span: Span) -> PResult<'_, T>
+where
+    T: FromStr,
+    <T as FromStr>::Err: Display,
+{
+    if lit.suffix.is_some() {
+        let msg = "literal suffixes are not supported in meta-variable expressions";
+        return Err(sess.span_diagnostic.struct_span_err(span, msg));
+    }
+    lit.symbol.as_str().parse::<T>().map_err(|e| {
+        sess.span_diagnostic.struct_span_err(
+            span,
+            &format!("failed to parse meta-variable expression argument: {}", e),
+        )
+    })
+}
+
+/// Parse a meta-variable `count` expression: `count(ident[, depth])`
+fn parse_count<'sess>(
+    iter: &mut tokenstream::Cursor,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, MetaVarExpr> {
+    let ident = parse_ident(iter, sess, span)?;
+    let depth = if try_eat_comma(iter) { Some(parse_depth(iter, sess, span)?) } else { None };
+    Ok(MetaVarExpr::Count(ident, depth))
+}
+
+/// Parses the depth used by index(depth) and length(depth).
+fn parse_depth<'sess>(
+    iter: &mut tokenstream::Cursor,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, usize> {
+    let Some(tt) = iter.next() else { return Ok(0) };
+    let tokenstream::TokenTree::Token(token::Token {
+        kind: token::TokenKind::Literal(lit),
+        span: literal_span,
+    }) = tt else {
+        return Err(sess.span_diagnostic.struct_span_err(
+            span,
+            "meta-expression depth must be a literal"
+        ));
+    };
+    convert_literal::<usize>(lit, sess, literal_span)
+}
+
+/// Parses an generic ident
+fn parse_ident<'sess>(
+    iter: &mut tokenstream::Cursor,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, Ident> {
+    let err_fn =
+        || sess.span_diagnostic.struct_span_err(span, "could not find an expected `ident` element");
+    if let Some(tt) = iter.next() {
+        match tt {
+            tokenstream::TokenTree::Token(token) => {
+                if let Some((elem, false)) = token.ident() {
+                    return Ok(elem);
+                }
+                let mut err = err_fn();
+                err.span_suggestion(
+                    token.span,
+                    &format!("Try removing `{}`", pprust::token_to_string(&token)),
+                    <_>::default(),
+                    Applicability::MaybeIncorrect,
+                );
+                return Err(err);
+            }
+            tokenstream::TokenTree::Delimited(delim_span, _, _) => {
+                let mut err = err_fn();
+                err.span_suggestion(
+                    delim_span.entire(),
+                    "Try removing the delimiter",
+                    <_>::default(),
+                    Applicability::MaybeIncorrect,
+                );
+                return Err(err);
+            }
+        }
+    }
+    Err(err_fn())
+}
+
+/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
+/// iterator is not modified and the result is `false`.
+fn try_eat_comma(iter: &mut tokenstream::Cursor) -> bool {
+    if let Some(tokenstream::TokenTree::Token(token::Token { kind: token::Comma, .. })) =
+        iter.look_ahead(0)
+    {
+        let _ = iter.next();
+        return true;
+    }
+    false
+}