Add metavariables to TokenDescription.

Pasted metavariables are wrapped in invisible delimiters, which
pretty-print as empty strings, and changing that can break some proc
macros. But error messages saying "expected identifer, found ``" are
bad. So this commit adds support for metavariables in `TokenDescription`
so they print as "metavariable" in error messages, instead of "``".

It's not used meaningfully yet, but will be needed to get rid of
interpolated tokens.

Author: nnethercote

compiler/rustc_parse/messages.ftl

@@ -207,6 +207,9 @@ parse_expected_identifier_found_doc_comment = expected identifier, found doc com
 parse_expected_identifier_found_doc_comment_str = expected identifier, found doc comment `{$token}`
 parse_expected_identifier_found_keyword = expected identifier, found keyword
 parse_expected_identifier_found_keyword_str = expected identifier, found keyword `{$token}`
+parse_expected_identifier_found_metavar = expected identifier, found metavariable
+# Deliberately doesn't print `$token`, which is empty.
+parse_expected_identifier_found_metavar_str = expected identifier, found metavariable
 parse_expected_identifier_found_reserved_identifier = expected identifier, found reserved identifier
 parse_expected_identifier_found_reserved_identifier_str = expected identifier, found reserved identifier `{$token}`
 parse_expected_identifier_found_reserved_keyword = expected identifier, found reserved keyword
@@ -217,7 +220,9 @@ parse_expected_mut_or_const_in_raw_pointer_type = expected `mut` or `const` keyw
     .suggestion = add `mut` or `const` here
 
 parse_expected_semi_found_doc_comment_str = expected `;`, found doc comment `{$token}`
+# Deliberately doesn't print `$token`, which is empty.
 parse_expected_semi_found_keyword_str = expected `;`, found keyword `{$token}`
+parse_expected_semi_found_metavar_str = expected `;`, found metavariable
 parse_expected_semi_found_reserved_identifier_str = expected `;`, found reserved identifier `{$token}`
 parse_expected_semi_found_reserved_keyword_str = expected `;`, found reserved keyword `{$token}`
 parse_expected_semi_found_str = expected `;`, found `{$token}`
@@ -815,7 +820,9 @@ parse_unexpected_token_after_not_default = use `!` to perform logical negation o
 parse_unexpected_token_after_not_logical = use `!` to perform logical negation
 parse_unexpected_token_after_struct_name = expected `where`, `{"{"}`, `(`, or `;` after struct name
 parse_unexpected_token_after_struct_name_found_doc_comment = expected `where`, `{"{"}`, `(`, or `;` after struct name, found doc comment `{$token}`
+# Deliberately doesn't print `$token`, which is empty.
 parse_unexpected_token_after_struct_name_found_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found keyword `{$token}`
+parse_unexpected_token_after_struct_name_found_metavar = expected `where`, `{"{"}`, `(`, or `;` after struct name, found metavar
 parse_unexpected_token_after_struct_name_found_other = expected `where`, `{"{"}`, `(`, or `;` after struct name, found `{$token}`
 
 parse_unexpected_token_after_struct_name_found_reserved_identifier = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved identifier `{$token}`

compiler/rustc_parse/src/errors.rs

@@ -1033,6 +1033,8 @@ pub(crate) enum ExpectedIdentifierFound {
     ReservedKeyword(#[primary_span] Span),
     #[label(parse_expected_identifier_found_doc_comment)]
     DocComment(#[primary_span] Span),
+    #[label(parse_expected_identifier_found_metavar)]
+    MetaVar(#[primary_span] Span),
     #[label(parse_expected_identifier)]
     Other(#[primary_span] Span),
 }
@@ -1046,6 +1048,7 @@ impl ExpectedIdentifierFound {
             Some(TokenDescription::Keyword) => ExpectedIdentifierFound::Keyword,
             Some(TokenDescription::ReservedKeyword) => ExpectedIdentifierFound::ReservedKeyword,
             Some(TokenDescription::DocComment) => ExpectedIdentifierFound::DocComment,
+            Some(TokenDescription::MetaVar(_)) => ExpectedIdentifierFound::MetaVar,
             None => ExpectedIdentifierFound::Other,
         })(span)
     }
@@ -1080,6 +1083,9 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedIdentifier {
                 Some(TokenDescription::DocComment) => {
                     fluent::parse_expected_identifier_found_doc_comment_str
                 }
+                Some(TokenDescription::MetaVar(_)) => {
+                    fluent::parse_expected_identifier_found_metavar_str
+                }
                 None => fluent::parse_expected_identifier_found_str,
             },
         );
@@ -1138,6 +1144,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedSemi {
                 Some(TokenDescription::DocComment) => {
                     fluent::parse_expected_semi_found_doc_comment_str
                 }
+                Some(TokenDescription::MetaVar(_)) => fluent::parse_expected_semi_found_metavar_str,
                 None => fluent::parse_expected_semi_found_str,
             },
         );
@@ -1822,6 +1829,13 @@ pub(crate) enum UnexpectedTokenAfterStructName {
         span: Span,
         token: Token,
     },
+    #[diag(parse_unexpected_token_after_struct_name_found_metavar)]
+    MetaVar {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token, // unused
+    },
     #[diag(parse_unexpected_token_after_struct_name_found_other)]
     Other {
         #[primary_span]
@@ -1838,6 +1852,7 @@ impl UnexpectedTokenAfterStructName {
             Some(TokenDescription::Keyword) => Self::Keyword { span, token },
             Some(TokenDescription::ReservedKeyword) => Self::ReservedKeyword { span, token },
             Some(TokenDescription::DocComment) => Self::DocComment { span, token },
+            Some(TokenDescription::MetaVar(_)) => Self::MetaVar { span, token },
             None => Self::Other { span, token },
         }
     }

compiler/rustc_parse/src/parser/mod.rs

@@ -20,7 +20,9 @@ pub use path::PathStyle;
 
 use core::fmt;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
+use rustc_ast::token::{
+    self, Delimiter, IdentIsRaw, InvisibleOrigin, Nonterminal, NonterminalKind, Token, TokenKind,
+};
 use rustc_ast::tokenstream::{AttributesData, DelimSpacing, DelimSpan, Spacing};
 use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
 use rustc_ast::util::case::Case;
@@ -385,6 +387,12 @@ pub enum TokenDescription {
     Keyword,
     ReservedKeyword,
     DocComment,
+
+    // Expanded metavariables are wrapped in invisible delimiters which aren't
+    // pretty-printed. In error messages we must handle these specially
+    // otherwise we get confusing things in messages like "expected `(`, found
+    // ``". It's better to say "expected `(`, found type metavariable".
+    MetaVar(NonterminalKind),
 }
 
 impl TokenDescription {
@@ -394,26 +402,31 @@ impl TokenDescription {
             _ if token.is_used_keyword() => Some(TokenDescription::Keyword),
             _ if token.is_unused_keyword() => Some(TokenDescription::ReservedKeyword),
             token::DocComment(..) => Some(TokenDescription::DocComment),
+            token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
+                Some(TokenDescription::MetaVar(kind))
+            }
             _ => None,
         }
     }
 }
 
 pub fn token_descr(token: &Token) -> String {
-    let name = pprust::token_to_string(token).to_string();
-
-    let kind = match (TokenDescription::from_token(token), &token.kind) {
-        (Some(TokenDescription::ReservedIdentifier), _) => Some("reserved identifier"),
-        (Some(TokenDescription::Keyword), _) => Some("keyword"),
-        (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
-        (Some(TokenDescription::DocComment), _) => Some("doc comment"),
-        (None, TokenKind::NtIdent(..)) => Some("identifier"),
-        (None, TokenKind::NtLifetime(..)) => Some("lifetime"),
-        (None, TokenKind::Interpolated(node)) => Some(node.descr()),
-        (None, _) => None,
-    };
-
-    if let Some(kind) = kind { format!("{kind} `{name}`") } else { format!("`{name}`") }
+    use TokenDescription::*;
+
+    let s = pprust::token_to_string(token).to_string();
+
+    match (TokenDescription::from_token(token), &token.kind) {
+        (Some(ReservedIdentifier), _) => format!("reserved identifier `{s}`"),
+        (Some(Keyword), _) => format!("keyword `{s}`"),
+        (Some(ReservedKeyword), _) => format!("reserved keyword `{s}`"),
+        (Some(DocComment), _) => format!("doc comment `{s}`"),
+        // Deliberately doesn't print `s`, which is empty.
+        (Some(MetaVar(kind)), _) => format!("`{kind}` metavariable"),
+        (None, TokenKind::NtIdent(..)) => format!("identifier `{s}`"),
+        (None, TokenKind::NtLifetime(..)) => format!("lifetime `{s}`"),
+        (None, TokenKind::Interpolated(node)) => format!("{} `{s}`", node.descr()),
+        (None, _) => format!("`{s}`"),
+    }
 }
 
 impl<'a> Parser<'a> {