Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 2677d59

Browse files
CentrilCentril
authored
Rollup merge of rust-lang#69801 - petrochenkov:nonorm, r=Centril
rustc_parse: Remove `Parser::normalized(_prev)_token` Perform the "normalization" (renamed to "uninterpolation") on the fly when necessary. The final part of rust-lang#69579 rust-lang#69384 rust-lang#69376 rust-lang#69211 rust-lang#69034 rust-lang#69006. r? @Centril
2 parents 2409e70 + 7a30bb1 commit 2677d59

File tree

14 files changed

+200
-191
lines changed

14 files changed

+200
-191
lines changed

‎src/librustc_ast/attr/mod.rs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,10 @@ impl MetaItem {
286286
}
287287

288288
impl AttrItem {
289+
pub fn span(&self) -> Span {
290+
self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
291+
}
292+
289293
pub fn meta(&self, span: Span) -> Option<MetaItem> {
290294
Some(MetaItem {
291295
path: self.path.clone(),
@@ -437,7 +441,7 @@ impl MetaItem {
437441
I: Iterator<Item = TokenTree>,
438442
{
439443
// FIXME: Share code with `parse_path`.
440-
let path = match tokens.next() {
444+
let path = match tokens.next().map(TokenTree::uninterpolate) {
441445
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span }))
442446
| Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
443447
let mut segments = if let token::Ident(name, _) = kind {
@@ -453,7 +457,7 @@ impl MetaItem {
453457
};
454458
loop {
455459
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
456-
tokens.next()
460+
tokens.next().map(TokenTree::uninterpolate)
457461
{
458462
segments.push(PathSegment::from_ident(Ident::new(name, span)));
459463
} else {
@@ -470,7 +474,6 @@ impl MetaItem {
470474
Path { span, segments }
471475
}
472476
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
473-
token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
474477
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
475478
token::Nonterminal::NtPath(ref path) => path.clone(),
476479
_ => return None,

‎src/librustc_ast/token.rs‎

Lines changed: 69 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ use rustc_macros::HashStable_Generic;
1414
use rustc_span::symbol::kw;
1515
use rustc_span::symbol::Symbol;
1616
use rustc_span::{self, Span, DUMMY_SP};
17-
use std::fmt;
18-
use std::mem;
17+
use std::borrow::Cow;
18+
use std::{fmt, mem};
1919

2020
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
2121
#[derive(HashStable_Generic)]
@@ -225,8 +225,15 @@ pub enum TokenKind {
225225
/* Literals */
226226
Literal(Lit),
227227

228-
/* Name components */
228+
/// Identifier token.
229+
/// Do not forget about `NtIdent` when you want to match on identifiers.
230+
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
231+
/// treat regular and interpolated identifiers in the same way.
229232
Ident(ast::Name, /* is_raw */ bool),
233+
/// Lifetime identifier token.
234+
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
235+
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
236+
/// treat regular and interpolated lifetime identifiers in the same way.
230237
Lifetime(ast::Name),
231238

232239
Interpolated(Lrc<Nonterminal>),
@@ -328,6 +335,19 @@ impl Token {
328335
mem::replace(self, Token::dummy())
329336
}
330337

338+
/// For interpolated tokens, returns a span of the fragment to which the interpolated
339+
/// token refers. For all other tokens this is just a regular span.
340+
/// It is particularly important to use this for identifiers and lifetimes
341+
/// for which spans affect name resolution and edition checks.
342+
/// Note that keywords are also identifiers, so they should use this
343+
/// if they keep spans or perform edition checks.
344+
pub fn uninterpolated_span(&self) -> Span {
345+
match &self.kind {
346+
Interpolated(nt) => nt.span(),
347+
_ => self.span,
348+
}
349+
}
350+
331351
pub fn is_op(&self) -> bool {
332352
match self.kind {
333353
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
@@ -345,7 +365,7 @@ impl Token {
345365

346366
/// Returns `true` if the token can appear at the start of an expression.
347367
pub fn can_begin_expr(&self) -> bool {
348-
match self.kind {
368+
match self.uninterpolate().kind {
349369
Ident(name, is_raw) =>
350370
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
351371
OpenDelim(..) | // tuple, array or block
@@ -363,12 +383,10 @@ impl Token {
363383
Lifetime(..) | // labeled loop
364384
Pound => true, // expression attributes
365385
Interpolated(ref nt) => match **nt {
366-
NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
367386
NtLiteral(..) |
368387
NtExpr(..) |
369388
NtBlock(..) |
370-
NtPath(..) |
371-
NtLifetime(..) => true,
389+
NtPath(..) => true,
372390
_ => false,
373391
},
374392
_ => false,
@@ -377,7 +395,7 @@ impl Token {
377395

378396
/// Returns `true` if the token can appear at the start of a type.
379397
pub fn can_begin_type(&self) -> bool {
380-
match self.kind {
398+
match self.uninterpolate().kind {
381399
Ident(name, is_raw) =>
382400
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
383401
OpenDelim(Paren) | // tuple
@@ -391,8 +409,7 @@ impl Token {
391409
Lt | BinOp(Shl) | // associated path
392410
ModSep => true, // global path
393411
Interpolated(ref nt) => match **nt {
394-
NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
395-
NtTy(..) | NtPath(..) | NtLifetime(..) => true,
412+
NtTy(..) | NtPath(..) => true,
396413
_ => false,
397414
},
398415
_ => false,
@@ -433,38 +450,48 @@ impl Token {
433450
///
434451
/// Keep this in sync with `Lit::from_token`.
435452
pub fn can_begin_literal_or_bool(&self) -> bool {
436-
match self.kind {
453+
match self.uninterpolate().kind {
437454
Literal(..) | BinOp(Minus) => true,
438455
Ident(name, false) if name.is_bool_lit() => true,
439456
Interpolated(ref nt) => match &**nt {
440-
NtIdent(ident, false) if ident.name.is_bool_lit() => true,
441457
NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
442458
_ => false,
443459
},
444460
_ => false,
445461
}
446462
}
447463

464+
// A convenience function for matching on identifiers during parsing.
465+
// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
466+
// into the regular identifier or lifetime token it refers to,
467+
// otherwise returns the original token.
468+
pub fn uninterpolate(&self) -> Cow<'_, Token> {
469+
match &self.kind {
470+
Interpolated(nt) => match **nt {
471+
NtIdent(ident, is_raw) => {
472+
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
473+
}
474+
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
475+
_ => Cow::Borrowed(self),
476+
},
477+
_ => Cow::Borrowed(self),
478+
}
479+
}
480+
448481
/// Returns an identifier if this token is an identifier.
449482
pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
450-
match self.kind {
451-
Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
452-
Interpolated(ref nt) => match **nt {
453-
NtIdent(ident, is_raw) => Some((ident, is_raw)),
454-
_ => None,
455-
},
483+
let token = self.uninterpolate();
484+
match token.kind {
485+
Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)),
456486
_ => None,
457487
}
458488
}
459489

460490
/// Returns a lifetime identifier if this token is a lifetime.
461491
pub fn lifetime(&self) -> Option<ast::Ident> {
462-
match self.kind {
463-
Lifetime(name) => Some(ast::Ident::new(name, self.span)),
464-
Interpolated(ref nt) => match **nt {
465-
NtLifetime(ident) => Some(ident),
466-
_ => None,
467-
},
492+
let token = self.uninterpolate();
493+
match token.kind {
494+
Lifetime(name) => Some(ast::Ident::new(name, token.span)),
468495
_ => None,
469496
}
470497
}
@@ -714,6 +741,24 @@ pub enum Nonterminal {
714741
#[cfg(target_arch = "x86_64")]
715742
rustc_data_structures::static_assert_size!(Nonterminal, 40);
716743

744+
impl Nonterminal {
745+
fn span(&self) -> Span {
746+
match self {
747+
NtItem(item) => item.span,
748+
NtBlock(block) => block.span,
749+
NtStmt(stmt) => stmt.span,
750+
NtPat(pat) => pat.span,
751+
NtExpr(expr) | NtLiteral(expr) => expr.span,
752+
NtTy(ty) => ty.span,
753+
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
754+
NtMeta(attr_item) => attr_item.span(),
755+
NtPath(path) => path.span,
756+
NtVis(vis) => vis.span,
757+
NtTT(tt) => tt.span(),
758+
}
759+
}
760+
}
761+
717762
impl PartialEq for Nonterminal {
718763
fn eq(&self, rhs: &Self) -> bool {
719764
match (self, rhs) {

‎src/librustc_ast/tokenstream.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,13 @@ impl TokenTree {
116116
pub fn close_tt(span: DelimSpan, delim: DelimToken) -> TokenTree {
117117
TokenTree::token(token::CloseDelim(delim), span.close)
118118
}
119+
120+
pub fn uninterpolate(self) -> TokenTree {
121+
match self {
122+
TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
123+
tt => tt,
124+
}
125+
}
119126
}
120127

121128
impl<CTX> HashStable<CTX> for TokenStream

‎src/librustc_ast/util/literal.rs‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -191,23 +191,16 @@ impl Lit {
191191
///
192192
/// Keep this in sync with `Token::can_begin_literal_or_bool`.
193193
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
194-
let lit = match token.kind {
194+
let lit = match token.uninterpolate().kind {
195195
token::Ident(name, false) if name.is_bool_lit() => {
196196
token::Lit::new(token::Bool, name, None)
197197
}
198198
token::Literal(lit) => lit,
199199
token::Interpolated(ref nt) => {
200-
match &**nt {
201-
token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
202-
let lit = token::Lit::new(token::Bool, ident.name, None);
203-
return Lit::from_lit_token(lit, ident.span);
200+
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
201+
if let ast::ExprKind::Lit(lit) = &expr.kind {
202+
return Ok(lit.clone());
204203
}
205-
token::NtExpr(expr) | token::NtLiteral(expr) => {
206-
if let ast::ExprKind::Lit(lit) = &expr.kind {
207-
return Ok(lit.clone());
208-
}
209-
}
210-
_ => {}
211204
}
212205
return Err(LitError::NotLiteral);
213206
}

‎src/librustc_builtin_macros/format.rs‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -156,44 +156,43 @@ fn parse_args<'a>(
156156
if p.token == token::Eof {
157157
break;
158158
} // accept trailing commas
159-
if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
160-
named = true;
161-
let name = if let token::Ident(name, _) = p.normalized_token.kind {
159+
match p.token.ident() {
160+
Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => {
161+
named = true;
162162
p.bump();
163-
name
164-
} else {
165-
unreachable!();
166-
};
163+
p.expect(&token::Eq)?;
164+
let e = p.parse_expr()?;
165+
if let Some(prev) = names.get(&ident.name) {
166+
ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", ident))
167+
.span_label(args[*prev].span, "previously here")
168+
.span_label(e.span, "duplicate argument")
169+
.emit();
170+
continue;
171+
}
167172

168-
p.expect(&token::Eq)?;
169-
let e = p.parse_expr()?;
170-
if let Some(prev) = names.get(&name) {
171-
ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name))
172-
.span_label(args[*prev].span, "previously here")
173-
.span_label(e.span, "duplicate argument")
174-
.emit();
175-
continue;
173+
// Resolve names into slots early.
174+
// Since all the positional args are already seen at this point
175+
// if the input is valid, we can simply append to the positional
176+
// args. And remember the names.
177+
let slot = args.len();
178+
names.insert(ident.name, slot);
179+
args.push(e);
176180
}
177-
178-
// Resolve names into slots early.
179-
// Since all the positional args are already seen at this point
180-
// if the input is valid, we can simply append to the positional
181-
// args. And remember the names.
182-
let slot = args.len();
183-
names.insert(name, slot);
184-
args.push(e);
185-
} else {
186-
let e = p.parse_expr()?;
187-
if named {
188-
let mut err = ecx
189-
.struct_span_err(e.span, "positional arguments cannot follow named arguments");
190-
err.span_label(e.span, "positional arguments must be before named arguments");
191-
for pos in names.values() {
192-
err.span_label(args[*pos].span, "named argument");
181+
_ => {
182+
let e = p.parse_expr()?;
183+
if named {
184+
let mut err = ecx.struct_span_err(
185+
e.span,
186+
"positional arguments cannot follow named arguments",
187+
);
188+
err.span_label(e.span, "positional arguments must be before named arguments");
189+
for pos in names.values() {
190+
err.span_label(args[*pos].span, "named argument");
191+
}
192+
err.emit();
193193
}
194-
err.emit();
194+
args.push(e);
195195
}
196-
args.push(e);
197196
}
198197
}
199198
Ok((fmtstr, args, names))

‎src/librustc_expand/mbe/macro_parser.rs‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -750,17 +750,8 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
750750

751751
/// The token is an identifier, but not `_`.
752752
/// We prohibit passing `_` to macros expecting `ident` for now.
753-
fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
754-
match token.kind {
755-
token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
756-
token::Interpolated(ref nt) => match **nt {
757-
token::NtIdent(ident, is_raw) if ident.name != kw::Underscore => {
758-
Some((ident.name, is_raw))
759-
}
760-
_ => None,
761-
},
762-
_ => None,
763-
}
753+
fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
754+
token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
764755
}
765756

766757
/// Checks whether a non-terminal may begin with a particular token.
@@ -783,7 +774,7 @@ fn may_begin_with(token: &Token, name: Name) -> bool {
783774
&& !token.is_keyword(kw::Let)
784775
}
785776
sym::ty => token.can_begin_type(),
786-
sym::ident => get_macro_name(token).is_some(),
777+
sym::ident => get_macro_ident(token).is_some(),
787778
sym::literal => token.can_begin_literal_or_bool(),
788779
sym::vis => match token.kind {
789780
// The follow-set of :vis + "priv" keyword + interpolated
@@ -888,9 +879,9 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
888879
sym::ty => token::NtTy(p.parse_ty()?),
889880
// this could be handled like a token, since it is one
890881
sym::ident => {
891-
if let Some((name, is_raw)) = get_macro_name(&p.token) {
882+
if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
892883
p.bump();
893-
token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
884+
token::NtIdent(ident, is_raw)
894885
} else {
895886
let token_str = pprust::token_to_string(&p.token);
896887
let msg = &format!("expected ident, found {}", &token_str);

‎src/librustc_parse/lib.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#![feature(crate_visibility_modifier)]
55

66
use rustc_ast::ast;
7-
use rustc_ast::token::{self, Nonterminal, Token};
7+
use rustc_ast::token::{self, Nonterminal};
88
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
99
use rustc_ast_pretty::pprust;
1010
use rustc_data_structures::sync::Lrc;
@@ -171,8 +171,7 @@ fn maybe_source_file_to_parser(
171171
let mut parser = stream_to_parser(sess, stream, None);
172172
parser.unclosed_delims = unclosed_delims;
173173
if parser.token == token::Eof {
174-
let span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
175-
parser.set_token(Token::new(token::Eof, span));
174+
parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
176175
}
177176

178177
Ok(parser)

0 commit comments

Comments
 (0)