From 851f1698ab9defd3728137f1e04a60dc10070f22 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 17 Dec 2025 16:57:22 +0100 Subject: [PATCH 1/2] Fix invalid handling of field followed by negated macro call --- src/librustdoc/html/highlight.rs | 76 +++++++++++-------- .../field-followed-by-exclamation.rs | 26 +++++++ 2 files changed, 71 insertions(+), 31 deletions(-) create mode 100644 tests/rustdoc/macro-expansion/field-followed-by-exclamation.rs diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 6f6345cd86664..bc268d9495f85 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -832,6 +832,20 @@ impl<'a> PeekIter<'a> { .copied() } + fn peek_next_if bool>( + &mut self, + f: F, + ) -> Option<(TokenKind, &'a str)> { + let next = self.peek_next()?; + if f(next) { + Some(next) + } else { + // We go one step back. + self.peek_pos -= 1; + None + } + } + fn stop_peeking(&mut self) { self.peek_pos = 0; } @@ -903,18 +917,19 @@ fn classify<'src>( } } - if let Some((TokenKind::Colon | TokenKind::Ident, _)) = classifier.tokens.peek() { - let tokens = classifier.get_full_ident_path(); - for &(token, start, end) in &tokens { - let text = &classifier.src[start..end]; - classifier.advance(token, text, sink, start as u32); - classifier.byte_pos += text.len() as u32; - } - if !tokens.is_empty() { - continue; + if let Some((TokenKind::Colon | TokenKind::Ident, _)) = classifier.tokens.peek() + && let Some(nb_items) = classifier.get_full_ident_path() + { + let start = classifier.byte_pos as usize; + let mut len = 0; + for _ in 0..nb_items { + if let Some((_, text, _)) = classifier.next() { + len += text.len(); + } } - } - if let Some((token, text, before)) = classifier.next() { + let text = &classifier.src[start..start + len]; + classifier.advance(TokenKind::Ident, text, sink, start as u32); + } else if let Some((token, text, before)) = classifier.next() { classifier.advance(token, text, sink, before); } else { break; @@ -957,47 +972,47 @@ impl<'src> Classifier<'src> { } /// Concatenate colons and idents as one when possible. - fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> { - let start = self.byte_pos as usize; - let mut pos = start; + fn get_full_ident_path(&mut self) -> Option { let mut has_ident = false; + let mut nb_items = 0; - loop { + let ret = loop { let mut nb = 0; - while let Some((TokenKind::Colon, _)) = self.tokens.peek() { - self.tokens.next(); + while self.tokens.peek_next_if(|(token, _)| token == TokenKind::Colon).is_some() { nb += 1; + nb_items += 1; } // Ident path can start with "::" but if we already have content in the ident path, // the "::" is mandatory. if has_ident && nb == 0 { - return vec![(TokenKind::Ident, start, pos)]; + break Some(nb_items); } else if nb != 0 && nb != 2 { if has_ident { - return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)]; + // Following `;` will be handled on its own. + break Some(nb_items - 1); } else { - return vec![(TokenKind::Colon, start, pos + nb)]; + break None; } } - if let Some((TokenKind::Ident, text)) = self.tokens.peek() + if let Some((TokenKind::Ident, text)) = + self.tokens.peek_next_if(|(token, _)| token == TokenKind::Ident) && let symbol = Symbol::intern(text) && (symbol.is_path_segment_keyword() || !is_keyword(symbol)) { - // We only "add" the colon if there is an ident behind. - pos += text.len() + nb; has_ident = true; - self.tokens.next(); + nb_items += 1; } else if nb > 0 && has_ident { - return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)]; - } else if nb > 0 { - return vec![(TokenKind::Colon, start, start + nb)]; + // Following `;` will be handled on its own. + break Some(nb_items - 1); } else if has_ident { - return vec![(TokenKind::Ident, start, pos)]; + break Some(nb_items); } else { - return Vec::new(); + break None; } - } + }; + self.tokens.stop_peeking(); + ret } /// Wraps the tokens iteration to ensure that the `byte_pos` is always correct. @@ -1243,7 +1258,6 @@ impl<'src> Classifier<'src> { Class::MacroNonTerminal } TokenKind::Ident => { - let file_span = self.file_span; let span = || new_span(before, text, file_span); match text { diff --git a/tests/rustdoc/macro-expansion/field-followed-by-exclamation.rs b/tests/rustdoc/macro-expansion/field-followed-by-exclamation.rs new file mode 100644 index 0000000000000..e80b4c4ec8731 --- /dev/null +++ b/tests/rustdoc/macro-expansion/field-followed-by-exclamation.rs @@ -0,0 +1,26 @@ +// This test ensures that the macro expansion is correctly handled in cases like: +// `field: !f!`, because the `:` was simply not considered because of how paths +// are handled. + +//@ compile-flags: -Zunstable-options --generate-macro-expansion + +#![crate_name = "foo"] + +//@ has 'src/foo/field-followed-by-exclamation.rs.html' + +struct Bar { + bla: bool, +} + +macro_rules! f { + () => {{ false }} +} + +const X: Bar = Bar { + //@ has - '//*[@class="expansion"]/*[@class="original"]/*[@class="macro"]' 'f!' + //@ has - '//*[@class="expansion"]/*[@class="original"]' 'f!()' + //@ has - '//*[@class="expansion"]/*[@class="expanded"]' '{ false }' + // It includes both original and expanded code. + //@ has - '//*[@class="expansion"]' ' bla: !{ false }f!()' + bla: !f!(), +}; From 2114b2107068617b0303f9d0e91ad12ad00a296c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 17 Dec 2025 23:27:38 +0100 Subject: [PATCH 2/2] Improve code by using `iterator::sum` instead of looping --- src/librustdoc/html/highlight.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index bc268d9495f85..f70b350de283b 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -921,12 +921,10 @@ fn classify<'src>( && let Some(nb_items) = classifier.get_full_ident_path() { let start = classifier.byte_pos as usize; - let mut len = 0; - for _ in 0..nb_items { - if let Some((_, text, _)) = classifier.next() { - len += text.len(); - } - } + let len: usize = iter::from_fn(|| classifier.next()) + .take(nb_items) + .map(|(_, text, _)| text.len()) + .sum(); let text = &classifier.src[start..start + len]; classifier.advance(TokenKind::Ident, text, sink, start as u32); } else if let Some((token, text, before)) = classifier.next() {