Fix invalid handling of field followed by negated macro call

Author: GuillaumeGomez

src/librustdoc/html/highlight.rs

@@ -823,15 +823,31 @@ impl<'a> PeekIter<'a> {
         self.peek_pos += 1;
         if self.peek_pos - 1 < self.stored.len() {
             self.stored.get(self.peek_pos - 1)
-        } else if let Some(next) = self.iter.next() {
-            self.stored.push_back(next);
-            self.stored.back()
         } else {
-            None
+            if let Some(next) = self.iter.next() {
+                self.stored.push_back(next);
+                self.stored.back()
+            } else {
+                None
+            }
         }
         .copied()
     }
 
+    fn peek_next_if<F: Fn((TokenKind, &'a str)) -> bool>(
+        &mut self,
+        f: F,
+    ) -> Option<(TokenKind, &'a str)> {
+        let next = self.peek_next()?;
+        if f(next) {
+            Some(next)
+        } else {
+            // We go one step back.
+            self.peek_pos -= 1;
+            None
+        }
+    }
+
     fn stop_peeking(&mut self) {
         self.peek_pos = 0;
     }
@@ -903,18 +919,19 @@ fn classify<'src>(
             }
         }
 
-        if let Some((TokenKind::Colon | TokenKind::Ident, _)) = classifier.tokens.peek() {
-            let tokens = classifier.get_full_ident_path();
-            for &(token, start, end) in &tokens {
-                let text = &classifier.src[start..end];
-                classifier.advance(token, text, sink, start as u32);
-                classifier.byte_pos += text.len() as u32;
-            }
-            if !tokens.is_empty() {
-                continue;
+        if let Some((TokenKind::Colon | TokenKind::Ident, _)) = classifier.tokens.peek()
+            && let Some(nb_items) = classifier.get_full_ident_path()
+        {
+            let start = classifier.byte_pos as usize;
+            let mut len = 0;
+            for _ in 0..nb_items {
+                if let Some((_, text, _)) = classifier.next() {
+                    len += text.len();
+                }
             }
-        }
-        if let Some((token, text, before)) = classifier.next() {
+            let text = &classifier.src[start..start + len];
+            classifier.advance(TokenKind::Ident, text, sink, start as u32);
+        } else if let Some((token, text, before)) = classifier.next() {
             classifier.advance(token, text, sink, before);
         } else {
             break;
@@ -957,47 +974,47 @@ impl<'src> Classifier<'src> {
     }
 
     /// Concatenate colons and idents as one when possible.
-    fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> {
-        let start = self.byte_pos as usize;
-        let mut pos = start;
+    fn get_full_ident_path(&mut self) -> Option<usize> {
         let mut has_ident = false;
+        let mut nb_items = 0;
 
-        loop {
+        let ret = loop {
             let mut nb = 0;
-            while let Some((TokenKind::Colon, _)) = self.tokens.peek() {
-                self.tokens.next();
+            while self.tokens.peek_next_if(|(token, _)| token == TokenKind::Colon).is_some() {
                 nb += 1;
+                nb_items += 1;
             }
             // Ident path can start with "::" but if we already have content in the ident path,
             // the "::" is mandatory.
             if has_ident && nb == 0 {
-                return vec![(TokenKind::Ident, start, pos)];
+                break Some(nb_items);
             } else if nb != 0 && nb != 2 {
                 if has_ident {
-                    return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
+                    // Following `;` will be handled on its own.
+                    break Some(nb_items - 1);
                 } else {
-                    return vec![(TokenKind::Colon, start, pos + nb)];
+                    break None;
                 }
             }
 
-            if let Some((TokenKind::Ident, text)) = self.tokens.peek()
+            if let Some((TokenKind::Ident, text)) =
+                self.tokens.peek_next_if(|(token, _)| token == TokenKind::Ident)
                 && let symbol = Symbol::intern(text)
                 && (symbol.is_path_segment_keyword() || !is_keyword(symbol))
             {
-                // We only "add" the colon if there is an ident behind.
-                pos += text.len() + nb;
                 has_ident = true;
-                self.tokens.next();
+                nb_items += 1;
             } else if nb > 0 && has_ident {
-                return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
-            } else if nb > 0 {
-                return vec![(TokenKind::Colon, start, start + nb)];
+                // Following `;` will be handled on its own.
+                break Some(nb_items - 1);
             } else if has_ident {
-                return vec![(TokenKind::Ident, start, pos)];
+                break Some(nb_items);
             } else {
-                return Vec::new();
+                break None;
             }
-        }
+        };
+        self.tokens.stop_peeking();
+        ret
     }
 
     /// Wraps the tokens iteration to ensure that the `byte_pos` is always correct.
@@ -1243,7 +1260,6 @@ impl<'src> Classifier<'src> {
                 Class::MacroNonTerminal
             }
             TokenKind::Ident => {
-                let file_span = self.file_span;
                 let span = || new_span(before, text, file_span);
 
                 match text {

tests/rustdoc/macro-expansion/field-followed-by-exclamation.rs

@@ -0,0 +1,26 @@
+// This test ensures that the macro expansion is correctly handled in cases like:
+// `field: !f!`, because the `:` was simply not considered because of how paths
+// are handled.
+
+//@ compile-flags: -Zunstable-options --generate-macro-expansion
+
+#![crate_name = "foo"]
+
+//@ has 'src/foo/field-followed-by-exclamation.rs.html'
+
+struct Bar {
+    bla: bool,
+}
+
+macro_rules! f {
+    () => {{ false }}
+}
+
+const X: Bar = Bar {
+    //@ has - '//*[@class="expansion"]/*[@class="original"]/*[@class="macro"]' 'f!'
+    //@ has - '//*[@class="expansion"]/*[@class="original"]' 'f!()'
+    //@ has - '//*[@class="expansion"]/*[@class="expanded"]' '{ false }'
+    // It includes both original and expanded code.
+    //@ has - '//*[@class="expansion"]' '    bla: !{ false }f!()'
+    bla: !f!(),
+};