replace TokenStream::from_str panics with LexErrors

Author: cyrgani

compiler/rustc_expand/src/proc_macro_server.rs

@@ -562,13 +562,17 @@ impl server::TokenStream for Rustc<'_, '_> {
         stream.is_empty()
     }
 
-    fn from_str(&mut self, src: &str) -> Self::TokenStream {
-        unwrap_or_emit_fatal(source_str_to_stream(
+    fn from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
+        source_str_to_stream(
             self.psess(),
             FileName::proc_macro_source_code(src),
             src.to_string(),
             Some(self.call_site),
-        ))
+        )
+        .map_err(|diags| {
+            diags.into_iter().for_each(Diag::cancel);
+            "cannot parse string into token stream".to_string()
+        })
     }
 
     fn to_string(&mut self, stream: &Self::TokenStream) -> String {

library/proc_macro/src/bridge/mod.rs

@@ -61,7 +61,7 @@ macro_rules! with_api {
                 fn clone($self: &$S::TokenStream) -> $S::TokenStream;
                 fn is_empty($self: &$S::TokenStream) -> bool;
                 fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
-                fn from_str(src: &str) -> $S::TokenStream;
+                fn from_str(src: &str) -> Result<$S::TokenStream, String>;
                 fn to_string($self: &$S::TokenStream) -> String;
                 fn from_token_tree(
                     tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol>,

library/proc_macro/src/lib.rs

@@ -110,12 +110,12 @@ impl !Sync for TokenStream {}
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 #[non_exhaustive]
 #[derive(Debug)]
-pub struct LexError;
+pub struct LexError(String);
 
 #[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")]
 impl fmt::Display for LexError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str("cannot parse string into token stream")
+        f.write_str(&self.0)
     }
 }
 
@@ -194,7 +194,7 @@ impl FromStr for TokenStream {
     type Err = LexError;
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
-        Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src))))
+        Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src).map_err(LexError)?)))
     }
 }
 
@@ -1561,7 +1561,7 @@ impl FromStr for Literal {
     fn from_str(src: &str) -> Result<Self, LexError> {
         match bridge::client::FreeFunctions::literal_from_str(src) {
             Ok(literal) => Ok(Literal(literal)),
-            Err(()) => Err(LexError),
+            Err(()) => Err(LexError("cannot parse string into token stream".to_string())),
         }
     }
 }

src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs

@@ -65,6 +65,12 @@ impl server::TokenStream for RaSpanServer {
     fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
         stream.is_empty()
     }
+    #[cfg(not(bootstrap))]
+    fn from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
+        Self::TokenStream::from_str(src, self.call_site)
+            .map_err(|e| format!("failed to parse str to token stream: {e}"))
+    }
+    #[cfg(bootstrap)]
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
         Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| {
             Self::TokenStream::from_str(

src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs

@@ -54,6 +54,12 @@ impl server::TokenStream for SpanIdServer {
     fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
         stream.is_empty()
     }
+    #[cfg(not(bootstrap))]
+    fn from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
+        Self::TokenStream::from_str(src, self.call_site)
+            .map_err(|e| format!("failed to parse str to token stream: {e}"))
+    }
+    #[cfg(bootstrap)]
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
         Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| {
             Self::TokenStream::from_str(

tests/ui/proc-macro/auxiliary/invalid-punct-ident.rs

@@ -20,5 +20,6 @@ pub fn invalid_raw_ident(_: TokenStream) -> TokenStream {
 
 #[proc_macro]
 pub fn lexer_failure(_: TokenStream) -> TokenStream {
-    "a b ) c".parse().expect("parsing failed without panic")
+    assert!("a b ) c".parse::<TokenStream>().is_err());
+    TokenStream::new()
 }

tests/ui/proc-macro/invalid-punct-ident-4.rs

@@ -1,13 +1,9 @@
 //@ proc-macro: invalid-punct-ident.rs
-//@ needs-unwind proc macro panics to report errors
+//@ check-pass
 
 #[macro_use]
 extern crate invalid_punct_ident;
 
 lexer_failure!();
-//~^ ERROR proc macro panicked
-//~| ERROR unexpected closing delimiter: `)`
 
-fn main() {
-    let _recovery_witness: () = 0; //~ ERROR mismatched types
-}
+fn main() {}