Merge #6165

6165: Cleanup r=matklad a=matklad

bors r+
🤖

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

Author: bors[bot]

crates/mbe/src/subtree_source.rs

@@ -2,7 +2,7 @@
 
 use parser::{Token, TokenSource};
 use std::cell::{Cell, Ref, RefCell};
-use syntax::{tokenize, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use tt::buffer::{Cursor, TokenBuffer};
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -155,17 +155,15 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
 }
 
 fn convert_literal(l: &tt::Literal) -> TtToken {
-    let mut kinds = tokenize(&l.text).0.into_iter().map(|token| token.kind);
-
-    let kind = match kinds.next() {
-        Some(kind) if kind.is_literal() => Some(kind),
-        Some(SyntaxKind::MINUS) => match kinds.next() {
-            Some(kind) if kind.is_literal() => Some(kind),
-            _ => None,
-        },
-        _ => None,
-    }
-    .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
+    let is_negated = l.text.starts_with('-');
+    let inner_text = &l.text[if is_negated { 1 } else { 0 }..];
+
+    let kind = lex_single_syntax_kind(inner_text)
+        .map(|(kind, _error)| kind)
+        .filter(|kind| {
+            kind.is_literal() && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
+        })
+        .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
 
     TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
 }

crates/syntax/src/parsing/lexer.rs

@@ -1,10 +1,10 @@
 //! Lexer analyzes raw input string and produces lexemes (tokens).
 //! It is just a bridge to `rustc_lexer`.
 
-use rustc_lexer::{LiteralKind as LK, RawStrError};
-
 use std::convert::TryInto;
 
+use rustc_lexer::{LiteralKind as LK, RawStrError};
+
 use crate::{
     SyntaxError,
     SyntaxKind::{self, *},
@@ -61,27 +61,30 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
     (tokens, errors)
 }
 
-/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
-/// encountered at the beginning of the string.
+/// Returns `SyntaxKind` and `Option<SyntaxError>` if `text` parses as a single token.
 ///
 /// Returns `None` if the string contains zero *or two or more* tokens.
 /// The token is malformed if the returned error is not `None`.
 ///
 /// Beware that unescape errors are not checked at tokenization time.
 pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
-    lex_first_token(text)
-        .filter(|(token, _)| token.len == TextSize::of(text))
-        .map(|(token, error)| (token.kind, error))
+    let (first_token, err) = lex_first_token(text)?;
+    if first_token.len != TextSize::of(text) {
+        return None;
+    }
+    Some((first_token.kind, err))
 }
 
 /// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and
 /// returns `None` if any tokenization error occured.
 ///
 /// Beware that unescape errors are not checked at tokenization time.
 pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
-    lex_first_token(text)
-        .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text))
-        .map(|(token, _error)| token.kind)
+    let (single_token, err) = lex_single_syntax_kind(text)?;
+    if err.is_some() {
+        return None;
+    }
+    Some(single_token)
 }
 
 /// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token