From c2663d3e7205a97aa84a4f74a3e95e4962a28951 Mon Sep 17 00:00:00 2001 From: Steve Dignam Date: Thu, 21 May 2026 23:17:35 -0400 Subject: [PATCH] parser: improve error reporting for malformed literals --- crates/squawk_lexer/src/lib.rs | 19 ++++--- .../squawk_lexer__tests__numeric.snap | 20 ++++---- ...lexer__tests__numeric_with_seperators.snap | 2 +- crates/squawk_lexer/src/token.rs | 4 +- crates/squawk_parser/src/lexed_str.rs | 41 +++++++++------ .../snapshots/tests__select_literal_err.snap | 50 +++++++++---------- crates/squawk_parser/tests/tests.rs | 15 +++--- crates/squawk_syntax/src/parsing.rs | 8 +-- 8 files changed, 88 insertions(+), 71 deletions(-) diff --git a/crates/squawk_lexer/src/lib.rs b/crates/squawk_lexer/src/lib.rs index 40d04bbc9..5129adfc8 100644 --- a/crates/squawk_lexer/src/lib.rs +++ b/crates/squawk_lexer/src/lib.rs @@ -311,13 +311,14 @@ impl Cursor<'_> { match self.first() { '.' => self.eat_fractional(base), 'e' | 'E' => { + let exponent_start = self.pos_within_token(); self.bump(); - let empty_exponent = !self.eat_numeric_exponent(); + let empty_exponent_start = (!self.eat_numeric_exponent()).then_some(exponent_start); let trailing_junk_start = self.pos_within_token(); self.eat_identifier(); LiteralKind::Numeric { base, - empty_exponent, + empty_exponent_start, trailing_junk_start, } } @@ -504,21 +505,27 @@ impl Cursor<'_> { // might have stuff after the ., and if it does, it needs to start // with a number self.bump(); - let mut empty_exponent = false; + let mut empty_exponent_start = None; if self.first().is_ascii_digit() { self.eat_decimal_digits(); match self.first() { 'e' | 'E' => { + let exponent_start = self.pos_within_token(); self.bump(); - empty_exponent = !self.eat_numeric_exponent(); + if !self.eat_numeric_exponent() { + empty_exponent_start = Some(exponent_start); + } } _ => (), } } else { match self.first() { 'e' | 'E' => { + let exponent_start = self.pos_within_token(); self.bump(); - empty_exponent = !self.eat_numeric_exponent(); + if !self.eat_numeric_exponent() { + empty_exponent_start = Some(exponent_start); + } } _ => (), } @@ -527,7 +534,7 @@ impl Cursor<'_> { self.eat_identifier(); LiteralKind::Numeric { base, - empty_exponent, + empty_exponent_start, trailing_junk_start, } } diff --git a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap index 50d2593f4..491bb1b5f 100644 --- a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap +++ b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap @@ -6,24 +6,24 @@ expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n "\n" @ Whitespace, "42" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 2 } }, "\n" @ Whitespace, - "3.5" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 3 } }, + "3.5" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 3 } }, "\n" @ Whitespace, - "4." @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 2 } }, + "4." @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 2 } }, "\n" @ Whitespace, - ".001" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 4 } }, + ".001" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 4 } }, "\n" @ Whitespace, - ".123e10" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 7 } }, + ".123e10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 7 } }, "\n" @ Whitespace, - "5e2" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 3 } }, + "5e2" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 3 } }, "\n" @ Whitespace, - "1.925e-3" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 8 } }, + "1.925e-3" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 8 } }, "\n" @ Whitespace, - "1e-10" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 5 } }, + "1e-10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 5 } }, "\n" @ Whitespace, - "1e+10" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 5 } }, + "1e+10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 5 } }, "\n" @ Whitespace, - "1e10" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 4 } }, + "1e10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 4 } }, "\n" @ Whitespace, - "4664.E+5" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 8 } }, + "4664.E+5" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 8 } }, "\n" @ Whitespace, ] diff --git a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap index 0e0fddb4e..cc8a89a2c 100644 --- a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap +++ b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap @@ -12,6 +12,6 @@ expression: "lex(r#\"\n1_500_000_000\n0b10001000_00000000\n0o_1_755\n0xFFFF_FFFF "\n" @ Whitespace, "0xFFFF_FFFF" @ Literal { kind: Int { base: Hexadecimal, empty_int: false, trailing_junk_start: 11 } }, "\n" @ Whitespace, - "1.618_034" @ Literal { kind: Numeric { base: Decimal, empty_exponent: false, trailing_junk_start: 9 } }, + "1.618_034" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 9 } }, "\n" @ Whitespace, ] diff --git a/crates/squawk_lexer/src/token.rs b/crates/squawk_lexer/src/token.rs index 438225e00..eb2c3ee5e 100644 --- a/crates/squawk_lexer/src/token.rs +++ b/crates/squawk_lexer/src/token.rs @@ -137,7 +137,9 @@ pub enum LiteralKind { /// see: Numeric { base: Base, - empty_exponent: bool, + // e.g., `1e` instead of `1e10` + empty_exponent_start: Option, + // e.g., `1foo` where `foo` is the junk trailing_junk_start: u32, }, /// String, e.g., `'foo'` diff --git a/crates/squawk_parser/src/lexed_str.rs b/crates/squawk_parser/src/lexed_str.rs index 003031f3b..20aa46d66 100644 --- a/crates/squawk_parser/src/lexed_str.rs +++ b/crates/squawk_parser/src/lexed_str.rs @@ -15,7 +15,7 @@ pub struct LexedStr<'a> { struct LexError { msg: String, - token: u32, + range: ops::Range, } impl<'a> LexedStr<'a> { @@ -105,10 +105,10 @@ impl<'a> LexedStr<'a> { // Some(self.error[err].msg.as_str()) // } - pub fn errors(&self) -> impl Iterator + '_ { + pub fn errors(&self) -> impl Iterator, &str)> + '_ { self.error .iter() - .map(|it| (it.token as usize, it.msg.as_str())) + .map(|it| (it.range.clone(), it.msg.as_str())) } fn push(&mut self, kind: SyntaxKind, offset: usize) { @@ -140,14 +140,16 @@ impl<'a> Converter<'a> { self.res } - fn push(&mut self, kind: SyntaxKind, len: usize, err: Option<&str>) { + fn push(&mut self, kind: SyntaxKind, len: usize, err: Option<(&str, ops::Range)>) { + let token_start = self.offset as u32; self.res.push(kind, self.offset); self.offset += len; - if let Some(err) = err { - let token = self.res.len() as u32; - let msg = err.to_owned(); - self.res.error.push(LexError { msg, token }); + if let Some((msg, err_range)) = err { + self.res.error.push(LexError { + msg: msg.to_owned(), + range: token_start + err_range.start..token_start + err_range.end, + }); } } @@ -157,6 +159,7 @@ impl<'a> Converter<'a> { // Storing that info in `SyntaxKind` is not possible due to its layout requirements of // being `u16` that come from `rowan::SyntaxKind`. let mut err = ""; + let mut err_range: Option> = None; let syntax_kind = { match kind { @@ -214,6 +217,7 @@ impl<'a> Converter<'a> { } => { if (*trailing_junk_start as usize) < token_text.len() { err = "trailing junk after positional parameter"; + err_range = Some(*trailing_junk_start..token_text.len() as u32); } SyntaxKind::POSITIONAL_PARAM } @@ -227,11 +231,13 @@ impl<'a> Converter<'a> { }; let err = if err.is_empty() { None } else { Some(err) }; + let err = err.map(|msg| (msg, err_range.unwrap_or(0..token_text.len() as u32))); self.push(syntax_kind, token_text.len(), err); } fn extend_literal(&mut self, token_text: &str, kind: &squawk_lexer::LiteralKind) { let mut err: Option = None; + let mut err_range: Option> = None; let syntax_kind = match *kind { squawk_lexer::LiteralKind::Int { @@ -243,18 +249,21 @@ impl<'a> Converter<'a> { err = Some("Missing digits after the integer base prefix".into()); } else if (trailing_junk_start as usize) < token_text.len() { err = Some("trailing junk after numeric literal".into()); + err_range = Some(trailing_junk_start..token_text.len() as u32); } SyntaxKind::INT_NUMBER } squawk_lexer::LiteralKind::Numeric { - empty_exponent, + empty_exponent_start, base: _, trailing_junk_start, } => { - if empty_exponent { + if let Some(exponent_start) = empty_exponent_start { err = Some("Missing digits after the exponent symbol".into()); + err_range = Some(exponent_start..exponent_start + 1); } else if (trailing_junk_start as usize) < token_text.len() { err = Some("trailing junk after numeric literal".into()); + err_range = Some(trailing_junk_start..token_text.len() as u32); } SyntaxKind::NUMERIC_NUMBER } @@ -312,7 +321,10 @@ impl<'a> Converter<'a> { } }; - self.push(syntax_kind, token_text.len(), err.as_deref()); + let err = err + .as_deref() + .map(|msg| (msg, err_range.unwrap_or(0..token_text.len() as u32))); + self.push(syntax_kind, token_text.len(), err); } } @@ -328,11 +340,12 @@ mod tests { let renderer = Renderer::plain().decor_style(DecorStyle::Unicode); let mut res = String::new(); - for (token, msg) in lexed.errors() { + for (range, msg) in lexed.errors() { + let span = range.start as usize..range.end as usize; let group = Level::ERROR.primary_title(msg).element( Snippet::source(text) .fold(true) - .annotation(AnnotationKind::Primary.span(lexed.text_range(token))), + .annotation(AnnotationKind::Primary.span(span)), ); res.push_str(&renderer.render(&[group]).to_string()); res.push('\n'); @@ -357,7 +370,7 @@ mod tests { error: Missing digits after the exponent symbol ╭▸ 1 │ select 1e; - ╰╴ ━━ + ╰╴ ━ "); } diff --git a/crates/squawk_parser/tests/snapshots/tests__select_literal_err.snap b/crates/squawk_parser/tests/snapshots/tests__select_literal_err.snap index 691e1fb4c..973ae8137 100644 --- a/crates/squawk_parser/tests/snapshots/tests__select_literal_err.snap +++ b/crates/squawk_parser/tests/snapshots/tests__select_literal_err.snap @@ -273,83 +273,83 @@ SOURCE_FILE error[syntax-error]: trailing junk after numeric literal ╭▸ 1 │ SELECT 123abc; - ╰╴ ━ + ╰╴ ━━━ error[syntax-error]: trailing junk after numeric literal ╭▸ 2 │ SELECT 0x0o; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 3 │ SELECT 0.a; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 4 │ SELECT 0.0a; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 5 │ SELECT .0a; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 6 │ SELECT 0.0e1a; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: Missing digits after the exponent symbol ╭▸ 7 │ SELECT 0.0e; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: Missing digits after the exponent symbol ╭▸ 8 │ SELECT 0.0e+a; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after positional parameter ╭▸ 9 │ SELECT $1a; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: Missing digits after the integer base prefix ╭▸ 10 │ SELECT 0b; - ╰╴ ━ + ╰╴ ━━ error[syntax-error]: trailing junk after numeric literal ╭▸ 11 │ SELECT 1b; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 12 │ SELECT 0b0x; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: Missing digits after the integer base prefix ╭▸ 13 │ SELECT 0o; - ╰╴ ━ + ╰╴ ━━ error[syntax-error]: trailing junk after numeric literal ╭▸ 14 │ SELECT 1o; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 15 │ SELECT 0o0x; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: Missing digits after the integer base prefix ╭▸ 16 │ SELECT 0x; - ╰╴ ━ + ╰╴ ━━ error[syntax-error]: trailing junk after numeric literal ╭▸ 17 │ SELECT 1x; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 18 │ SELECT 0x0y; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 19 │ SELECT 100_; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after numeric literal ╭▸ 20 │ SELECT 100__000; - ╰╴ ━ + ╰╴ ━━━━━ error[syntax-error]: missing comma ╭▸ 21 │ SELECT _1_000.5; @@ -357,7 +357,7 @@ error[syntax-error]: missing comma error[syntax-error]: trailing junk after numeric literal ╭▸ 22 │ SELECT 1_000_.5; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: missing comma ╭▸ 22 │ SELECT 1_000_.5; @@ -365,16 +365,16 @@ error[syntax-error]: missing comma error[syntax-error]: trailing junk after numeric literal ╭▸ 23 │ SELECT 1_000._5; - ╰╴ ━ + ╰╴ ━━ error[syntax-error]: trailing junk after numeric literal ╭▸ 24 │ SELECT 1_000.5_; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: Missing digits after the exponent symbol ╭▸ 25 │ SELECT 1_000.5e_1; - ╰╴ ━ + ╰╴ ━ error[syntax-error]: trailing junk after positional parameter ╭▸ 26 │ SELECT $0_1; - ╰╴ ━ + ╰╴ ━━ diff --git a/crates/squawk_parser/tests/tests.rs b/crates/squawk_parser/tests/tests.rs index 0f62cb09a..b49d8ee37 100644 --- a/crates/squawk_parser/tests/tests.rs +++ b/crates/squawk_parser/tests/tests.rs @@ -116,7 +116,7 @@ fn parse_text(text: &str) -> (String, Option) { let output = parse(&input); let mut buf = String::new(); - let mut errors = Vec::new(); + let mut errors: Vec<(std::ops::Range, String)> = Vec::new(); let mut indent = String::new(); let mut depth = 0; let mut len = 0; @@ -140,7 +140,7 @@ fn parse_text(text: &str) -> (String, Option) { } squawk_parser::StrStep::Error { msg, pos } => { assert!(depth > 0); - errors.push((pos, msg.to_string())); + errors.push((pos..pos + 1, msg.to_string())); } }); assert_eq!( @@ -151,23 +151,22 @@ fn parse_text(text: &str) -> (String, Option) { text ); - for (token, msg) in lexed.errors() { - let pos = lexed.text_start(token); - errors.push((pos, msg.to_string())); + for (range, msg) in lexed.errors() { + errors.push((range.start as usize..range.end as usize, msg.to_string())); } let error_message = if !errors.is_empty() { - errors.sort_by_key(|(pos, _)| *pos); + errors.sort_by_key(|(range, _)| range.start); let renderer = Renderer::plain().decor_style(DecorStyle::Unicode); let mut out = "---\n".to_owned(); - for (pos, msg) in &errors { + for (range, msg) in &errors { let group = Level::ERROR.primary_title(msg).id("syntax-error").element( Snippet::source(text) .fold(true) - .annotation(AnnotationKind::Primary.span(*pos..*pos + 1)), + .annotation(AnnotationKind::Primary.span(range.clone())), ); let rendered = renderer.render(&[group]).to_string(); diff --git a/crates/squawk_syntax/src/parsing.rs b/crates/squawk_syntax/src/parsing.rs index 8d0ec4385..c3edd051a 100644 --- a/crates/squawk_syntax/src/parsing.rs +++ b/crates/squawk_syntax/src/parsing.rs @@ -52,12 +52,8 @@ pub(crate) fn build_tree( }); let (node, mut errors) = builder.finish_raw(); - for (i, err) in lexed.errors() { - let text_range = lexed.text_range(i); - let text_range = TextRange::new( - text_range.start.try_into().unwrap(), - text_range.end.try_into().unwrap(), - ); + for (text_range, err) in lexed.errors() { + let text_range = TextRange::new(text_range.start.into(), text_range.end.into()); errors.push(SyntaxError::new(err, text_range)) }