lexer: add parsing and erroring for num trailing suffix

crates/squawk_lexer/src/lib.rs

@@ -116,7 +116,11 @@ impl Cursor<'_> {
                     while self.first().is_ascii_digit() {
                         self.bump();
                     }
-                    TokenKind::PositionalParam
+                    let trailing_junk_start = self.pos_within_token();
+                    self.eat_identifier();
+                    TokenKind::PositionalParam {
+                        trailing_junk_start,
+                    }
                 }
             }
             '`' => TokenKind::Backtick,
@@ -240,9 +244,12 @@ impl Cursor<'_> {
                     base = Base::Binary;
                     self.bump();
                     if !self.eat_decimal_digits() {
+                        let trailing_junk_start = self.pos_within_token();
+                        self.eat_identifier();
                         return LiteralKind::Int {
                             base,
                             empty_int: true,
+                            trailing_junk_start,
                         };
                     }
                 }
@@ -251,9 +258,12 @@ impl Cursor<'_> {
                     base = Base::Octal;
                     self.bump();
                     if !self.eat_decimal_digits() {
+                        let trailing_junk_start = self.pos_within_token();
+                        self.eat_identifier();
                         return LiteralKind::Int {
                             base,
                             empty_int: true,
+                            trailing_junk_start,
                         };
                     }
                 }
@@ -262,9 +272,12 @@ impl Cursor<'_> {
                     base = Base::Hexadecimal;
                     self.bump();
                     if !self.eat_hexadecimal_digits() {
+                        let trailing_junk_start = self.pos_within_token();
+                        self.eat_identifier();
                         return LiteralKind::Int {
                             base,
                             empty_int: true,
+                            trailing_junk_start,
                         };
                     }
                 }
@@ -278,9 +291,12 @@ impl Cursor<'_> {
 
                 // Just a 0.
                 _ => {
+                    let trailing_junk_start = self.pos_within_token();
+                    self.eat_identifier();
                     return LiteralKind::Int {
                         base,
                         empty_int: false,
+                        trailing_junk_start,
                     };
                 }
             }
@@ -313,23 +329,34 @@ impl Cursor<'_> {
                         _ => (),
                     }
                 }
+                let trailing_junk_start = self.pos_within_token();
+                self.eat_identifier();
                 LiteralKind::Float {
                     base,
                     empty_exponent,
+                    trailing_junk_start,
                 }
             }
             'e' | 'E' => {
                 self.bump();
                 let empty_exponent = !self.eat_float_exponent();
+                let trailing_junk_start = self.pos_within_token();
+                self.eat_identifier();
                 LiteralKind::Float {
                     base,
                     empty_exponent,
+                    trailing_junk_start,
+                }
+            }
+            _ => {
+                let trailing_junk_start = self.pos_within_token();
+                self.eat_identifier();
+                LiteralKind::Int {
+                    base,
+                    empty_int: false,
+                    trailing_junk_start,
                 }
             }
-            _ => LiteralKind::Int {
-                base,
-                empty_int: false,
-            },
         }
     }
 
@@ -452,7 +479,7 @@ impl Cursor<'_> {
         let mut has_digits = false;
         loop {
             match self.first() {
-                '_' => {
+                '_' if self.second().is_ascii_digit() => {
                     self.bump();
                 }
                 '0'..='9' => {
@@ -469,7 +496,7 @@ impl Cursor<'_> {
         let mut has_digits = false;
         loop {
             match self.first() {
-                '_' => {
+                '_' if self.second().is_ascii_hexdigit() => {
                     self.bump();
                 }
                 '0'..='9' | 'a'..='f' | 'A'..='F' => {
@@ -485,11 +512,20 @@ impl Cursor<'_> {
     /// Eats the float exponent. Returns true if at least one digit was met,
     /// and returns false otherwise.
     fn eat_float_exponent(&mut self) -> bool {
+        if self.first() == '_' {
+            return false;
+        }
         if self.first() == '-' || self.first() == '+' {
             self.bump();
         }
         self.eat_decimal_digits()
     }
+
+    fn eat_identifier(&mut self) {
+        if is_ident_start(self.first()) {
+            self.eat_while(is_ident_cont);
+        }
+    }
 }
 
 /// Creates an iterator that produces tokens from the input string.

crates/squawk_lexer/src/snapshots/squawk_lexer__tests__lex_statement.snap

@@ -5,6 +5,6 @@ expression: result
 [
     "select" @ Ident,
     " " @ Whitespace,
-    "1" @ Literal { kind: Int { base: Decimal, empty_int: false } },
+    "1" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 1 } },
     ";" @ Semi,
 ]

crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap

@@ -4,26 +4,26 @@ expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n
 ---
 [
     "\n" @ Whitespace,
-    "42" @ Literal { kind: Int { base: Decimal, empty_int: false } },
+    "42" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 2 } },
     "\n" @ Whitespace,
-    "3.5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "3.5" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 3 } },
     "\n" @ Whitespace,
-    "4." @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "4." @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 2 } },
     "\n" @ Whitespace,
-    ".001" @ Literal { kind: Int { base: Decimal, empty_int: false } },
+    ".001" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 4 } },
     "\n" @ Whitespace,
-    ".123e10" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    ".123e10" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 7 } },
     "\n" @ Whitespace,
-    "5e2" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "5e2" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 3 } },
     "\n" @ Whitespace,
-    "1.925e-3" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "1.925e-3" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 8 } },
     "\n" @ Whitespace,
-    "1e-10" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "1e-10" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 5 } },
     "\n" @ Whitespace,
-    "1e+10" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "1e+10" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 5 } },
     "\n" @ Whitespace,
-    "1e10" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "1e10" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 4 } },
     "\n" @ Whitespace,
-    "4664.E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "4664.E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 8 } },
     "\n" @ Whitespace,
 ]

crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_non_decimal.snap

@@ -4,16 +4,16 @@ expression: "lex(r#\"\n0b100101\n0B10011001\n0o273\n0O755\n0x42f\n0XFFFF\n\"#)"
 ---
 [
     "\n" @ Whitespace,
-    "0b100101" @ Literal { kind: Int { base: Binary, empty_int: false } },
+    "0b100101" @ Literal { kind: Int { base: Binary, empty_int: false, trailing_junk_start: 8 } },
     "\n" @ Whitespace,
-    "0B10011001" @ Literal { kind: Int { base: Binary, empty_int: false } },
+    "0B10011001" @ Literal { kind: Int { base: Binary, empty_int: false, trailing_junk_start: 10 } },
     "\n" @ Whitespace,
-    "0o273" @ Literal { kind: Int { base: Octal, empty_int: false } },
+    "0o273" @ Literal { kind: Int { base: Octal, empty_int: false, trailing_junk_start: 5 } },
     "\n" @ Whitespace,
-    "0O755" @ Literal { kind: Int { base: Octal, empty_int: false } },
+    "0O755" @ Literal { kind: Int { base: Octal, empty_int: false, trailing_junk_start: 5 } },
     "\n" @ Whitespace,
-    "0x42f" @ Literal { kind: Int { base: Hexadecimal, empty_int: false } },
+    "0x42f" @ Literal { kind: Int { base: Hexadecimal, empty_int: false, trailing_junk_start: 5 } },
     "\n" @ Whitespace,
-    "0XFFFF" @ Literal { kind: Int { base: Hexadecimal, empty_int: false } },
+    "0XFFFF" @ Literal { kind: Int { base: Hexadecimal, empty_int: false, trailing_junk_start: 6 } },
     "\n" @ Whitespace,
 ]

crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap

@@ -4,14 +4,14 @@ expression: "lex(r#\"\n1_500_000_000\n0b10001000_00000000\n0o_1_755\n0xFFFF_FFFF
 ---
 [
     "\n" @ Whitespace,
-    "1_500_000_000" @ Literal { kind: Int { base: Decimal, empty_int: false } },
+    "1_500_000_000" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 13 } },
     "\n" @ Whitespace,
-    "0b10001000_00000000" @ Literal { kind: Int { base: Binary, empty_int: false } },
+    "0b10001000_00000000" @ Literal { kind: Int { base: Binary, empty_int: false, trailing_junk_start: 19 } },
     "\n" @ Whitespace,
-    "0o_1_755" @ Literal { kind: Int { base: Octal, empty_int: false } },
+    "0o_1_755" @ Literal { kind: Int { base: Octal, empty_int: false, trailing_junk_start: 8 } },
     "\n" @ Whitespace,
-    "0xFFFF_FFFF" @ Literal { kind: Int { base: Hexadecimal, empty_int: false } },
+    "0xFFFF_FFFF" @ Literal { kind: Int { base: Hexadecimal, empty_int: false, trailing_junk_start: 11 } },
     "\n" @ Whitespace,
-    "1.618_034" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
+    "1.618_034" @ Literal { kind: Float { base: Decimal, empty_exponent: false, trailing_junk_start: 9 } },
     "\n" @ Whitespace,
 ]

crates/squawk_lexer/src/snapshots/squawk_lexer__tests__params.snap

@@ -6,21 +6,21 @@ expression: "lex(r#\"\nselect $1 + $2;\n\nselect $1123123123123;\n\nselect $;\n\
     "\n" @ Whitespace,
     "select" @ Ident,
     " " @ Whitespace,
-    "$1" @ PositionalParam,
+    "$1" @ PositionalParam { trailing_junk_start: 2 },
     " " @ Whitespace,
     "+" @ Plus,
     " " @ Whitespace,
-    "$2" @ PositionalParam,
+    "$2" @ PositionalParam { trailing_junk_start: 2 },
     ";" @ Semi,
     "\n\n" @ Whitespace,
     "select" @ Ident,
     " " @ Whitespace,
-    "$1123123123123" @ PositionalParam,
+    "$1123123123123" @ PositionalParam { trailing_junk_start: 14 },
     ";" @ Semi,
     "\n\n" @ Whitespace,
     "select" @ Ident,
     " " @ Whitespace,
-    "$" @ PositionalParam,
+    "$" @ PositionalParam { trailing_junk_start: 1 },
     ";" @ Semi,
     "\n" @ Whitespace,
 ]

crates/squawk_lexer/src/token.rs

@@ -84,7 +84,7 @@ pub enum TokenKind {
     /// Positional Parameter, e.g., `$1`
     ///
     /// see: <https://www.postgresql.org/docs/16/sql-expressions.html#SQL-EXPRESSIONS-PARAMETERS-POSITIONAL>
-    PositionalParam,
+    PositionalParam { trailing_junk_start: u32 },
     /// Quoted Identifier, e.g., `"update"` in `update "my_table" set "a" = 5;`
     ///
     /// These are case-sensitive, unlike [`TokenKind::Ident`]
@@ -127,11 +127,19 @@ pub enum LiteralKind {
     /// Integer Numeric, e.g., `42`
     ///
     /// see: <https://www.postgresql.org/docs/16/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS-NUMERIC>
-    Int { base: Base, empty_int: bool },
+    Int {
+        base: Base,
+        empty_int: bool,
+        trailing_junk_start: u32,
+    },
     /// Float Numeric, e.g., `1.925e-3`
     ///
     /// see: <https://www.postgresql.org/docs/16/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS-NUMERIC>
-    Float { base: Base, empty_exponent: bool },
+    Float {
+        base: Base,
+        empty_exponent: bool,
+        trailing_junk_start: u32,
+    },
     /// String, e.g., `'foo'`
     ///
     /// see: <https://www.postgresql.org/docs/16/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS>

crates/squawk_parser/src/lexed_str.rs

@@ -209,7 +209,14 @@ impl<'a> Converter<'a> {
                 }
                 squawk_lexer::TokenKind::Eof => SyntaxKind::EOF,
                 squawk_lexer::TokenKind::Backtick => SyntaxKind::BACKTICK,
-                squawk_lexer::TokenKind::PositionalParam => SyntaxKind::POSITIONAL_PARAM,
+                squawk_lexer::TokenKind::PositionalParam {
+                    trailing_junk_start,
+                } => {
+                    if (*trailing_junk_start as usize) < token_text.len() {
+                        err = "trailing junk after positional parameter";
+                    }
+                    SyntaxKind::POSITIONAL_PARAM
+                }
                 squawk_lexer::TokenKind::QuotedIdent { terminated } => {
                     if !terminated {
                         err = "Missing trailing \" to terminate the quoted identifier"
@@ -227,18 +234,27 @@ impl<'a> Converter<'a> {
         let mut err: Option<String> = None;
 
         let syntax_kind = match *kind {
-            squawk_lexer::LiteralKind::Int { empty_int, base: _ } => {
+            squawk_lexer::LiteralKind::Int {
+                empty_int,
+                base: _,
+                trailing_junk_start,
+            } => {
                 if empty_int {
                     err = Some("Missing digits after the integer base prefix".into());
+                } else if (trailing_junk_start as usize) < token_text.len() {
+                    err = Some("trailing junk after numeric literal".into());
                 }
                 SyntaxKind::INT_NUMBER
             }
             squawk_lexer::LiteralKind::Float {
                 empty_exponent,
                 base: _,
+                trailing_junk_start,
             } => {
                 if empty_exponent {
                     err = Some("Missing digits after the exponent symbol".into());
+                } else if (trailing_junk_start as usize) < token_text.len() {
+                    err = Some("trailing junk after numeric literal".into());
                 }
                 SyntaxKind::FLOAT_NUMBER
             }

crates/squawk_parser/tests/data/err/select_literal.sql

@@ -0,0 +1,26 @@
+SELECT 123abc;
+SELECT 0x0o;
+SELECT 0.a;
+SELECT 0.0a;
+SELECT .0a;
+SELECT 0.0e1a;
+SELECT 0.0e;
+SELECT 0.0e+a;
+SELECT $1a;
+SELECT 0b;
+SELECT 1b;
+SELECT 0b0x;
+SELECT 0o;
+SELECT 1o;
+SELECT 0o0x;
+SELECT 0x;
+SELECT 1x;
+SELECT 0x0y;
+SELECT 100_;
+SELECT 100__000;
+SELECT _1_000.5;
+SELECT 1_000_.5;
+SELECT 1_000._5;
+SELECT 1_000.5_;
+SELECT 1_000.5e_1;
+SELECT $0_1;