diff --git a/crates/squawk_lexer/src/lib.rs b/crates/squawk_lexer/src/lib.rs index 28212677..e4fc7466 100644 --- a/crates/squawk_lexer/src/lib.rs +++ b/crates/squawk_lexer/src/lib.rs @@ -59,7 +59,7 @@ impl Cursor<'_> { false, ) } else { - self.ident_or_unknown_prefix() + self.ident() } } // escaped strings @@ -180,17 +180,6 @@ impl Cursor<'_> { TokenKind::Whitespace } - fn ident_or_unknown_prefix(&mut self) -> TokenKind { - // Start is already eaten, eat the rest of identifier. - self.eat_while(is_ident_cont); - // Known string prefixes must have been handled earlier. So if - // we see a prefix here, it is definitely unknown. - match self.first() { - '\'' => TokenKind::UnknownPrefix, - _ => TokenKind::Ident, - } - } - // see: https://github.com/postgres/postgres/blob/db0c96cc18aec417101e37e59fcc53d4bf647915/src/backend/parser/scan.l#L227 // comment ("--"{non_newline}*) pub(crate) fn line_comment(&mut self) -> TokenKind { @@ -248,14 +237,14 @@ impl Cursor<'_> { let terminated = self.double_quoted_string(); TokenKind::QuotedIdent { terminated } } - _ => self.ident_or_unknown_prefix(), + _ => self.ident(), } } fn number(&mut self, first_digit: char) -> LiteralKind { let mut base = Base::Decimal; if first_digit == '.' { - return self.eat_fractional(base); + return self.eat_fractional(); } if first_digit == '0' { // Attempt to parse encoding base. @@ -306,7 +295,7 @@ impl Cursor<'_> { }; match self.first() { - '.' => self.eat_fractional(base), + '.' => self.eat_fractional(), 'e' | 'E' => { let exponent_start = self.pos_within_token(); self.bump(); @@ -314,7 +303,6 @@ impl Cursor<'_> { let trailing_junk_start = self.pos_within_token(); self.eat_identifier(); LiteralKind::Numeric { - base, empty_exponent_start, trailing_junk_start, } @@ -514,39 +502,27 @@ impl Cursor<'_> { } } - pub(crate) fn eat_fractional(&mut self, base: Base) -> crate::LiteralKind { + pub(crate) fn eat_fractional(&mut self) -> crate::LiteralKind { // might have stuff after the ., and if it does, it needs to start // with a number self.bump(); let mut empty_exponent_start = None; if self.first().is_ascii_digit() { self.eat_decimal_digits(); - match self.first() { - 'e' | 'E' => { - let exponent_start = self.pos_within_token(); - self.bump(); - if !self.eat_numeric_exponent() { - empty_exponent_start = Some(exponent_start); - } - } - _ => (), - } - } else { - match self.first() { - 'e' | 'E' => { - let exponent_start = self.pos_within_token(); - self.bump(); - if !self.eat_numeric_exponent() { - empty_exponent_start = Some(exponent_start); - } + } + match self.first() { + 'e' | 'E' => { + let exponent_start = self.pos_within_token(); + self.bump(); + if !self.eat_numeric_exponent() { + empty_exponent_start = Some(exponent_start); } - _ => (), } + _ => (), } let trailing_junk_start = self.pos_within_token(); self.eat_identifier(); LiteralKind::Numeric { - base, empty_exponent_start, trailing_junk_start, } @@ -770,6 +746,33 @@ x'1FF' "#); } + #[test] + fn ident_prefix_then_string_is_consistent() { + assert_debug_snapshot!( + lex("N1'foo' E1'foo' B1'foo' X1'foo' U1'foo' uuid'00000000'"), + @r#" + [ + "N1" @ Ident, + "'foo'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "E1" @ Ident, + "'foo'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "B1" @ Ident, + "'foo'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "X1" @ Ident, + "'foo'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "U1" @ Ident, + "'foo'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "uuid" @ Ident, + "'00000000'" @ Literal { kind: Str { terminated: true } }, + ] + "#); + } + #[test] fn string() { assert_debug_snapshot!(lex(r#" diff --git a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap index 491bb1b5..b2a49214 100644 --- a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap +++ b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap @@ -6,24 +6,24 @@ expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n "\n" @ Whitespace, "42" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 2 } }, "\n" @ Whitespace, - "3.5" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 3 } }, + "3.5" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 3 } }, "\n" @ Whitespace, - "4." @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 2 } }, + "4." @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 2 } }, "\n" @ Whitespace, - ".001" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 4 } }, + ".001" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 4 } }, "\n" @ Whitespace, - ".123e10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 7 } }, + ".123e10" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 7 } }, "\n" @ Whitespace, - "5e2" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 3 } }, + "5e2" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 3 } }, "\n" @ Whitespace, - "1.925e-3" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 8 } }, + "1.925e-3" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 8 } }, "\n" @ Whitespace, - "1e-10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 5 } }, + "1e-10" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 5 } }, "\n" @ Whitespace, - "1e+10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 5 } }, + "1e+10" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 5 } }, "\n" @ Whitespace, - "1e10" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 4 } }, + "1e10" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 4 } }, "\n" @ Whitespace, - "4664.E+5" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 8 } }, + "4664.E+5" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 8 } }, "\n" @ Whitespace, ] diff --git a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap index cc8a89a2..9749ba38 100644 --- a/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap +++ b/crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric_with_seperators.snap @@ -12,6 +12,6 @@ expression: "lex(r#\"\n1_500_000_000\n0b10001000_00000000\n0o_1_755\n0xFFFF_FFFF "\n" @ Whitespace, "0xFFFF_FFFF" @ Literal { kind: Int { base: Hexadecimal, empty_int: false, trailing_junk_start: 11 } }, "\n" @ Whitespace, - "1.618_034" @ Literal { kind: Numeric { base: Decimal, empty_exponent_start: None, trailing_junk_start: 9 } }, + "1.618_034" @ Literal { kind: Numeric { empty_exponent_start: None, trailing_junk_start: 9 } }, "\n" @ Whitespace, ] diff --git a/crates/squawk_lexer/src/token.rs b/crates/squawk_lexer/src/token.rs index eb2c3ee5..d4873a58 100644 --- a/crates/squawk_lexer/src/token.rs +++ b/crates/squawk_lexer/src/token.rs @@ -79,8 +79,6 @@ pub enum TokenKind { OpenParen, /// `,` Comma, - /// Error case that we need to report later on. - UnknownPrefix, /// Positional Parameter, e.g., `$1` /// /// see: @@ -136,7 +134,6 @@ pub enum LiteralKind { /// /// see: Numeric { - base: Base, // e.g., `1e` instead of `1e10` empty_exponent_start: Option, // e.g., `1foo` where `foo` is the junk diff --git a/crates/squawk_parser/src/lexed_str.rs b/crates/squawk_parser/src/lexed_str.rs index 72ee8038..33e1fb80 100644 --- a/crates/squawk_parser/src/lexed_str.rs +++ b/crates/squawk_parser/src/lexed_str.rs @@ -216,10 +216,6 @@ impl<'a> Converter<'a> { squawk_lexer::TokenKind::Caret => SyntaxKind::CARET, squawk_lexer::TokenKind::Percent => SyntaxKind::PERCENT, squawk_lexer::TokenKind::Unknown => SyntaxKind::ERROR, - squawk_lexer::TokenKind::UnknownPrefix => { - err = "unknown literal prefix"; - SyntaxKind::IDENT - } squawk_lexer::TokenKind::Eof => SyntaxKind::EOF, squawk_lexer::TokenKind::Backtick => SyntaxKind::BACKTICK, squawk_lexer::TokenKind::PositionalParam { @@ -295,7 +291,6 @@ impl<'a> Converter<'a> { } squawk_lexer::LiteralKind::Numeric { empty_exponent_start, - base: _, trailing_junk_start, } => { if let Some(exponent_start) = empty_exponent_start {