From 038e6440e77d5be0bfd38568f76ba99edc476918 Mon Sep 17 00:00:00 2001 From: Steve Dignam Date: Wed, 20 May 2026 19:32:57 -0400 Subject: [PATCH] lexer: fix unicode escape string issue --- crates/squawk_lexer/src/cursor.rs | 5 ++ crates/squawk_lexer/src/lib.rs | 12 +-- .../tests/data/ok/select_operators.sql | 7 ++ .../snapshots/tests__select_operators_ok.snap | 79 +++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/crates/squawk_lexer/src/cursor.rs b/crates/squawk_lexer/src/cursor.rs index c70c5febc..7388f4196 100644 --- a/crates/squawk_lexer/src/cursor.rs +++ b/crates/squawk_lexer/src/cursor.rs @@ -33,6 +33,11 @@ impl<'a> Cursor<'a> { self.chars.clone().next().unwrap_or(EOF_CHAR) } + /// Peeks the second symbol from the input stream without consuming it. + pub(crate) fn second(&self) -> char { + self.chars.clone().nth(1).unwrap_or(EOF_CHAR) + } + /// Checks if there is nothing more to consume. pub(crate) fn is_eof(&self) -> bool { self.chars.as_str().is_empty() diff --git a/crates/squawk_lexer/src/lib.rs b/crates/squawk_lexer/src/lib.rs index 7adcd0a31..8f92513da 100644 --- a/crates/squawk_lexer/src/lib.rs +++ b/crates/squawk_lexer/src/lib.rs @@ -50,17 +50,17 @@ impl Cursor<'_> { c if is_whitespace(c) => self.whitespace(), // https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS-UESCAPE - 'u' | 'U' => match self.first() { - '&' => { + 'u' | 'U' => { + if self.first() == '&' && matches!(self.second(), '\'' | '"') { self.bump(); self.prefixed_string( |terminated| LiteralKind::UnicodeEscStr { terminated }, true, ) + } else { + self.ident_or_unknown_prefix() } - _ => self.ident_or_unknown_prefix(), - }, - + } // escaped strings 'e' | 'E' => { self.prefixed_string(|terminated| LiteralKind::EscStr { terminated }, false) @@ -165,7 +165,7 @@ impl Cursor<'_> { // Known prefixes must have been handled earlier. So if // we see a prefix here, it is definitely an unknown prefix. match self.first() { - '#' | '"' | '\'' => TokenKind::UnknownPrefix, + '"' | '\'' => TokenKind::UnknownPrefix, _ => TokenKind::Ident, } } diff --git a/crates/squawk_parser/tests/data/ok/select_operators.sql b/crates/squawk_parser/tests/data/ok/select_operators.sql index ebcbd5738..3096cdfd5 100644 --- a/crates/squawk_parser/tests/data/ok/select_operators.sql +++ b/crates/squawk_parser/tests/data/ok/select_operators.sql @@ -46,6 +46,13 @@ select lseg '[(0,0),(1,1)]' # lseg '[(1,0),(0,1)]'; -- box intersection select box '(2,2),(-1,-1)' # box '(1,1),(-2,-2)'; +-- operators that sort of look like prefixes +select u#foo; +select u&foo; +select u& foo; +select u&&foo; +select u&$1; + -- closest point select point '(0,0)' ## lseg '[(2,0),(0,2)]'; diff --git a/crates/squawk_parser/tests/snapshots/tests__select_operators_ok.snap b/crates/squawk_parser/tests/snapshots/tests__select_operators_ok.snap index 7710830b9..9be374171 100644 --- a/crates/squawk_parser/tests/snapshots/tests__select_operators_ok.snap +++ b/crates/squawk_parser/tests/snapshots/tests__select_operators_ok.snap @@ -483,6 +483,85 @@ SOURCE_FILE STRING "'(1,1),(-2,-2)'" SEMICOLON ";" WHITESPACE "\n\n" + COMMENT "-- operators that sort of look like prefixes" + WHITESPACE "\n" + SELECT + SELECT_CLAUSE + SELECT_KW "select" + WHITESPACE " " + TARGET_LIST + TARGET + BIN_EXPR + NAME_REF + IDENT "u" + CUSTOM_OP + POUND "#" + NAME_REF + IDENT "foo" + SEMICOLON ";" + WHITESPACE "\n" + SELECT + SELECT_CLAUSE + SELECT_KW "select" + WHITESPACE " " + TARGET_LIST + TARGET + BIN_EXPR + NAME_REF + IDENT "u" + CUSTOM_OP + AMP "&" + NAME_REF + IDENT "foo" + SEMICOLON ";" + WHITESPACE "\n" + SELECT + SELECT_CLAUSE + SELECT_KW "select" + WHITESPACE " " + TARGET_LIST + TARGET + BIN_EXPR + NAME_REF + IDENT "u" + CUSTOM_OP + AMP "&" + WHITESPACE " " + NAME_REF + IDENT "foo" + SEMICOLON ";" + WHITESPACE "\n" + SELECT + SELECT_CLAUSE + SELECT_KW "select" + WHITESPACE " " + TARGET_LIST + TARGET + BIN_EXPR + NAME_REF + IDENT "u" + CUSTOM_OP + AMP "&" + AMP "&" + NAME_REF + IDENT "foo" + SEMICOLON ";" + WHITESPACE "\n" + SELECT + SELECT_CLAUSE + SELECT_KW "select" + WHITESPACE " " + TARGET_LIST + TARGET + BIN_EXPR + NAME_REF + IDENT "u" + CUSTOM_OP + AMP "&" + LITERAL + POSITIONAL_PARAM "$1" + SEMICOLON ";" + WHITESPACE "\n\n" COMMENT "-- closest point" WHITESPACE "\n" SELECT