diff --git a/crates/squawk_ide/src/column_name.rs b/crates/squawk_ide/src/column_name.rs index 2133ef50..04eed9b8 100644 --- a/crates/squawk_ide/src/column_name.rs +++ b/crates/squawk_ide/src/column_name.rs @@ -485,7 +485,15 @@ fn name_from_expr(expr: ast::Expr, in_type: bool) -> Option<(ColumnName, SyntaxN return name_from_expr(base, in_type); } } - ast::Expr::Literal(_) | ast::Expr::PrefixExpr(_) => { + ast::Expr::Literal(literal) => { + if literal.syntax().first_token().is_some_and(|token| { + token.kind() == SyntaxKind::STRING && token.text().starts_with(['n', 'N']) + }) { + return Some((ColumnName::UnknownColumn(Some("bpchar".to_string())), node)); + } + return Some((ColumnName::UnknownColumn(None), node)); + } + ast::Expr::PrefixExpr(_) => { return Some((ColumnName::UnknownColumn(None), node)); } ast::Expr::PostfixExpr(postfix_expr) => match postfix_expr.op() { @@ -533,6 +541,8 @@ fn examples() { assert_snapshot!(name("1 + 2"), @"?column?"); assert_snapshot!(name("42"), @"?column?"); assert_snapshot!(name("'string'"), @"?column?"); + assert_snapshot!(name("n'string'"), @"bpchar"); + assert_snapshot!(name("N'string'"), @"bpchar"); // prefix assert_snapshot!(name("-42"), @"?column?"); assert_snapshot!(name("|/ 42"), @"?column?"); diff --git a/crates/squawk_lexer/src/lib.rs b/crates/squawk_lexer/src/lib.rs index 99348b59..28212677 100644 --- a/crates/squawk_lexer/src/lib.rs +++ b/crates/squawk_lexer/src/lib.rs @@ -56,6 +56,7 @@ impl Cursor<'_> { self.prefixed_string( |terminated| LiteralKind::UnicodeEscStr { terminated }, true, + false, ) } else { self.ident_or_unknown_prefix() @@ -63,18 +64,34 @@ impl Cursor<'_> { } // escaped strings 'e' | 'E' => { - self.prefixed_string(|terminated| LiteralKind::EscStr { terminated }, false) + self.prefixed_string(|terminated| LiteralKind::EscStr { terminated }, false, true) } // bit string - 'b' | 'B' => { - self.prefixed_string(|terminated| LiteralKind::BitStr { terminated }, false) - } + 'b' | 'B' => self.prefixed_string( + |terminated| LiteralKind::BitStr { terminated }, + false, + false, + ), // hexadecimal byte string - 'x' | 'X' => { - self.prefixed_string(|terminated| LiteralKind::ByteStr { terminated }, false) - } + 'x' | 'X' => self.prefixed_string( + |terminated| LiteralKind::ByteStr { terminated }, + false, + false, + ), + + // national character string + 'n' | 'N' => match self.first() { + '\'' => { + self.bump(); + let terminated = self.single_quoted_string(false); + TokenKind::Literal { + kind: LiteralKind::Str { terminated }, + } + } + _ => self.ident(), + }, // Identifier (this should be checked after other variant that can // start as identifier). @@ -137,7 +154,7 @@ impl Cursor<'_> { // String literal '\'' => { - let terminated = self.single_quoted_string(); + let terminated = self.single_quoted_string(false); let kind = LiteralKind::Str { terminated }; TokenKind::Literal { kind } } @@ -179,7 +196,7 @@ impl Cursor<'_> { pub(crate) fn line_comment(&mut self) -> TokenKind { self.bump(); - self.eat_while(|c| c != '\n'); + self.eat_while(|c| c != '\n' && c != '\r'); TokenKind::LineComment } @@ -217,11 +234,12 @@ impl Cursor<'_> { &mut self, mk_kind: fn(bool) -> LiteralKind, allows_double: bool, + backslash_escapes: bool, ) -> TokenKind { match self.first() { '\'' => { self.bump(); - let terminated = self.single_quoted_string(); + let terminated = self.single_quoted_string(backslash_escapes); let kind = mk_kind(terminated); TokenKind::Literal { kind } } @@ -313,10 +331,16 @@ impl Cursor<'_> { } } - fn single_quoted_string(&mut self) -> bool { + fn single_quoted_string(&mut self, backslash_escapes: bool) -> bool { // Parse until either quotes are terminated or error is detected. loop { match self.first() { + '\\' if backslash_escapes => { + // backslash + self.bump(); + // escaped char + self.bump(); + } // Quotes might be terminated. '\'' => { self.bump(); @@ -397,31 +421,30 @@ impl Cursor<'_> { } } else { loop { - self.eat_while(|c| c != start[0]); + self.eat_while(|c| c != '$'); if self.is_eof() { return TokenKind::Literal { kind: LiteralKind::DollarQuotedString { terminated: false }, }; } - // might be the start of our start/end sequence - let mut match_count = 0; + // Eat the leading '$' of a possible closing delimiter. + self.bump(); + + let mut matches_tag = true; for start_char in &start { if self.first() == *start_char { self.bump(); - match_count += 1; } else { - self.bump(); + matches_tag = false; break; } } - // closing '$' - let terminated = match_count == start.len(); - if self.first() == '$' && terminated { + if matches_tag && self.first() == '$' { self.bump(); return TokenKind::Literal { - kind: LiteralKind::DollarQuotedString { terminated }, + kind: LiteralKind::DollarQuotedString { terminated: true }, }; } } @@ -615,6 +638,25 @@ mod tests { assert_debug_snapshot!(result); } + #[test] + fn line_comment_cr_newline() { + assert_debug_snapshot!(lex("select 1; -- comment\rselect 2;"), @r#" + [ + "select" @ Ident, + " " @ Whitespace, + "1" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 1 } }, + ";" @ Semi, + " " @ Whitespace, + "-- comment" @ LineComment, + "\r" @ Whitespace, + "select" @ Ident, + " " @ Whitespace, + "2" @ Literal { kind: Int { base: Decimal, empty_int: false, trailing_junk_start: 1 } }, + ";" @ Semi, + ] + "#); + } + #[test] fn line_comment_whitespace() { assert_debug_snapshot!(lex(r#" @@ -714,6 +756,20 @@ x'1FF' "#)) } + #[test] + fn national_character_string() { + assert_debug_snapshot!(lex("N'foo' n'bar' numeric'1'"), @r#" + [ + "N'foo'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "n'bar'" @ Literal { kind: Str { terminated: true } }, + " " @ Whitespace, + "numeric" @ Ident, + "'1'" @ Literal { kind: Str { terminated: true } }, + ] + "#); + } + #[test] fn string() { assert_debug_snapshot!(lex(r#" @@ -763,6 +819,33 @@ e'\uAAAA \UFFFFFFFF' "#)) } + #[test] + fn escape_string_with_backslash_escaped_quote() { + assert_debug_snapshot!(lex(r"E'foo\'bar'"), @r#" + [ + "E'foo\\'bar'" @ Literal { kind: EscStr { terminated: true } }, + ] + "#); + } + + #[test] + fn escape_string_with_escaped_terminal_quote_is_unterminated() { + assert_debug_snapshot!(lex(r"E'foo\';"), @r#" + [ + "E'foo\\';" @ Literal { kind: EscStr { terminated: false } }, + ] + "#); + } + + #[test] + fn escape_string_with_even_backslashes_before_quote_is_terminated() { + assert_debug_snapshot!(lex(r"E'foo\\'"), @r#" + [ + "E'foo\\\\'" @ Literal { kind: EscStr { terminated: true } }, + ] + "#); + } + #[test] fn string_unicode_escape() { // https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS-UESCAPE @@ -804,6 +887,18 @@ U&"d!0061t!+000061" UESCAPE '!' "#); } + #[test] + fn tagged_dollar_quote_requires_leading_dollar() { + assert_debug_snapshot!(lex("select $foo$abcfoo$def$foo$;"), @r#" + [ + "select" @ Ident, + " " @ Whitespace, + "$foo$abcfoo$def$foo$" @ Literal { kind: DollarQuotedString { terminated: true } }, + ";" @ Semi, + ] + "#); + } + #[test] fn ident_non_ascii_above_latin1() { assert_debug_snapshot!(lex("ẞ Ā 漢字 𐐷"), @r#" diff --git a/crates/squawk_syntax/src/quote.rs b/crates/squawk_syntax/src/quote.rs index 3b364a99..fd0e27a6 100644 --- a/crates/squawk_syntax/src/quote.rs +++ b/crates/squawk_syntax/src/quote.rs @@ -88,11 +88,13 @@ pub fn strip_unicode_esc_prefix(text: &str) -> Option<&str> { strip_quotes(text.strip_prefix(['u', 'U'])?.strip_prefix('&')?) } +pub fn dollar_quote_tag(text: &str) -> Option<&str> { + text.strip_prefix('$')?.split_once('$').map(|(tag, _)| tag) +} + pub fn strip_dollar_quotes(text: &str) -> Option<&str> { - let after_first = text.strip_prefix('$')?; - let tag_end = after_first.find('$')?; - let tag = &after_first[..tag_end]; - let body = &after_first[tag_end + 1..]; + let tag = dollar_quote_tag(text)?; + let body = &text[tag.len() + 2..]; let closing = format!("${tag}$"); body.strip_suffix(&closing) } diff --git a/crates/squawk_syntax/src/snapshots/squawk_syntax__test__dollar_quoted_string_validation.snap b/crates/squawk_syntax/src/snapshots/squawk_syntax__test__dollar_quoted_string_validation.snap new file mode 100644 index 00000000..4ce707fd --- /dev/null +++ b/crates/squawk_syntax/src/snapshots/squawk_syntax__test__dollar_quoted_string_validation.snap @@ -0,0 +1,24 @@ +--- +source: crates/squawk_syntax/src/test.rs +input_file: crates/squawk_syntax/test_data/validation/dollar_quoted_string.sql +--- +SOURCE_FILE@0..30 + SELECT@0..29 + SELECT_CLAUSE@0..28 + SELECT_KW@0..6 "select" + WHITESPACE@6..7 " " + TARGET_LIST@7..28 + TARGET@7..28 + LITERAL@7..28 + DOLLAR_QUOTED_STRING@7..28 "$foo-bar$abc$foo-bar$" + SEMICOLON@28..29 ";" + WHITESPACE@29..30 "\n" + +error[syntax-error]: "-" is not allowed in dollar quote tags + ╭▸ +1 │ select $foo-bar$abc$foo-bar$; + ╰╴ ━ +error[syntax-error]: "-" is not allowed in dollar quote tags + ╭▸ +1 │ select $foo-bar$abc$foo-bar$; + ╰╴ ━ diff --git a/crates/squawk_syntax/src/snapshots/squawk_syntax__test__validate_string_continuation_validation.snap b/crates/squawk_syntax/src/snapshots/squawk_syntax__test__validate_string_continuation_validation.snap index a9dc8c2d..3e77f845 100644 --- a/crates/squawk_syntax/src/snapshots/squawk_syntax__test__validate_string_continuation_validation.snap +++ b/crates/squawk_syntax/src/snapshots/squawk_syntax__test__validate_string_continuation_validation.snap @@ -2,7 +2,7 @@ source: crates/squawk_syntax/src/test.rs input_file: crates/squawk_syntax/test_data/validation/validate_string_continuation.sql --- -SOURCE_FILE@0..1228 +SOURCE_FILE@0..1287 COMMENT@0..28 "-- ok strings with ne ..." WHITESPACE@28..29 "\n" SELECT@29..49 @@ -482,7 +482,21 @@ SOURCE_FILE@0..1228 WHITESPACE@1221..1222 "\n" STRING@1222..1226 "'G0'" SEMICOLON@1226..1227 ";" - WHITESPACE@1227..1228 "\n" + WHITESPACE@1227..1229 "\n\n" + COMMENT@1229..1270 "-- ok CR-only newline ..." + WHITESPACE@1270..1271 "\n" + SELECT@1271..1286 + SELECT_CLAUSE@1271..1285 + SELECT_KW@1271..1277 "select" + WHITESPACE@1277..1278 " " + TARGET_LIST@1278..1285 + TARGET@1278..1285 + LITERAL@1278..1285 + STRING@1278..1281 "'a'" + WHITESPACE@1281..1282 "\r" + STRING@1282..1285 "'b'" + SEMICOLON@1285..1286 ";" + WHITESPACE@1286..1287 "\n" error[syntax-error]: Expected new line or comma between string literals ╭▸ diff --git a/crates/squawk_syntax/src/validation.rs b/crates/squawk_syntax/src/validation.rs index affb9416..7d35505d 100644 --- a/crates/squawk_syntax/src/validation.rs +++ b/crates/squawk_syntax/src/validation.rs @@ -7,6 +7,7 @@ use std::ops::Range; use crate::ast::{AstNode, LitKind}; +use crate::quote::{dollar_quote_tag, strip_dollar_quotes}; use crate::unescape::{escape_unicode_esc_str, uescape_char}; use crate::{SyntaxNode, SyntaxToken, ast, match_ast, syntax_error::SyntaxError}; use rowan::{TextRange, TextSize}; @@ -214,7 +215,9 @@ fn validate_literal(lit: ast::Literal, acc: &mut Vec) { } LookingFor::CloseString(text_range_end, seen_new_line) => match token.kind() { WHITESPACE => { - let seen_new_line = token.text().contains("\n"); + let seen_new_line = seen_new_line + || token.text().contains('\n') + || token.text().contains('\r'); state = LookingFor::CloseString(text_range_end, seen_new_line); } COMMENT => { @@ -246,9 +249,51 @@ fn validate_literal(lit: ast::Literal, acc: &mut Vec) { validate_unicode_esc_string(&lit, acc); validate_prefixed_strings(&lit, acc); + validate_dollar_quoted_string(&lit, acc); validate_default_literal(&lit, acc); } +fn validate_dollar_quoted_string(lit: &ast::Literal, acc: &mut Vec) { + let Some(LitKind::DollarQuotedString(token)) = lit.kind() else { + return; + }; + let text = token.text(); + let Some(tag) = dollar_quote_tag(text) else { + return; + }; + let closing_tag_start = strip_dollar_quotes(text).map(|_| text.len() - tag.len() - 1); + let token_start = token.text_range().start(); + for tag_start in [Some(1), closing_tag_start].into_iter().flatten() { + validate_dollar_quote_tag(tag, token_start + TextSize::new(tag_start as u32), acc); + } +} + +// dolq_start [A-Za-z\200-\377_] +const fn is_dollar_quote_tag_start(c: char) -> bool { + matches!(c, 'a'..='z' | 'A'..='Z' | '_' | '\u{80}'..) +} + +// dolq_cont [A-Za-z\200-\377_0-9] +const fn is_dollar_quote_tag_cont(c: char) -> bool { + matches!(c, 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' | '\u{80}'..) +} + +fn validate_dollar_quote_tag(tag: &str, tag_start: TextSize, acc: &mut Vec) { + for (i, c) in tag.char_indices() { + let is_valid = if i == 0 { + is_dollar_quote_tag_start(c) + } else { + is_dollar_quote_tag_cont(c) + }; + if !is_valid { + acc.push(SyntaxError::new( + format!(r#""{c}" is not allowed in dollar quote tags"#), + offset_range(tag_start, i..i + c.len_utf8()), + )); + } + } +} + fn validate_default_literal(lit: &ast::Literal, acc: &mut Vec) { if !matches!(lit.kind(), Some(LitKind::Default(_))) { return; @@ -367,7 +412,7 @@ fn validate_bit_string_content(inner: &str, inner_start: TextSize, acc: &mut Vec for (i, c) in inner.char_indices() { if c != '0' && c != '1' { acc.push(SyntaxError::new( - format!("\"{c}\" is not a valid binary digit"), + format!(r#""{c}" is not a valid binary digit"#), offset_range(inner_start, i..i + c.len_utf8()), )); } @@ -378,7 +423,7 @@ fn validate_byte_string_content(inner: &str, inner_start: TextSize, acc: &mut Ve for (i, c) in inner.char_indices() { if !c.is_ascii_hexdigit() { acc.push(SyntaxError::new( - format!("\"{c}\" is not a valid hexadecimal digit"), + format!(r#""{c}" is not a valid hexadecimal digit"#), offset_range(inner_start, i..i + c.len_utf8()), )); } diff --git a/crates/squawk_syntax/test_data/validation/dollar_quoted_string.sql b/crates/squawk_syntax/test_data/validation/dollar_quoted_string.sql new file mode 100644 index 00000000..6038363f --- /dev/null +++ b/crates/squawk_syntax/test_data/validation/dollar_quoted_string.sql @@ -0,0 +1 @@ +select $foo-bar$abc$foo-bar$; diff --git a/crates/squawk_syntax/test_data/validation/validate_string_continuation.sql b/crates/squawk_syntax/test_data/validation/validate_string_continuation.sql index ce4e4450..0e3f5930 100644 --- a/crates/squawk_syntax/test_data/validation/validate_string_continuation.sql +++ b/crates/squawk_syntax/test_data/validation/validate_string_continuation.sql @@ -57,3 +57,6 @@ select x'1F1' -- comment select x'1F' 'F0' '11'; select x'1F' 'G0'; + +-- ok CR-only newline string continuation +select 'a' 'b';