feat: support parsing sql with custom tokenizer

Author: Alex Sarapulov

Cargo.toml

@@ -137,7 +137,7 @@ rand = "0.8"
 regex = "1.8"
 rstest = "0.22.0"
 serde_json = "1"
-sqlparser = { version = "0.50.0", features = ["visitor"] }
+sqlparser = { git = "https://github.com/tarantool/datafusion-sqlparser-rs.git", features = ["visitor"], branch = "release-0.50.0" }
 tempfile = "3"
 thiserror = "1.0.44"
 tokio = { version = "1.36", features = ["macros", "rt", "sync"] }

datafusion/sql/src/parser.rs

@@ -27,7 +27,7 @@ use sqlparser::{
     },
     dialect::{keywords::Keyword, Dialect, GenericDialect},
     parser::{Parser, ParserError},
-    tokenizer::{Token, TokenWithLocation, Tokenizer, Word},
+    tokenizer::{Token, TokenWithLocation, Tokenize, Tokenizer, Word},
 };
 
 // Use `Parser::expected` instead, if possible
@@ -279,9 +279,17 @@ impl<'a> DFParser<'a> {
         sql: &str,
         dialect: &'a dyn Dialect,
     ) -> Result<Self, ParserError> {
-        let mut tokenizer = Tokenizer::new(dialect, sql);
-        let tokens = tokenizer.tokenize()?;
+        let tokenizer = Tokenizer::new(dialect, sql);
+        Self::new_with_dialect_and_tokenizer(dialect, tokenizer)
+    }
 
+    /// Create a new parser for the specified tokens with
+    /// specified dialect and tokenizer.
+    pub fn new_with_dialect_and_tokenizer<T: Tokenize>(
+        dialect: &'a dyn Dialect,
+        mut tokenizer: T,
+    ) -> Result<Self, ParserError> {
+        let tokens = tokenizer.tokenize()?;
         Ok(DFParser {
             parser: Parser::new(dialect).with_tokens(tokens),
         })
@@ -300,7 +308,17 @@ impl<'a> DFParser<'a> {
         sql: &str,
         dialect: &dyn Dialect,
     ) -> Result<VecDeque<Statement>, ParserError> {
-        let mut parser = DFParser::new_with_dialect(sql, dialect)?;
+        let tokenizer = Tokenizer::new(dialect, sql);
+        Self::parse_sql_with_dialect_and_tokenizer(dialect, tokenizer)
+    }
+
+    /// Parse a SQL string and produce one or more [`Statement`]s with
+    /// with the specified dialect and tokenizer.
+    pub fn parse_sql_with_dialect_and_tokenizer<T: Tokenize>(
+        dialect: &dyn Dialect,
+        tokenizer: T,
+    ) -> Result<VecDeque<Statement>, ParserError> {
+        let mut parser = DFParser::new_with_dialect_and_tokenizer(dialect, tokenizer)?;
         let mut stmts = VecDeque::new();
         let mut expecting_statement_delimiter = false;
         loop {