diff --git a/Cargo.toml b/Cargo.toml index 06fed2c68..87adec1ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ path = "src/lib.rs" default = ["std", "recursive-protection"] std = [] recursive-protection = ["std", "recursive"] +parser = [] # Enable JSON output in the `cli` example: json_example = ["serde_json", "serde"] visitor = ["sqlparser_derive"] @@ -63,4 +64,12 @@ pretty_assertions = "1" [package.metadata.docs.rs] # Document these features on docs.rs -features = ["serde", "visitor"] +features = ["parser", "serde", "visitor"] + +[[example]] +name = "cli" +required-features = ["parser"] + +[[example]] +name = "parse_select" +required-features = ["parser"] diff --git a/sqlparser_bench/Cargo.toml b/sqlparser_bench/Cargo.toml index 2c1f0ae4d..af5114d95 100644 --- a/sqlparser_bench/Cargo.toml +++ b/sqlparser_bench/Cargo.toml @@ -23,7 +23,7 @@ authors = ["Dandandan "] edition = "2018" [dependencies] -sqlparser = { path = "../" } +sqlparser = { path = "../", features = ["parser"] } [dev-dependencies] criterion = "0.5" diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index 2a44cef3e..81cbdc3a0 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -588,6 +588,7 @@ impl TryFrom for CreateTableBuilder { /// Helper return type when parsing configuration for a `CREATE TABLE` statement. #[derive(Default)] +#[cfg(feature = "parser")] pub(crate) struct CreateTableConfiguration { pub partition_by: Option>, pub cluster_by: Option>>, diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 58fa27aa8..6b0245e3a 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -59,10 +59,13 @@ fn union_spans>(iter: I) -> Span { /// /// # Example /// ``` +/// # #[cfg(feature = "parser")] /// # use sqlparser::parser::{Parser, ParserError}; /// # use sqlparser::ast::Spanned; +/// # #[cfg(feature = "parser")] /// # use sqlparser::dialect::GenericDialect; /// # use sqlparser::tokenizer::Location; +/// # #[cfg(feature = "parser")] /// # fn main() -> Result<(), ParserError> { /// let dialect = GenericDialect {}; /// let sql = r#"SELECT * @@ -78,6 +81,9 @@ fn union_spans>(iter: I) -> Span { /// assert_eq!(span.end, Location::new(2, 15)); /// # Ok(()) /// # } +/// # +/// # #[cfg(not(feature = "parser"))] +/// # fn main() {} /// ``` /// pub trait Spanned { @@ -2167,6 +2173,7 @@ impl Spanned for TableObject { } #[cfg(test)] +#[cfg(feature = "parser")] pub mod tests { use crate::dialect::{Dialect, GenericDialect, SnowflakeDialect}; use crate::parser::Parser; diff --git a/src/lib.rs b/src/lib.rs index 5d72f9f0e..e94045486 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,9 +32,12 @@ //! # Example parsing SQL text //! //! ``` +//! # #[cfg(feature = "parser")] //! use sqlparser::dialect::GenericDialect; +//! # #[cfg(feature = "parser")] //! use sqlparser::parser::Parser; //! +//! # #[cfg(feature = "parser")] //! let dialect = GenericDialect {}; // or AnsiDialect //! //! let sql = "SELECT a, b, 123, myfunc(b) \ @@ -42,8 +45,10 @@ //! WHERE a > b AND b < 100 \ //! ORDER BY a DESC, b"; //! +//! # #[cfg(feature = "parser")] //! let ast = Parser::parse_sql(&dialect, sql).unwrap(); //! +//! # #[cfg(feature = "parser")] //! println!("AST: {:?}", ast); //! ``` //! @@ -54,13 +59,17 @@ //! useful for tools that analyze and manipulate SQL. //! //! ``` +//! # #[cfg(feature = "parser")] //! # use sqlparser::dialect::GenericDialect; +//! # #[cfg(feature = "parser")] //! # use sqlparser::parser::Parser; //! let sql = "SELECT a FROM table_1"; //! +//! # #[cfg(feature = "parser")] //! // parse to a Vec //! let ast = Parser::parse_sql(&GenericDialect, sql).unwrap(); //! +//! # #[cfg(feature = "parser")] //! // The original SQL text can be generated from the AST //! assert_eq!(ast[0].to_string(), sql); //! ``` @@ -141,13 +150,17 @@ extern crate pretty_assertions; pub mod ast; #[macro_use] +#[cfg(feature = "parser")] pub mod dialect; pub mod keywords; pub mod parser; pub mod tokenizer; -#[doc(hidden)] // This is required to make utilities accessible by both the crate-internal // unit-tests and by the integration tests -// External users are not supposed to rely on this module. +// External users are not supposed to rely on these modules. +#[doc(hidden)] +#[cfg(feature = "parser")] +pub mod test_dialect_utils; +#[doc(hidden)] pub mod test_utils; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b09360078..426b30451 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -20,25 +20,36 @@ use alloc::{ vec, vec::Vec, }; +#[allow(unused_imports)] use core::{ fmt::{self, Display}, str::FromStr, }; +#[allow(unused_imports)] use helpers::attached_token::AttachedToken; +#[allow(unused_imports)] use log::debug; +#[cfg(feature = "parser")] use recursion::RecursionCounter; +#[allow(unused_imports)] use IsLateral::*; +#[allow(unused_imports)] use IsOptional::*; +#[cfg(feature = "parser")] use crate::ast::helpers::stmt_create_table::{CreateTableBuilder, CreateTableConfiguration}; +#[allow(unused_imports)] use crate::ast::Statement::CreatePolicy; use crate::ast::*; +#[cfg(feature = "parser")] use crate::dialect::*; +#[allow(unused_imports)] use crate::keywords::{Keyword, ALL_KEYWORDS}; use crate::tokenizer::*; +#[cfg(feature = "parser")] mod alter; #[derive(Debug, Clone, PartialEq, Eq)] @@ -49,6 +60,7 @@ pub enum ParserError { } // Use `Parser::expected` instead, if possible +#[allow(unused_macros)] macro_rules! parser_err { ($MSG:expr, $loc:expr) => { Err(ParserError::ParserError(format!("{}{}", $MSG, $loc))) @@ -56,6 +68,7 @@ macro_rules! parser_err { } #[cfg(feature = "std")] +#[cfg(feature = "parser")] /// Implementation [`RecursionCounter`] if std is available mod recursion { use std::cell::Cell; @@ -184,9 +197,11 @@ impl fmt::Display for ParserError { impl std::error::Error for ParserError {} // By default, allow expressions up to this deep before erroring +#[allow(unused)] const DEFAULT_REMAINING_DEPTH: usize = 50; // A constant EOF token that can be referenced. +#[allow(unused)] const EOF_TOKEN: TokenWithSpan = TokenWithSpan { token: Token::EOF, span: Span { @@ -207,8 +222,10 @@ const EOF_TOKEN: TokenWithSpan = TokenWithSpan { /// child type. /// /// See [Parser::parse_data_type] for details +#[cfg(feature = "parser")] struct MatchedTrailingBracket(bool); +#[cfg(feature = "parser")] impl From for MatchedTrailingBracket { fn from(value: bool) -> Self { Self(value) @@ -264,6 +281,7 @@ impl ParserOptions { } #[derive(Copy, Clone)] +#[allow(unused)] enum ParserState { /// The default state of the parser. Normal, @@ -309,8 +327,7 @@ enum ParserState { /// "foo" /// ] /// ``` -/// -/// +#[cfg(feature = "parser")] pub struct Parser<'a> { /// The tokens tokens: Vec, @@ -328,6 +345,7 @@ pub struct Parser<'a> { recursion_counter: RecursionCounter, } +#[cfg(feature = "parser")] impl<'a> Parser<'a> { /// Create a parser for a [`Dialect`] /// @@ -14271,6 +14289,7 @@ impl Word { } #[cfg(test)] +#[cfg(feature = "parser")] mod tests { use crate::test_utils::{all_dialects, TestedDialects}; diff --git a/src/test_dialect_utils.rs b/src/test_dialect_utils.rs new file mode 100644 index 000000000..33eb45fdf --- /dev/null +++ b/src/test_dialect_utils.rs @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// This module contains internal utilities used for testing the library. +/// While technically public, the library's users are not supposed to rely +/// on this module, as it will change without notice. +// +// It's re-exported in `src/test_utils.rs` and used in integration tests +// via `tests::test_utils::*`. +use core::fmt::Debug; + +use crate::ast::*; +use crate::dialect::*; +use crate::parser::Parser; +use crate::parser::{ParserError, ParserOptions}; +use crate::tokenizer::{Token, Tokenizer}; + +/// Tests use the methods on this struct to invoke the parser on one or +/// multiple dialects. +pub struct TestedDialects { + pub dialects: Vec>, + pub options: Option, + pub recursion_limit: Option, +} + +impl TestedDialects { + /// Create a TestedDialects with default options and the given dialects. + pub fn new(dialects: Vec>) -> Self { + Self { + dialects, + options: None, + recursion_limit: None, + } + } + + pub fn new_with_options(dialects: Vec>, options: ParserOptions) -> Self { + Self { + dialects, + options: Some(options), + recursion_limit: None, + } + } + + pub fn with_recursion_limit(mut self, recursion_limit: usize) -> Self { + self.recursion_limit = Some(recursion_limit); + self + } + + fn new_parser<'a>(&self, dialect: &'a dyn Dialect) -> Parser<'a> { + let parser = Parser::new(dialect); + let parser = if let Some(options) = &self.options { + parser.with_options(options.clone()) + } else { + parser + }; + + let parser = if let Some(recursion_limit) = &self.recursion_limit { + parser.with_recursion_limit(*recursion_limit) + } else { + parser + }; + + parser + } + + /// Run the given function for all of `self.dialects`, assert that they + /// return the same result, and return that result. + pub fn one_of_identical_results(&self, f: F) -> T + where + F: Fn(&dyn Dialect) -> T, + { + let parse_results = self.dialects.iter().map(|dialect| (dialect, f(&**dialect))); + parse_results + .fold(None, |s, (dialect, parsed)| { + if let Some((prev_dialect, prev_parsed)) = s { + assert_eq!( + prev_parsed, parsed, + "Parse results with {prev_dialect:?} are different from {dialect:?}" + ); + } + Some((dialect, parsed)) + }) + .expect("tested dialects cannot be empty") + .1 + } + + pub fn run_parser_method(&self, sql: &str, f: F) -> T + where + F: Fn(&mut Parser) -> T, + { + self.one_of_identical_results(|dialect| { + let mut parser = self.new_parser(dialect).try_with_sql(sql).unwrap(); + f(&mut parser) + }) + } + + /// Parses a single SQL string into multiple statements, ensuring + /// the result is the same for all tested dialects. + pub fn parse_sql_statements(&self, sql: &str) -> Result, ParserError> { + self.one_of_identical_results(|dialect| { + let mut tokenizer = Tokenizer::new(dialect, sql); + if let Some(options) = &self.options { + tokenizer = tokenizer.with_unescape(options.unescape); + } + let tokens = tokenizer.tokenize()?; + self.new_parser(dialect) + .with_tokens(tokens) + .parse_statements() + }) + // To fail the `ensure_multiple_dialects_are_tested` test: + // Parser::parse_sql(&**self.dialects.first().unwrap(), sql) + } + + /// Ensures that `sql` parses as a single [Statement] for all tested + /// dialects. + /// + /// In general, the canonical SQL should be the same (see crate + /// documentation for rationale) and you should prefer the `verified_` + /// variants in testing, such as [`verified_statement`] or + /// [`verified_query`]. + /// + /// If `canonical` is non empty,this function additionally asserts + /// that: + /// + /// 1. parsing `sql` results in the same [`Statement`] as parsing + /// `canonical`. + /// + /// 2. re-serializing the result of parsing `sql` produces the same + /// `canonical` sql string + pub fn one_statement_parses_to(&self, sql: &str, canonical: &str) -> Statement { + let mut statements = self.parse_sql_statements(sql).expect(sql); + assert_eq!(statements.len(), 1); + if !canonical.is_empty() && sql != canonical { + assert_eq!(self.parse_sql_statements(canonical).unwrap(), statements); + } + + let only_statement = statements.pop().unwrap(); + + if !canonical.is_empty() { + assert_eq!(canonical, only_statement.to_string()) + } + only_statement + } + + /// Ensures that `sql` parses as an [`Expr`], and that + /// re-serializing the parse result produces canonical + pub fn expr_parses_to(&self, sql: &str, canonical: &str) -> Expr { + let ast = self + .run_parser_method(sql, |parser| parser.parse_expr()) + .unwrap(); + assert_eq!(canonical, &ast.to_string()); + ast + } + + /// Ensures that `sql` parses as a single [Statement], and that + /// re-serializing the parse result produces the same `sql` + /// string (is not modified after a serialization round-trip). + pub fn verified_stmt(&self, sql: &str) -> Statement { + self.one_statement_parses_to(sql, sql) + } + + /// Ensures that `sql` parses as a single [Query], and that + /// re-serializing the parse result produces the same `sql` + /// string (is not modified after a serialization round-trip). + pub fn verified_query(&self, sql: &str) -> Query { + match self.verified_stmt(sql) { + Statement::Query(query) => *query, + _ => panic!("Expected Query"), + } + } + + /// Ensures that `sql` parses as a single [Query], and that + /// re-serializing the parse result matches the given canonical + /// sql string. + pub fn verified_query_with_canonical(&self, query: &str, canonical: &str) -> Query { + match self.one_statement_parses_to(query, canonical) { + Statement::Query(query) => *query, + _ => panic!("Expected Query"), + } + } + + /// Ensures that `sql` parses as a single [Select], and that + /// re-serializing the parse result produces the same `sql` + /// string (is not modified after a serialization round-trip). + pub fn verified_only_select(&self, query: &str) -> Select { + match *self.verified_query(query).body { + SetExpr::Select(s) => *s, + _ => panic!("Expected SetExpr::Select"), + } + } + + /// Ensures that `sql` parses as a single [`Select`], and that additionally: + /// + /// 1. parsing `sql` results in the same [`Statement`] as parsing + /// `canonical`. + /// + /// 2. re-serializing the result of parsing `sql` produces the same + /// `canonical` sql string + pub fn verified_only_select_with_canonical(&self, query: &str, canonical: &str) -> Select { + let q = match self.one_statement_parses_to(query, canonical) { + Statement::Query(query) => *query, + _ => panic!("Expected Query"), + }; + match *q.body { + SetExpr::Select(s) => *s, + _ => panic!("Expected SetExpr::Select"), + } + } + + /// Ensures that `sql` parses as an [`Expr`], and that + /// re-serializing the parse result produces the same `sql` + /// string (is not modified after a serialization round-trip). + pub fn verified_expr(&self, sql: &str) -> Expr { + self.expr_parses_to(sql, sql) + } + + /// Check that the tokenizer returns the expected tokens for the given SQL. + pub fn tokenizes_to(&self, sql: &str, expected: Vec) { + if self.dialects.is_empty() { + panic!("No dialects to test"); + } + + self.dialects.iter().for_each(|dialect| { + let mut tokenizer = Tokenizer::new(&**dialect, sql); + if let Some(options) = &self.options { + tokenizer = tokenizer.with_unescape(options.unescape); + } + let tokens = tokenizer.tokenize().unwrap(); + assert_eq!(expected, tokens, "Tokenized differently for {:?}", dialect); + }); + } +} + +/// Returns all available dialects. +pub fn all_dialects() -> TestedDialects { + TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(HiveDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(MySqlDialect {}), + Box::new(BigQueryDialect {}), + Box::new(SQLiteDialect {}), + Box::new(DuckDbDialect {}), + Box::new(DatabricksDialect {}), + Box::new(ClickHouseDialect {}), + ]) +} + +/// Returns all dialects matching the given predicate. +pub fn all_dialects_where(predicate: F) -> TestedDialects +where + F: Fn(&dyn Dialect) -> bool, +{ + let mut dialects = all_dialects(); + dialects.dialects.retain(|d| predicate(&**d)); + dialects +} + +/// Returns available dialects. The `except` predicate is used +/// to filter out specific dialects. +pub fn all_dialects_except(except: F) -> TestedDialects +where + F: Fn(&dyn Dialect) -> bool, +{ + all_dialects_where(|d| !except(d)) +} diff --git a/src/test_utils.rs b/src/test_utils.rs index 208984223..ca36f96a2 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -20,7 +20,7 @@ /// on this module, as it will change without notice. // // Integration tests (i.e. everything under `tests/`) import this -// via `tests/test_utils/helpers`. +// via `tests::test_utils::*`. #[cfg(not(feature = "std"))] use alloc::{ @@ -29,269 +29,16 @@ use alloc::{ vec, vec::Vec, }; -use core::fmt::Debug; -use crate::dialect::*; -use crate::parser::{Parser, ParserError}; -use crate::tokenizer::{Token, Tokenizer}; -use crate::{ast::*, parser::ParserOptions}; +use crate::ast::*; #[cfg(test)] use pretty_assertions::assert_eq; -/// Tests use the methods on this struct to invoke the parser on one or -/// multiple dialects. -pub struct TestedDialects { - pub dialects: Vec>, - pub options: Option, - pub recursion_limit: Option, -} - -impl TestedDialects { - /// Create a TestedDialects with default options and the given dialects. - pub fn new(dialects: Vec>) -> Self { - Self { - dialects, - options: None, - recursion_limit: None, - } - } - - pub fn new_with_options(dialects: Vec>, options: ParserOptions) -> Self { - Self { - dialects, - options: Some(options), - recursion_limit: None, - } - } - - pub fn with_recursion_limit(mut self, recursion_limit: usize) -> Self { - self.recursion_limit = Some(recursion_limit); - self - } - - fn new_parser<'a>(&self, dialect: &'a dyn Dialect) -> Parser<'a> { - let parser = Parser::new(dialect); - let parser = if let Some(options) = &self.options { - parser.with_options(options.clone()) - } else { - parser - }; - - let parser = if let Some(recursion_limit) = &self.recursion_limit { - parser.with_recursion_limit(*recursion_limit) - } else { - parser - }; - - parser - } - - /// Run the given function for all of `self.dialects`, assert that they - /// return the same result, and return that result. - pub fn one_of_identical_results(&self, f: F) -> T - where - F: Fn(&dyn Dialect) -> T, - { - let parse_results = self.dialects.iter().map(|dialect| (dialect, f(&**dialect))); - parse_results - .fold(None, |s, (dialect, parsed)| { - if let Some((prev_dialect, prev_parsed)) = s { - assert_eq!( - prev_parsed, parsed, - "Parse results with {prev_dialect:?} are different from {dialect:?}" - ); - } - Some((dialect, parsed)) - }) - .expect("tested dialects cannot be empty") - .1 - } - - pub fn run_parser_method(&self, sql: &str, f: F) -> T - where - F: Fn(&mut Parser) -> T, - { - self.one_of_identical_results(|dialect| { - let mut parser = self.new_parser(dialect).try_with_sql(sql).unwrap(); - f(&mut parser) - }) - } - - /// Parses a single SQL string into multiple statements, ensuring - /// the result is the same for all tested dialects. - pub fn parse_sql_statements(&self, sql: &str) -> Result, ParserError> { - self.one_of_identical_results(|dialect| { - let mut tokenizer = Tokenizer::new(dialect, sql); - if let Some(options) = &self.options { - tokenizer = tokenizer.with_unescape(options.unescape); - } - let tokens = tokenizer.tokenize()?; - self.new_parser(dialect) - .with_tokens(tokens) - .parse_statements() - }) - // To fail the `ensure_multiple_dialects_are_tested` test: - // Parser::parse_sql(&**self.dialects.first().unwrap(), sql) - } - - /// Ensures that `sql` parses as a single [Statement] for all tested - /// dialects. - /// - /// In general, the canonical SQL should be the same (see crate - /// documentation for rationale) and you should prefer the `verified_` - /// variants in testing, such as [`verified_statement`] or - /// [`verified_query`]. - /// - /// If `canonical` is non empty,this function additionally asserts - /// that: - /// - /// 1. parsing `sql` results in the same [`Statement`] as parsing - /// `canonical`. - /// - /// 2. re-serializing the result of parsing `sql` produces the same - /// `canonical` sql string - pub fn one_statement_parses_to(&self, sql: &str, canonical: &str) -> Statement { - let mut statements = self.parse_sql_statements(sql).expect(sql); - assert_eq!(statements.len(), 1); - if !canonical.is_empty() && sql != canonical { - assert_eq!(self.parse_sql_statements(canonical).unwrap(), statements); - } - - let only_statement = statements.pop().unwrap(); - - if !canonical.is_empty() { - assert_eq!(canonical, only_statement.to_string()) - } - only_statement - } - - /// Ensures that `sql` parses as an [`Expr`], and that - /// re-serializing the parse result produces canonical - pub fn expr_parses_to(&self, sql: &str, canonical: &str) -> Expr { - let ast = self - .run_parser_method(sql, |parser| parser.parse_expr()) - .unwrap(); - assert_eq!(canonical, &ast.to_string()); - ast - } - - /// Ensures that `sql` parses as a single [Statement], and that - /// re-serializing the parse result produces the same `sql` - /// string (is not modified after a serialization round-trip). - pub fn verified_stmt(&self, sql: &str) -> Statement { - self.one_statement_parses_to(sql, sql) - } - - /// Ensures that `sql` parses as a single [Query], and that - /// re-serializing the parse result produces the same `sql` - /// string (is not modified after a serialization round-trip). - pub fn verified_query(&self, sql: &str) -> Query { - match self.verified_stmt(sql) { - Statement::Query(query) => *query, - _ => panic!("Expected Query"), - } - } - - /// Ensures that `sql` parses as a single [Query], and that - /// re-serializing the parse result matches the given canonical - /// sql string. - pub fn verified_query_with_canonical(&self, query: &str, canonical: &str) -> Query { - match self.one_statement_parses_to(query, canonical) { - Statement::Query(query) => *query, - _ => panic!("Expected Query"), - } - } - - /// Ensures that `sql` parses as a single [Select], and that - /// re-serializing the parse result produces the same `sql` - /// string (is not modified after a serialization round-trip). - pub fn verified_only_select(&self, query: &str) -> Select { - match *self.verified_query(query).body { - SetExpr::Select(s) => *s, - _ => panic!("Expected SetExpr::Select"), - } - } - - /// Ensures that `sql` parses as a single [`Select`], and that additionally: - /// - /// 1. parsing `sql` results in the same [`Statement`] as parsing - /// `canonical`. - /// - /// 2. re-serializing the result of parsing `sql` produces the same - /// `canonical` sql string - pub fn verified_only_select_with_canonical(&self, query: &str, canonical: &str) -> Select { - let q = match self.one_statement_parses_to(query, canonical) { - Statement::Query(query) => *query, - _ => panic!("Expected Query"), - }; - match *q.body { - SetExpr::Select(s) => *s, - _ => panic!("Expected SetExpr::Select"), - } - } - - /// Ensures that `sql` parses as an [`Expr`], and that - /// re-serializing the parse result produces the same `sql` - /// string (is not modified after a serialization round-trip). - pub fn verified_expr(&self, sql: &str) -> Expr { - self.expr_parses_to(sql, sql) - } - - /// Check that the tokenizer returns the expected tokens for the given SQL. - pub fn tokenizes_to(&self, sql: &str, expected: Vec) { - if self.dialects.is_empty() { - panic!("No dialects to test"); - } - - self.dialects.iter().for_each(|dialect| { - let mut tokenizer = Tokenizer::new(&**dialect, sql); - if let Some(options) = &self.options { - tokenizer = tokenizer.with_unescape(options.unescape); - } - let tokens = tokenizer.tokenize().unwrap(); - assert_eq!(expected, tokens, "Tokenized differently for {:?}", dialect); - }); - } -} - -/// Returns all available dialects. -pub fn all_dialects() -> TestedDialects { - TestedDialects::new(vec![ - Box::new(GenericDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(HiveDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(MySqlDialect {}), - Box::new(BigQueryDialect {}), - Box::new(SQLiteDialect {}), - Box::new(DuckDbDialect {}), - Box::new(DatabricksDialect {}), - Box::new(ClickHouseDialect {}), - ]) -} - -/// Returns all dialects matching the given predicate. -pub fn all_dialects_where(predicate: F) -> TestedDialects -where - F: Fn(&dyn Dialect) -> bool, -{ - let mut dialects = all_dialects(); - dialects.dialects.retain(|d| predicate(&**d)); - dialects -} - -/// Returns available dialects. The `except` predicate is used -/// to filter out specific dialects. -pub fn all_dialects_except(except: F) -> TestedDialects -where - F: Fn(&dyn Dialect) -> bool, -{ - all_dialects_where(|d| !except(d)) -} +// Re-export everything from `src/test_dialect_utils.rs` since the symbols +// from test_dialect_utils used to be part of this module. +#[cfg(feature = "parser")] +pub use crate::test_dialect_utils::*; pub fn assert_eq_vec(expected: &[&str], actual: &[T]) { assert_eq!( diff --git a/src/tokenizer.rs b/src/tokenizer/mod.rs similarity index 99% rename from src/tokenizer.rs rename to src/tokenizer/mod.rs index 7742e8fae..e0ed63556 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer/mod.rs @@ -40,13 +40,13 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -use crate::dialect::Dialect; +use crate::ast::DollarQuotedString; +#[cfg(feature = "parser")] use crate::dialect::{ - BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, - SnowflakeDialect, + BigQueryDialect, Dialect, DuckDbDialect, GenericDialect, HiveDialect, MySqlDialect, + PostgreSqlDialect, SnowflakeDialect, }; use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; -use crate::{ast::DollarQuotedString, dialect::HiveDialect}; /// SQL Token enumeration #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -690,12 +690,14 @@ impl fmt::Display for TokenizerError { #[cfg(feature = "std")] impl std::error::Error for TokenizerError {} +#[allow(unused)] struct State<'a> { peekable: Peekable>, pub line: u64, pub col: u64, } +#[allow(unused)] impl State<'_> { /// return the next character and advance the stream pub fn next(&mut self) -> Option { @@ -727,6 +729,7 @@ impl State<'_> { } /// Represents how many quote characters enclose a string literal. +#[allow(unused)] #[derive(Copy, Clone)] enum NumStringQuoteChars { /// e.g. `"abc"`, `'abc'`, `r'abc'` @@ -736,6 +739,7 @@ enum NumStringQuoteChars { } /// Settings for tokenizing a quoted string literal. +#[allow(unused)] struct TokenizeQuotedStringSettings { /// The character used to quote the string. quote_style: char, @@ -753,6 +757,7 @@ struct TokenizeQuotedStringSettings { } /// SQL Tokenizer +#[cfg(feature = "parser")] pub struct Tokenizer<'a> { dialect: &'a dyn Dialect, query: &'a str, @@ -761,6 +766,7 @@ pub struct Tokenizer<'a> { unescape: bool, } +#[cfg(feature = "parser")] impl<'a> Tokenizer<'a> { /// Create a new SQL tokenizer for the specified SQL statement /// @@ -1948,6 +1954,7 @@ impl<'a> Tokenizer<'a> { /// Read from `chars` until `predicate` returns `false` or EOF is hit. /// Return the characters read as String, and keep the first non-matching /// char available as `chars.next()`. +#[cfg(feature = "parser")] fn peeking_take_while(chars: &mut State, mut predicate: impl FnMut(char) -> bool) -> String { let mut s = String::new(); while let Some(&ch) = chars.peek() { @@ -1962,6 +1969,7 @@ fn peeking_take_while(chars: &mut State, mut predicate: impl FnMut(char) -> bool } /// Same as peeking_take_while, but also passes the next character to the predicate. +#[cfg(feature = "parser")] fn peeking_next_take_while( chars: &mut State, mut predicate: impl FnMut(char, Option) -> bool, @@ -1979,14 +1987,17 @@ fn peeking_next_take_while( s } +#[cfg(feature = "parser")] fn unescape_single_quoted_string(chars: &mut State<'_>) -> Option { Unescape::new(chars).unescape() } +#[cfg(feature = "parser")] struct Unescape<'a: 'b, 'b> { chars: &'b mut State<'a>, } +#[cfg(feature = "parser")] impl<'a: 'b, 'b> Unescape<'a, 'b> { fn new(chars: &'b mut State<'a>) -> Self { Self { chars } @@ -2127,6 +2138,7 @@ impl<'a: 'b, 'b> Unescape<'a, 'b> { } } +#[cfg(feature = "parser")] fn unescape_unicode_single_quoted_string(chars: &mut State<'_>) -> Result { let mut unescaped = String::new(); chars.next(); // consume the opening quote @@ -2162,6 +2174,7 @@ fn unescape_unicode_single_quoted_string(chars: &mut State<'_>) -> Result, max_digits: usize, @@ -2186,6 +2199,7 @@ fn take_char_from_hex_digits( } #[cfg(test)] +#[cfg(feature = "parser")] mod tests { use super::*; use crate::dialect::{ diff --git a/tests/dialect/mod.rs b/tests/dialect/mod.rs new file mode 100644 index 000000000..4edc0cb9f --- /dev/null +++ b/tests/dialect/mod.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[cfg(feature = "parser")] +mod sqlparser_bigquery; +#[cfg(feature = "parser")] +mod sqlparser_clickhouse; +#[cfg(feature = "parser")] +mod sqlparser_common; +#[cfg(feature = "parser")] +mod sqlparser_custom_dialect; +#[cfg(feature = "parser")] +mod sqlparser_databricks; +#[cfg(feature = "parser")] +mod sqlparser_duckdb; +#[cfg(feature = "parser")] +mod sqlparser_hive; +#[cfg(feature = "parser")] +mod sqlparser_mssql; +#[cfg(feature = "parser")] +mod sqlparser_mysql; +#[cfg(feature = "parser")] +mod sqlparser_postgres; +#[cfg(feature = "parser")] +mod sqlparser_redshift; +#[cfg(feature = "parser")] +mod sqlparser_regression; +#[cfg(feature = "parser")] +mod sqlparser_snowflake; +#[cfg(feature = "parser")] +mod sqlparser_sqlite; diff --git a/tests/sqlparser_bigquery.rs b/tests/dialect/sqlparser_bigquery.rs similarity index 100% rename from tests/sqlparser_bigquery.rs rename to tests/dialect/sqlparser_bigquery.rs diff --git a/tests/sqlparser_clickhouse.rs b/tests/dialect/sqlparser_clickhouse.rs similarity index 100% rename from tests/sqlparser_clickhouse.rs rename to tests/dialect/sqlparser_clickhouse.rs diff --git a/tests/sqlparser_common.rs b/tests/dialect/sqlparser_common.rs similarity index 100% rename from tests/sqlparser_common.rs rename to tests/dialect/sqlparser_common.rs diff --git a/tests/sqlparser_custom_dialect.rs b/tests/dialect/sqlparser_custom_dialect.rs similarity index 100% rename from tests/sqlparser_custom_dialect.rs rename to tests/dialect/sqlparser_custom_dialect.rs diff --git a/tests/sqlparser_databricks.rs b/tests/dialect/sqlparser_databricks.rs similarity index 100% rename from tests/sqlparser_databricks.rs rename to tests/dialect/sqlparser_databricks.rs diff --git a/tests/sqlparser_duckdb.rs b/tests/dialect/sqlparser_duckdb.rs similarity index 100% rename from tests/sqlparser_duckdb.rs rename to tests/dialect/sqlparser_duckdb.rs diff --git a/tests/sqlparser_hive.rs b/tests/dialect/sqlparser_hive.rs similarity index 100% rename from tests/sqlparser_hive.rs rename to tests/dialect/sqlparser_hive.rs diff --git a/tests/sqlparser_mssql.rs b/tests/dialect/sqlparser_mssql.rs similarity index 100% rename from tests/sqlparser_mssql.rs rename to tests/dialect/sqlparser_mssql.rs diff --git a/tests/sqlparser_mysql.rs b/tests/dialect/sqlparser_mysql.rs similarity index 100% rename from tests/sqlparser_mysql.rs rename to tests/dialect/sqlparser_mysql.rs diff --git a/tests/sqlparser_postgres.rs b/tests/dialect/sqlparser_postgres.rs similarity index 100% rename from tests/sqlparser_postgres.rs rename to tests/dialect/sqlparser_postgres.rs diff --git a/tests/sqlparser_redshift.rs b/tests/dialect/sqlparser_redshift.rs similarity index 100% rename from tests/sqlparser_redshift.rs rename to tests/dialect/sqlparser_redshift.rs diff --git a/tests/sqlparser_regression.rs b/tests/dialect/sqlparser_regression.rs similarity index 100% rename from tests/sqlparser_regression.rs rename to tests/dialect/sqlparser_regression.rs diff --git a/tests/sqlparser_snowflake.rs b/tests/dialect/sqlparser_snowflake.rs similarity index 100% rename from tests/sqlparser_snowflake.rs rename to tests/dialect/sqlparser_snowflake.rs diff --git a/tests/sqlparser_sqlite.rs b/tests/dialect/sqlparser_sqlite.rs similarity index 100% rename from tests/sqlparser_sqlite.rs rename to tests/dialect/sqlparser_sqlite.rs