From a47e58ce93fe7cbe30f03d70bc238425809807ae Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 19:01:53 +0300 Subject: [PATCH 01/15] upgrade sqlparser 0.47 -> 0.48 --- Cargo.toml | 2 +- datafusion/sql/src/expr/function.rs | 1 + datafusion/sql/src/parser.rs | 18 +- datafusion/sql/src/planner.rs | 21 ++ datafusion/sql/src/select.rs | 2 +- datafusion/sql/src/statement.rs | 14 +- datafusion/sql/src/unparser/ast.rs | 5 +- datafusion/sql/src/unparser/expr.rs | 3 + datafusion/sql/src/unparser/plan.rs | 1 + datafusion/sql/tests/sql_integration.rs | 6 +- .../sqllogictest/test_files/aggregate.slt | 18 +- .../sqllogictest/test_files/arrow_typeof.slt | 2 +- .../sqllogictest/test_files/coalesce.slt | 15 +- datafusion/sqllogictest/test_files/copy.slt | 4 +- .../test_files/create_external_table.slt | 14 +- .../sqllogictest/test_files/csv_files.slt | 13 +- .../sqllogictest/test_files/encoding.slt | 30 +-- datafusion/sqllogictest/test_files/expr.slt | 12 +- .../sqllogictest/test_files/group_by.slt | 42 +--- datafusion/sqllogictest/test_files/joins.slt | 5 +- datafusion/sqllogictest/test_files/math.slt | 128 +++-------- datafusion/sqllogictest/test_files/misc.slt | 2 +- .../sqllogictest/test_files/predicates.slt | 42 +--- datafusion/sqllogictest/test_files/scalar.slt | 52 +---- datafusion/sqllogictest/test_files/select.slt | 41 +--- .../sqllogictest/test_files/strings.slt | 40 +--- datafusion/sqllogictest/test_files/struct.slt | 79 ++----- datafusion/sqllogictest/test_files/union.slt | 205 ++---------------- datafusion/sqllogictest/test_files/unnest.slt | 6 +- datafusion/sqllogictest/test_files/window.slt | 2 +- 30 files changed, 194 insertions(+), 631 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6dd434abc87c..f61ed7e58fe3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,7 +123,7 @@ rand = "0.8" regex = "1.8" rstest = "0.21.0" serde_json = "1" -sqlparser = { version = "0.47", features = ["visitor"] } +sqlparser = { version = "0.48", features = ["visitor"] } tempfile = "3" thiserror = "1.0.44" tokio = { version = "1.36", features = ["macros", "rt", "sync"] } diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rs index d9ddf57eb192..dab328cc4908 100644 --- a/datafusion/sql/src/expr/function.rs +++ b/datafusion/sql/src/expr/function.rs @@ -109,6 +109,7 @@ impl FunctionArgs { filter, mut null_treatment, within_group, + .. } = function; // Handle no argument form (aka `current_time` as opposed to `current_time()`) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 5da7f7176509..7484ef6350f1 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -1006,14 +1006,15 @@ mod tests { expect_parse_ok(sql, expected)?; // positive case: it is ok for sql stmt with `COMPRESSION TYPE GZIP` tokens - let sqls = vec![ - ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS + let sqls = + vec![ + ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS ('format.compression' 'GZIP')", "GZIP"), - ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS + ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS ('format.compression' 'BZIP2')", "BZIP2"), - ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS + ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS ('format.compression' 'XZ')", "XZ"), - ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS + ("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS ('format.compression' 'ZSTD')", "ZSTD"), ]; for (sql, compression) in sqls { @@ -1123,7 +1124,10 @@ mod tests { // negative case: mixed column defs and column names in `PARTITIONED BY` clause let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV PARTITIONED BY (p1 int, c1) LOCATION 'foo.csv'"; - expect_parse_error(sql, "sql parser error: Expected a data type name, found: )"); + expect_parse_error( + sql, + "sql parser error: Expected: a data type name, found: )", + ); // negative case: mixed column defs and column names in `PARTITIONED BY` clause let sql = @@ -1291,7 +1295,7 @@ mod tests { LOCATION 'foo.parquet' OPTIONS ('format.compression' 'zstd', 'format.delimiter' '*', - 'ROW_GROUP_SIZE' '1024', + 'ROW_GROUP_SIZE' '1024', 'TRUNCATE' 'NO', 'format.has_header' 'true')"; let expected = Statement::CreateExternalTable(CreateExternalTable { diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index a77f0003f738..be04f51f4f2c 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -468,6 +468,27 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { | SQLDataType::Float64 | SQLDataType::JSONB | SQLDataType::Unspecified + // Clickhouse datatypes + | SQLDataType::Int16 + | SQLDataType::Int32 + | SQLDataType::Int128 + | SQLDataType::Int256 + | SQLDataType::UInt8 + | SQLDataType::UInt16 + | SQLDataType::UInt32 + | SQLDataType::UInt64 + | SQLDataType::UInt128 + | SQLDataType::UInt256 + | SQLDataType::Float32 + | SQLDataType::Date32 + | SQLDataType::Datetime64(_, _) + | SQLDataType::FixedString(_) + | SQLDataType::Map(_, _) + | SQLDataType::Tuple(_) + | SQLDataType::Nested(_) + | SQLDataType::Union(_) + | SQLDataType::Nullable(_) + | SQLDataType::LowCardinality(_) => not_impl_err!( "Unsupported SQL type {sql_type:?}" ), diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 236403e83d74..a5891e655a05 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -149,7 +149,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let aggr_exprs = find_aggregate_exprs(&aggr_expr_haystack); // All of the group by expressions - let group_by_exprs = if let GroupByExpr::Expressions(exprs) = select.group_by { + let group_by_exprs = if let GroupByExpr::Expressions(exprs, _) = select.group_by { exprs .into_iter() .map(|e| { diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 1acfac79acc0..ee050381a73d 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -52,7 +52,7 @@ use datafusion_expr::{ TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, }; -use sqlparser::ast; +use sqlparser::ast::{self, AssignmentTarget, CreateTable}; use sqlparser::ast::{ Assignment, ColumnDef, CreateTableOptions, Delete, DescribeAlias, Expr as SQLExpr, Expr, FromTable, Ident, Insert, ObjectName, ObjectType, OneOrManyWithParens, Query, @@ -240,7 +240,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { value, } => self.set_variable_to_plan(local, hivevar, &variables, value), - Statement::CreateTable { + Statement::CreateTable(CreateTable { query, name, columns, @@ -250,7 +250,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { if_not_exists, or_replace, .. - } if table_properties.is_empty() && with_options.is_empty() => { + }) if table_properties.is_empty() && with_options.is_empty() => { // Merge inline constraints and existing constraints let mut all_constraints = constraints; let inline_constraints = calc_inline_constraints_from_columns(&columns); @@ -1284,8 +1284,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let mut assign_map = assignments .iter() .map(|assign| { - let col_name: &Ident = assign - .id + let cols = match &assign.target { + AssignmentTarget::ColumnName(cols) => cols, + _ => plan_err!("Tuples are not supported")?, + }; + let col_name: &Ident = cols + .0 .iter() .last() .ok_or_else(|| plan_datafusion_err!("Empty column id"))?; diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs index 7cbe34825c50..06b4d4a710a3 100644 --- a/datafusion/sql/src/unparser/ast.rs +++ b/datafusion/sql/src/unparser/ast.rs @@ -93,6 +93,8 @@ impl QueryBuilder { fetch: self.fetch.clone(), locks: self.locks.clone(), for_clause: self.for_clause.clone(), + settings: None, + format_clause: None, }) } fn create_empty() -> Self { @@ -234,6 +236,7 @@ impl SelectBuilder { value_table_mode: self.value_table_mode, connect_by: None, window_before_qualify: false, + prewhere: None, }) } fn create_empty() -> Self { @@ -245,7 +248,7 @@ impl SelectBuilder { from: Default::default(), lateral_views: Default::default(), selection: Default::default(), - group_by: Some(ast::GroupByExpr::Expressions(Vec::new())), + group_by: Some(ast::GroupByExpr::Expressions(Vec::new(), Vec::new())), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index eb149c819c8b..eaca6b9b1abb 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -175,6 +175,7 @@ impl Unparser<'_> { null_treatment: None, over: None, within_group: vec![], + parameters: ast::FunctionArguments::None, })) } Expr::Between(Between { @@ -305,6 +306,7 @@ impl Unparser<'_> { null_treatment: None, over, within_group: vec![], + parameters: ast::FunctionArguments::None, })) } Expr::SimilarTo(Like { @@ -350,6 +352,7 @@ impl Unparser<'_> { null_treatment: None, over: None, within_group: vec![], + parameters: ast::FunctionArguments::None, })) } Expr::ScalarSubquery(subq) => { diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 41a8c968841b..7a653f80be08 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -172,6 +172,7 @@ impl Unparser<'_> { .iter() .map(|expr| self.expr_to_sql(expr)) .collect::>>()?, + vec![], )); } Some(AggVariant::Window(window)) => { diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index aca0d040bb8d..e34e7e20a0f3 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -3627,7 +3627,7 @@ fn test_prepare_statement_to_plan_panic_prepare_wrong_syntax() { let sql = "PREPARE AS SELECT id, age FROM person WHERE age = $foo"; assert_eq!( logical_plan(sql).unwrap_err().strip_backtrace(), - "SQL error: ParserError(\"Expected AS, found: SELECT\")" + "SQL error: ParserError(\"Expected: AS, found: SELECT\")" ) } @@ -3668,7 +3668,7 @@ fn test_non_prepare_statement_should_infer_types() { #[test] #[should_panic( - expected = "value: SQL(ParserError(\"Expected [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: $1\"" + expected = "value: SQL(ParserError(\"Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: $1\"" )] fn test_prepare_statement_to_plan_panic_is_param() { let sql = "PREPARE my_plan(INT) AS SELECT id, age FROM person WHERE age is $1"; @@ -4347,7 +4347,7 @@ fn test_parse_escaped_string_literal_value() { let sql = r"SELECT character_length(E'\000') AS len"; assert_eq!( logical_plan(sql).unwrap_err().strip_backtrace(), - "SQL error: TokenizerError(\"Unterminated encoded string literal at Line: 1, Column 25\")" + "SQL error: TokenizerError(\"Unterminated encoded string literal at Line: 1, Column: 25\")" ) } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 6fafc0a74110..f15cf8accbbe 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -3634,7 +3634,7 @@ DROP TABLE min_bool; # Min_Max End # ################# -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) create table bool_aggregate_functions ( c1 boolean not null, c2 boolean not null, @@ -3651,28 +3651,20 @@ as values (true, true, false, false, null, true, false, null); # query_bool_and -query BBBBBBBB +query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found SELECT bool_and(c1), bool_and(c2), bool_and(c3), bool_and(c4), bool_and(c5), bool_and(c6), bool_and(c7), bool_and(c8) FROM bool_aggregate_functions ----- -true false false false false true false NULL # query_bool_and_distinct -query BBBBBBBB +query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found SELECT bool_and(distinct c1), bool_and(distinct c2), bool_and(distinct c3), bool_and(distinct c4), bool_and(distinct c5), bool_and(distinct c6), bool_and(distinct c7), bool_and(distinct c8) FROM bool_aggregate_functions ----- -true false false false false true false NULL # query_bool_or -query BBBBBBBB +query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found SELECT bool_or(c1), bool_or(c2), bool_or(c3), bool_or(c4), bool_or(c5), bool_or(c6), bool_or(c7), bool_or(c8) FROM bool_aggregate_functions ----- -true true true false true true false NULL # query_bool_or_distinct -query BBBBBBBB +query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found SELECT bool_or(distinct c1), bool_or(distinct c2), bool_or(distinct c3), bool_or(distinct c4), bool_or(distinct c5), bool_or(distinct c6), bool_or(distinct c7), bool_or(distinct c8) FROM bool_aggregate_functions ----- -true true true false true true false NULL # All supported timestamp types diff --git a/datafusion/sqllogictest/test_files/arrow_typeof.slt b/datafusion/sqllogictest/test_files/arrow_typeof.slt index ab4ff9e2ce92..448706744305 100644 --- a/datafusion/sqllogictest/test_files/arrow_typeof.slt +++ b/datafusion/sqllogictest/test_files/arrow_typeof.slt @@ -430,5 +430,5 @@ select arrow_cast('MyAwesomeString', 'Utf8View'), arrow_typeof(arrow_cast('MyAwe MyAwesomeString Utf8View # Fails until we update arrow-rs with support for https://github.com/apache/arrow-rs/pull/5894 -query error DataFusion error: SQL error: ParserError\("Expected an SQL statement, found: arrow_cast"\) +query error DataFusion error: SQL error: ParserError\("Expected: an SQL statement, found: arrow_cast"\) arrow_cast('MyAwesomeString', 'BinaryView'), arrow_typeof(arrow_cast('MyAwesomeString', 'BinaryView')) diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.slt index 17b0e774d9cb..5a0b4b0975b3 100644 --- a/datafusion/sqllogictest/test_files/coalesce.slt +++ b/datafusion/sqllogictest/test_files/coalesce.slt @@ -358,7 +358,7 @@ SELECT COALESCE(c1, c2, '-1') FROM test; statement ok drop table test -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test( c1 BIGINT, c2 BIGINT, @@ -369,21 +369,14 @@ CREATE TABLE test( (NULL, NULL); # coalesce sum with default value -query I +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT SUM(COALESCE(c1, c2, 0)) FROM test ----- -4 # coalesce mul with default value -query I +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT COALESCE(c1 * c2, 0) FROM test ----- -2 -0 -0 -0 -statement ok +statement error DataFusion error: Execution error: Table 'test' doesn't exist\. drop table test # coalesce date32 diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt index 21c34bc25cee..6a6ab15a065d 100644 --- a/datafusion/sqllogictest/test_files/copy.slt +++ b/datafusion/sqllogictest/test_files/copy.slt @@ -600,7 +600,7 @@ query error DataFusion error: Invalid or Unsupported Configuration: Config value COPY source_table to 'test_files/scratch/copy/table.json' STORED AS JSON OPTIONS ('format.row_group_size' 55); # Incomplete statement -query error DataFusion error: SQL error: ParserError\("Expected \), found: EOF"\) +query error DataFusion error: SQL error: ParserError\("Expected: \), found: EOF"\) COPY (select col2, sum(col1) from source_table # Copy from table with non literal @@ -609,4 +609,4 @@ COPY source_table to '/tmp/table.parquet' (row_group_size 55 + 102); # Copy using execution.keep_partition_by_columns with an invalid value query error DataFusion error: Invalid or Unsupported Configuration: provided value for 'execution.keep_partition_by_columns' was not recognized: "invalid_value" -COPY source_table to '/tmp/table.parquet' OPTIONS (execution.keep_partition_by_columns invalid_value); \ No newline at end of file +COPY source_table to '/tmp/table.parquet' OPTIONS (execution.keep_partition_by_columns invalid_value); diff --git a/datafusion/sqllogictest/test_files/create_external_table.slt b/datafusion/sqllogictest/test_files/create_external_table.slt index 607c909fd63d..e42d14e101f1 100644 --- a/datafusion/sqllogictest/test_files/create_external_table.slt +++ b/datafusion/sqllogictest/test_files/create_external_table.slt @@ -33,23 +33,23 @@ statement error DataFusion error: SQL error: ParserError\("Missing LOCATION clau CREATE EXTERNAL TABLE t STORED AS CSV # Option value is missing -statement error DataFusion error: SQL error: ParserError\("Expected string or numeric value, found: \)"\) +statement error DataFusion error: SQL error: ParserError\("Expected: string or numeric value, found: \)"\) CREATE EXTERNAL TABLE t STORED AS x OPTIONS ('k1' 'v1', k2 v2, k3) LOCATION 'blahblah' # Missing `(` in WITH ORDER clause -statement error DataFusion error: SQL error: ParserError\("Expected \(, found: c1"\) +statement error DataFusion error: SQL error: ParserError\("Expected: \(, found: c1"\) CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV WITH ORDER c1 LOCATION 'foo.csv' # Missing `)` in WITH ORDER clause -statement error DataFusion error: SQL error: ParserError\("Expected \), found: LOCATION"\) +statement error DataFusion error: SQL error: ParserError\("Expected: \), found: LOCATION"\) CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV WITH ORDER (c1 LOCATION 'foo.csv' # Missing `ROW` in WITH HEADER clause -statement error DataFusion error: SQL error: ParserError\("Expected ROW, found: LOCATION"\) +statement error DataFusion error: SQL error: ParserError\("Expected: ROW, found: LOCATION"\) CREATE EXTERNAL TABLE t STORED AS CSV WITH HEADER LOCATION 'abc' # Missing `BY` in PARTITIONED clause -statement error DataFusion error: SQL error: ParserError\("Expected BY, found: LOCATION"\) +statement error DataFusion error: SQL error: ParserError\("Expected: BY, found: LOCATION"\) CREATE EXTERNAL TABLE t STORED AS CSV PARTITIONED LOCATION 'abc' # Duplicate `STORED AS` clause @@ -69,11 +69,11 @@ statement error DataFusion error: SQL error: ParserError\("OPTIONS specified mor CREATE EXTERNAL TABLE t STORED AS CSV OPTIONS ('k1' 'v1', 'k2' 'v2') OPTIONS ('k3' 'v3') LOCATION 'foo.csv' # With typo error -statement error DataFusion error: SQL error: ParserError\("Expected HEADER, found: HEAD"\) +statement error DataFusion error: SQL error: ParserError\("Expected: HEADER, found: HEAD"\) CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV WITH HEAD ROW LOCATION 'foo.csv'; # Missing `anything` in WITH clause -statement error DataFusion error: SQL error: ParserError\("Expected HEADER, found: LOCATION"\) +statement error DataFusion error: SQL error: ParserError\("Expected: HEADER, found: LOCATION"\) CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV WITH LOCATION 'foo.csv'; # Unrecognized random clause diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.slt index a8a689cbb8b5..d1cdbaf705c2 100644 --- a/datafusion/sqllogictest/test_files/csv_files.slt +++ b/datafusion/sqllogictest/test_files/csv_files.slt @@ -164,7 +164,7 @@ physical_plan # ensure that correct quote character is used when writing to csv -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE table_with_necessary_quoting ( int_col INT, string_col TEXT, @@ -175,14 +175,12 @@ CREATE TABLE table_with_necessary_quoting ( (4, 'h|h|h'); # quote is required because `|` is delimiter and part of the data -query IT +query error DataFusion error: Error during planning: table 'datafusion\.public\.table_with_necessary_quoting' not found COPY table_with_necessary_quoting TO 'test_files/scratch/csv_files/table_with_necessary_quoting.csv' STORED AS csv OPTIONS ('format.quote' '~', 'format.delimiter' '|', 'format.has_header' 'true'); ----- -4 # read the stored csv file with quote character statement ok @@ -195,13 +193,8 @@ OPTIONS ('format.quote' '~', 'format.delimiter' '|', 'format.has_header' 'true'); -query TT +query error DataFusion error: Object Store error: Object at location /home/mabdeen/dev/arrow\-datafusion/datafusion/sqllogictest/test_files/scratch/csv_files/table_with_necessary_quoting\.csv not found: No such file or directory \(os error 2\) select * from stored_table_with_necessary_quoting; ----- -1 e|e|e -2 f|f|f -3 g|g|g -4 h|h|h # Read CSV file with comments statement ok diff --git a/datafusion/sqllogictest/test_files/encoding.slt b/datafusion/sqllogictest/test_files/encoding.slt index 626af88aa9b8..b890401c6bfd 100644 --- a/datafusion/sqllogictest/test_files/encoding.slt +++ b/datafusion/sqllogictest/test_files/encoding.slt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test( num INT, bin_field BYTEA, @@ -31,43 +31,27 @@ CREATE TABLE test( query error DataFusion error: Error during planning: The encode function can only accept utf8 or binary\. select encode(12, 'hex') -query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found select encode(bin_field, 'non_encoding') from test; query error DataFusion error: Error during planning: The decode function can only accept utf8 or binary\. select decode(12, 'hex') -query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found select decode(hex_field, 'non_encoding') from test; query error select to_hex(hex_field) from test; # Arrays tests -query T +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT encode(bin_field, 'hex') FROM test ORDER BY num; ----- -616263 -717765717765 -NULL -query T +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT arrow_cast(decode(base64_field, 'base64'), 'Utf8') FROM test ORDER BY num; ----- -abc -qweqwe -NULL -query T +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT arrow_cast(decode(hex_field, 'hex'), 'Utf8') FROM test ORDER BY num; ----- -abc -qweqwe -NULL -query T +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found select to_hex(num) from test ORDER BY num; ----- -0 -1 -2 diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 4e8f3b59a650..6d3a205b4923 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -2351,7 +2351,7 @@ NULL # query_binary_eq -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t_source( column1 String, column2 String, @@ -2365,7 +2365,7 @@ CREATE TABLE t_source( ('three', 'three', 'three', ''); -statement ok +statement error DataFusion error: Error during planning: table 'datafusion\.public\.t_source' not found CREATE TABLE test as select arrow_cast(column1, 'Binary') as c1, @@ -2374,14 +2374,8 @@ select arrow_cast(column4, 'LargeBinary') as c4 from t_source; -query BBBB +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT sha256(c1)=digest('one', 'sha256'), sha256(c2)=sha256('two'), digest(c1, 'blake2b')=digest(c3, 'blake2b'), c2=c4 FROM test; ----- -true false true true -false true false true -NULL NULL NULL NULL -false false true true -false false true false ############# diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt index 04a1fcc78fe7..dd4a3e602fd5 100644 --- a/datafusion/sqllogictest/test_files/group_by.slt +++ b/datafusion/sqllogictest/test_files/group_by.slt @@ -4486,7 +4486,7 @@ LIMIT 5 1 TUR 4 2022-01-03T10:00:00 TRY 100 # Create a table with timestamp data -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE src_table ( t1 TIMESTAMP, c2 INT, @@ -4503,33 +4503,25 @@ CREATE TABLE src_table ( ('2020-12-19T00:00:00.00Z', 9); # Use src_table to create a partitioned file -query PI +query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/0.csv' STORED AS CSV; ----- -10 -query PI +query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/1.csv' STORED AS CSV; ----- -10 -query PI +query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/2.csv' STORED AS CSV; ----- -10 -query PI +query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/3.csv' STORED AS CSV; ----- -10 # Create a table from the generated CSV files: statement ok @@ -4542,26 +4534,18 @@ LOCATION 'test_files/scratch/group_by/timestamp_table' OPTIONS ('format.has_header' 'true'); # Group By using date_trunc -query PI rowsort +query error DataFusion error: Object Store error: Object at location /home/mabdeen/dev/arrow\-datafusion/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table not found: No such file or directory \(os error 2\) SELECT date_trunc('week', t1) as week, sum(c2) FROM timestamp_table GROUP BY date_trunc('week', t1) ----- -2020-12-07T00:00:00 24 -2020-12-14T00:00:00 156 # GROUP BY using LIMIT -query IP +query error DataFusion error: Object Store error: Object at location /home/mabdeen/dev/arrow\-datafusion/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table not found: No such file or directory \(os error 2\) SELECT c2, MAX(t1) FROM timestamp_table GROUP BY c2 ORDER BY MAX(t1) DESC LIMIT 4; ----- -9 2020-12-19T00:00:00 -8 2020-12-18T00:00:00 -7 2020-12-17T00:00:00 -6 2020-12-16T00:00:00 # Explain the GROUP BY with LIMIT to ensure the plan contains `lim=[4]` query TT @@ -4577,19 +4561,9 @@ logical_plan 02)--Sort: MAX(timestamp_table.t1) DESC NULLS FIRST, fetch=4 03)----Aggregate: groupBy=[[timestamp_table.c2]], aggr=[[MAX(timestamp_table.t1)]] 04)------TableScan: timestamp_table projection=[t1, c2] -physical_plan -01)GlobalLimitExec: skip=0, fetch=4 -02)--SortPreservingMergeExec: [MAX(timestamp_table.t1)@1 DESC], fetch=4 -03)----SortExec: TopK(fetch=4), expr=[MAX(timestamp_table.t1)@1 DESC], preserve_partitioning=[true] -04)------AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[MAX(timestamp_table.t1)], lim=[4] -05)--------CoalesceBatchesExec: target_batch_size=2 -06)----------RepartitionExec: partitioning=Hash([c2@0], 8), input_partitions=8 -07)------------AggregateExec: mode=Partial, gby=[c2@1 as c2], aggr=[MAX(timestamp_table.t1)], lim=[4] -08)--------------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=4 -09)----------------CsvExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/3.csv]]}, projection=[t1, c2], has_header=true # Clean up -statement ok +statement error DataFusion error: Execution error: Table 'src_table' doesn't exist\. DROP TABLE src_table; statement ok diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt index df66bffab8e8..b9897f81a107 100644 --- a/datafusion/sqllogictest/test_files/joins.slt +++ b/datafusion/sqllogictest/test_files/joins.slt @@ -3844,7 +3844,7 @@ EXPLAIN SELECT * FROM ( ---- logical_plan EmptyRelation -# Left ANTI join with empty right table +# Left ANTI join with empty right table query TT EXPLAIN SELECT * FROM ( SELECT 1 as a @@ -3855,7 +3855,7 @@ logical_plan 02)--Projection: Int64(1) AS a 03)----EmptyRelation -# Right ANTI join with empty left table +# Right ANTI join with empty left table query TT EXPLAIN SELECT * FROM ( SELECT 1 as a WHERE 1=0 @@ -4043,4 +4043,3 @@ physical_plan 03)----MemoryExec: partitions=1, partition_sizes=[1] 04)----SortExec: expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false] 05)------MemoryExec: partitions=1, partition_sizes=[1] - diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index 573441ab4401..2b03c1b075cf 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -271,7 +271,7 @@ statement ok drop table test_nullable_integer -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test_non_nullable_integer( c1 TINYINT NOT NULL, c2 SMALLINT NOT NULL, @@ -283,69 +283,65 @@ CREATE TABLE test_non_nullable_integer( c8 BIGINT UNSIGNED NOT NULL, ); -query I +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found INSERT INTO test_non_nullable_integer VALUES(1, 1, 1, 1, 1, 1, 1, 1) ----- -1 -query IIIIIIII rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found select c1*0, c2*0, c3*0, c4*0, c5*0, c6*0, c7*0, c8*0 from test_non_nullable_integer ----- -0 0 0 0 0 0 0 0 -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c1/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c2/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c3/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c4/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c5/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c6/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c7/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c8/0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c1%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c2%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c3%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c4%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c5%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c6%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c7%0 FROM test_non_nullable_integer -query error DataFusion error: Arrow error: Divide by zero error +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found SELECT c8%0 FROM test_non_nullable_integer -statement ok +statement error DataFusion error: Execution error: Table 'test_non_nullable_integer' doesn't exist\. drop table test_non_nullable_integer -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test_nullable_float( c1 float, c2 double, @@ -356,110 +352,56 @@ CREATE TABLE test_nullable_float( (0., 0.), ('NaN'::double, 'NaN'::double); -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found SELECT c1*0, c2*0 FROM test_nullable_float ----- -0 0 -0 0 -0 0 -NULL NULL -NaN NaN -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found SELECT c1/0, c2/0 FROM test_nullable_float ----- --Infinity -Infinity -Infinity Infinity -NULL NULL -NaN NaN -NaN NaN -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found SELECT c1%0, c2%0 FROM test_nullable_float ----- -NULL NULL -NaN NaN -NaN NaN -NaN NaN -NaN NaN -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found SELECT c1%1, c2%1 FROM test_nullable_float ----- -0 0 -0 0 -0 0 -NULL NULL -NaN NaN # abs: return type -query TT rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found SELECT arrow_typeof(abs(c1)), arrow_typeof(abs(c2)) FROM test_nullable_float limit 1 ----- -Float32 Float64 # abs: floats -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found SELECT abs(c1), abs(c2) from test_nullable_float ----- -0 0 -1 1 -1 1 -NULL NULL -NaN NaN -statement ok +statement error DataFusion error: Execution error: Table 'test_nullable_float' doesn't exist\. drop table test_nullable_float -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test_non_nullable_float( c1 float NOT NULL, c2 double NOT NULL, ); -query I +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found INSERT INTO test_non_nullable_float VALUES (-1.0, -1.0), (1.0, 1.0), (0., 0.), ('NaN'::double, 'NaN'::double) ----- -4 -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found SELECT c1*0, c2*0 FROM test_non_nullable_float ----- -0 0 -0 0 -0 0 -NaN NaN -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found SELECT c1/0, c2/0 FROM test_non_nullable_float ----- --Infinity -Infinity -Infinity Infinity -NaN NaN -NaN NaN -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found SELECT c1%0, c2%0 FROM test_non_nullable_float ----- -NaN NaN -NaN NaN -NaN NaN -NaN NaN -query RR rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found SELECT c1%1, c2%1 FROM test_non_nullable_float ----- -0 0 -0 0 -0 0 -NaN NaN -statement ok +statement error DataFusion error: Execution error: Table 'test_non_nullable_float' doesn't exist\. drop table test_non_nullable_float diff --git a/datafusion/sqllogictest/test_files/misc.slt b/datafusion/sqllogictest/test_files/misc.slt index 66606df83480..9f4710eb9bcc 100644 --- a/datafusion/sqllogictest/test_files/misc.slt +++ b/datafusion/sqllogictest/test_files/misc.slt @@ -37,4 +37,4 @@ select 1 where NULL and 1 = 1 query I select 1 where NULL or 1 = 1 ---- -1 \ No newline at end of file +1 diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.slt index ffaae7204eca..4ecdecf5f03a 100644 --- a/datafusion/sqllogictest/test_files/predicates.slt +++ b/datafusion/sqllogictest/test_files/predicates.slt @@ -584,7 +584,7 @@ DROP TABLE data_index_bloom_encoding_stats; # String coercion ######## -statement error DataFusion error: SQL error: ParserError\("Expected a data type name, found: ,"\) +statement error DataFusion error: SQL error: ParserError\("Expected: a data type name, found: ,"\) CREATE TABLE t(vendor_id_utf8, vendor_id_dict) AS VALUES (arrow_cast('124', 'Utf8'), arrow_cast('124', 'Dictionary(Int16, Utf8)')), @@ -686,7 +686,7 @@ physical_plan ######## # TPCH Q19 - Pull predicates to inner join (simplified) ######## -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE IF NOT EXISTS partsupp ( ps_partkey BIGINT, ps_suppkey BIGINT, @@ -696,7 +696,7 @@ CREATE TABLE IF NOT EXISTS partsupp ( ) AS VALUES (63700, 7311, 100, 993.49, 'ven ideas. quickly even packages print. pending multipliers must have to are fluff'); -query IRRI +query error DataFusion error: Error during planning: table 'datafusion\.public\.partsupp' not found SELECT p_partkey, sum(l_extendedprice), @@ -719,11 +719,9 @@ OR AND ps_partkey = p_partkey ) GROUP BY p_partkey; ----- -63700 13309.6 0.1 1 -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.partsupp' not found EXPLAIN SELECT p_partkey, sum(l_extendedprice), @@ -746,38 +744,6 @@ OR AND ps_partkey = p_partkey ) GROUP BY p_partkey; ----- -logical_plan -01)Aggregate: groupBy=[[part.p_partkey]], aggr=[[sum(lineitem.l_extendedprice), avg(lineitem.l_discount), count(DISTINCT partsupp.ps_suppkey)]] -02)--Projection: lineitem.l_extendedprice, lineitem.l_discount, part.p_partkey, partsupp.ps_suppkey -03)----Inner Join: part.p_partkey = partsupp.ps_partkey -04)------Projection: lineitem.l_extendedprice, lineitem.l_discount, part.p_partkey -05)--------Inner Join: lineitem.l_partkey = part.p_partkey -06)----------TableScan: lineitem projection=[l_partkey, l_extendedprice, l_discount] -07)----------Projection: part.p_partkey -08)------------Filter: part.p_brand = Utf8("Brand#12") OR part.p_brand = Utf8("Brand#23") -09)--------------TableScan: part projection=[p_partkey, p_brand], partial_filters=[part.p_brand = Utf8("Brand#12") OR part.p_brand = Utf8("Brand#23")] -10)------TableScan: partsupp projection=[ps_partkey, ps_suppkey] -physical_plan -01)AggregateExec: mode=SinglePartitioned, gby=[p_partkey@2 as p_partkey], aggr=[sum(lineitem.l_extendedprice), avg(lineitem.l_discount), count(DISTINCT partsupp.ps_suppkey)] -02)--CoalesceBatchesExec: target_batch_size=8192 -03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@2, ps_partkey@0)], projection=[l_extendedprice@0, l_discount@1, p_partkey@2, ps_suppkey@4] -04)------CoalesceBatchesExec: target_batch_size=8192 -05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_extendedprice@1, l_discount@2, p_partkey@3] -06)----------CoalesceBatchesExec: target_batch_size=8192 -07)------------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4 -08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -09)----------------CsvExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/lineitem.csv]]}, projection=[l_partkey, l_extendedprice, l_discount], has_header=true -10)----------CoalesceBatchesExec: target_batch_size=8192 -11)------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4 -12)--------------ProjectionExec: expr=[p_partkey@0 as p_partkey] -13)----------------CoalesceBatchesExec: target_batch_size=8192 -14)------------------FilterExec: p_brand@1 = Brand#12 OR p_brand@1 = Brand#23 -15)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -16)----------------------CsvExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/part.csv]]}, projection=[p_partkey, p_brand], has_header=true -17)------CoalesceBatchesExec: target_batch_size=8192 -18)--------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=1 -19)----------MemoryExec: partitions=1, partition_sizes=[1] # Inlist simplification diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 85ac5b0c242d..376350d530de 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -1575,7 +1575,7 @@ SELECT CASE WHEN 'cpu' != 'cpu' THEN true ELSE false END ---- false -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t1( a boolean, b boolean, @@ -1592,67 +1592,27 @@ CREATE TABLE t1( ; # csv query boolean eq neq -query BBBBBB rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found SELECT a, b, a = b as eq, b = true as eq_scalar, a != b as neq, a != true as neq_scalar FROM t1 ----- -NULL NULL NULL NULL NULL NULL -NULL false NULL false NULL NULL -NULL true NULL true NULL NULL -false NULL NULL NULL NULL true -false NULL NULL NULL NULL true -false true false true true true -true NULL NULL NULL NULL false -true false false false true false -true true true true false false # csv query boolean lt lt eq -query BBBBBB rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found SELECT a, b, a < b as lt, b = true as lt_scalar, a <= b as lt_eq, a <= true as lt_eq_scalar FROM t1 ----- -NULL NULL NULL NULL NULL NULL -NULL false NULL false NULL NULL -NULL true NULL true NULL NULL -false NULL NULL NULL NULL true -false NULL NULL NULL NULL true -false true true true true true -true NULL NULL NULL NULL true -true false false false false true -true true false true true true # csv query boolean gt gt eq -query BBBBBB rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found SELECT a, b, a > b as gt, b = true as gt_scalar, a >= b as gt_eq, a >= true as gt_eq_scalar FROM t1 ----- -NULL NULL NULL NULL NULL NULL -NULL false NULL false NULL NULL -NULL true NULL true NULL NULL -false NULL NULL NULL NULL false -false NULL NULL NULL NULL false -false true false true false false -true NULL NULL NULL NULL true -true false true false true true -true true false true true true # csv query boolean distinct from -query BBBBBB rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found SELECT a, b, a is distinct from b as df, b is distinct from true as df_scalar, a is not distinct from b as ndf, a is not distinct from true as ndf_scalar FROM t1 ----- -NULL NULL false true true false -NULL false true true false false -NULL true true false false false -false NULL true true false false -false NULL true true false false -false true true false false false -true NULL true true false true -true false true true false true -true true false false true true -statement ok +statement error DataFusion error: Execution error: Table 't1' doesn't exist\. drop table t1 # like nlike with null lt diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index 95f67245a981..d7c8a9314e9e 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -97,7 +97,7 @@ One 1 Three 1 -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test ( c1 BIGINT NOT NULL, c2 BIGINT NOT NULL, @@ -145,43 +145,16 @@ CREATE TABLE test ( # parallel_query_with_filter -query II +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT c1, c2 FROM test WHERE c1 > 0 AND c1 < 3; ----- -1 1 -1 10 -1 2 -1 3 -1 4 -1 5 -1 6 -1 7 -1 8 -1 9 -2 1 -2 10 -2 2 -2 3 -2 4 -2 5 -2 6 -2 7 -2 8 -2 9 ###### # Boolean literal ###### -query IB +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT c1, c3 FROM test WHERE c1 > 2 AND c3 = true; ----- -3 true -3 true -3 true -3 true -3 true -statement ok +statement error DataFusion error: Execution error: Table 'test' doesn't exist\. drop table test; ###### @@ -336,13 +309,13 @@ three 1 NULL 1 # select_values_list -statement error DataFusion error: SQL error: ParserError\("Expected \(, found: EOF"\) +statement error DataFusion error: SQL error: ParserError\("Expected: \(, found: EOF"\) VALUES -statement error DataFusion error: SQL error: ParserError\("Expected an expression:, found: \)"\) +statement error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \)"\) VALUES () -statement error DataFusion error: SQL error: ParserError\("Expected an expression:, found: \)"\) +statement error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \)"\) VALUES (1),() statement error DataFusion error: Error during planning: Inconsistent data length across values list: got 2 values in row 1 but expected 1 diff --git a/datafusion/sqllogictest/test_files/strings.slt b/datafusion/sqllogictest/test_files/strings.slt index 3cd6c339b44f..0d632db013ff 100644 --- a/datafusion/sqllogictest/test_files/strings.slt +++ b/datafusion/sqllogictest/test_files/strings.slt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE test( s TEXT, ) as VALUES @@ -32,52 +32,24 @@ CREATE TABLE test( ; # LIKE -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT s FROM test WHERE s LIKE 'p1%'; ----- -p1 -p1e1 -p1m1e1 -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT s FROM test WHERE s LIKE '%m1%'; ----- -P1m1e1 -p1m1e1 -p2m1e1 # NOT LIKE -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT s FROM test WHERE s NOT LIKE 'p1%'; ----- -P1 -P1e1 -P1m1e1 -e1 -p2 -p2e1 -p2m1e1 # ILIKE -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT s FROM test WHERE s ILIKE 'p1%'; ----- -P1 -P1e1 -P1m1e1 -p1 -p1e1 -p1m1e1 # NOT ILIKE -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found SELECT s FROM test WHERE s NOT ILIKE 'p1%'; ----- -e1 -p2 -p2e1 -p2m1e1 ## VARCHAR with length support diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt index fd6e25ea749d..c46da32d69df 100644 --- a/datafusion/sqllogictest/test_files/struct.slt +++ b/datafusion/sqllogictest/test_files/struct.slt @@ -19,7 +19,7 @@ ## Struct Expressions Tests ############# -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE values( a INT, b FLOAT, @@ -65,12 +65,8 @@ select struct(1, 3.14, 'h')['c0'], struct(3, 2.55, 'b')['c1'], struct(2, 6.43, ' 1 2.55 a # struct[i] with columns -query R +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select struct(a, b, c)['c1'] from values; ----- -1.1 -2.2 -3.3 # struct scalar function #1 query ? @@ -91,32 +87,17 @@ select struct(1, 3.14 as name1, 'e', true); {c0: 1, name1: 3.14, c2: e, c3: true} # struct scalar function with columns #1 -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select struct(a, b, c) from values; ----- -{c0: 1, c1: 1.1, c2: a} -{c0: 2, c1: 2.2, c2: b} -{c0: 3, c1: 3.3, c2: c} # struct scalar function with columns and scalars -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select struct(a, 'foo') from values; ----- -{c0: 1, c1: foo} -{c0: 2, c1: foo} -{c0: 3, c1: foo} # explain struct scalar function with columns #1 -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found explain select struct(a, b, c) from values; ----- -logical_plan -01)Projection: struct(values.a, values.b, values.c) -02)--TableScan: values projection=[a, b, c] -physical_plan -01)ProjectionExec: expr=[struct(a@0, b@1, c@2) as struct(values.a,values.b,values.c)] -02)--MemoryExec: partitions=1, partition_sizes=[1] # error on 0 arguments query error @@ -131,7 +112,7 @@ query error DataFusion error: Execution error: named_struct requires an even num select named_struct(1); # error on odd number of arguments #3 -query error DataFusion error: Execution error: named_struct requires an even number of arguments, got 1 instead +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct(values.a) from values; # error on odd number of arguments #4 @@ -147,66 +128,38 @@ query error DataFusion error: Execution error: named_struct even arguments must select named_struct('corret', 1, 0, 'wrong'); # error on even argument not a string literal #3 -query error DataFusion error: Execution error: named_struct even arguments must be string literals, got values\.a instead at position 0 +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct(values.a, 'a') from values; # error on even argument not a string literal #4 -query error DataFusion error: Execution error: named_struct even arguments must be string literals, got values\.c instead at position 0 +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct(values.c, 'c') from values; # named_struct with mixed scalar and array values #1 -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct('scalar', 27, 'array', values.a, 'null', NULL) from values; ----- -{scalar: 27, array: 1, null: } -{scalar: 27, array: 2, null: } -{scalar: 27, array: 3, null: } -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select {'scalar': 27, 'array': values.a, 'null': NULL} from values; ----- -{scalar: 27, array: 1, null: } -{scalar: 27, array: 2, null: } -{scalar: 27, array: 3, null: } # named_struct with mixed scalar and array values #2 -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct('array', values.a, 'scalar', 27, 'null', NULL) from values; ----- -{array: 1, scalar: 27, null: } -{array: 2, scalar: 27, null: } -{array: 3, scalar: 27, null: } -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select {'array': values.a, 'scalar': 27, 'null': NULL} from values; ----- -{array: 1, scalar: 27, null: } -{array: 2, scalar: 27, null: } -{array: 3, scalar: 27, null: } # named_struct with mixed scalar and array values #3 -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct('null', NULL, 'array', values.a, 'scalar', 27) from values; ----- -{null: , array: 1, scalar: 27} -{null: , array: 2, scalar: 27} -{null: , array: 3, scalar: 27} # named_struct with mixed scalar and array values #4 -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct('null_array', values.n, 'array', values.a, 'scalar', 27, 'null', NULL) from values; ----- -{null_array: , array: 1, scalar: 27, null: } -{null_array: , array: 2, scalar: 27, null: } -{null_array: , array: 3, scalar: 27, null: } # named_struct arrays only -query ? +query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found select named_struct('field_a', a, 'field_b', b) from values; ----- -{field_a: 1, field_b: 1.1} -{field_a: 2, field_b: 2.2} -{field_a: 3, field_b: 3.3} # named_struct scalars only query ? @@ -214,7 +167,7 @@ select named_struct('field_a', 1, 'field_b', 2); ---- {field_a: 1, field_b: 2} -statement ok +statement error DataFusion error: Execution error: Table 'values' doesn't exist\. drop table values; query T diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index 5ede68a42aae..26fdc4bebd9c 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -19,7 +19,7 @@ ## UNION Tests ########## -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t1( id INT, name TEXT, @@ -29,7 +29,7 @@ CREATE TABLE t1( (3, 'Alice') ; -statement ok +statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t2( id TINYINT, name TEXT, @@ -40,7 +40,7 @@ CREATE TABLE t2( ; # union with EXCEPT(JOIN) -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found ( SELECT name FROM t1 EXCEPT @@ -52,12 +52,9 @@ UNION ALL EXCEPT SELECT name FROM t1 ) ----- -Alice -John # union with type coercion -query IT rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found ( SELECT * FROM t1 EXCEPT @@ -69,9 +66,6 @@ UNION ALL EXCEPT SELECT * FROM t1 ) ----- -3 Alice -3 John # union all query I rowsort @@ -82,10 +76,8 @@ SELECT 2 as x 1 2 -query I +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found select count(*) from (select id from t1 union all select id from t2) ----- -6 # csv_union_all statement ok @@ -178,7 +170,7 @@ SELECT 1 UNION SELECT 2 2 # union_with_except_input -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found ( SELECT name FROM t1 EXCEPT @@ -190,82 +182,29 @@ UNION ALL EXCEPT SELECT name FROM t1 ) ----- -Alice -John # nested_union -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found SELECT name FROM t1 UNION (SELECT name from t2 UNION SELECT name || '_new' from t2) ----- -Alex -Alex_new -Alice -Bob -Bob_new -John -John_new # should be un-nested, with a single (logical) aggregate -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found EXPLAIN SELECT name FROM t1 UNION (SELECT name from t2 UNION SELECT name || '_new' from t2) ----- -logical_plan -01)Aggregate: groupBy=[[t1.name]], aggr=[[]] -02)--Union -03)----TableScan: t1 projection=[name] -04)----TableScan: t2 projection=[name] -05)----Projection: t2.name || Utf8("_new") AS name -06)------TableScan: t2 projection=[name] -physical_plan -01)AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] -02)--CoalesceBatchesExec: target_batch_size=8192 -03)----RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 -05)--------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] -06)----------UnionExec -07)------------MemoryExec: partitions=1, partition_sizes=[1] -08)------------MemoryExec: partitions=1, partition_sizes=[1] -09)------------ProjectionExec: expr=[name@0 || _new as name] -10)--------------MemoryExec: partitions=1, partition_sizes=[1] # nested_union_all -query T rowsort +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found SELECT name FROM t1 UNION ALL (SELECT name from t2 UNION ALL SELECT name || '_new' from t2) ----- -Alex -Alex -Alex_new -Alice -Bob -Bob -Bob_new -John -John_new # Plan is unnested -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found EXPLAIN SELECT name FROM t1 UNION ALL (SELECT name from t2 UNION ALL SELECT name || '_new' from t2) ----- -logical_plan -01)Union -02)--TableScan: t1 projection=[name] -03)--TableScan: t2 projection=[name] -04)--Projection: t2.name || Utf8("_new") AS name -05)----TableScan: t2 projection=[name] -physical_plan -01)UnionExec -02)--MemoryExec: partitions=1, partition_sizes=[1] -03)--MemoryExec: partitions=1, partition_sizes=[1] -04)--ProjectionExec: expr=[name@0 || _new as name] -05)----MemoryExec: partitions=1, partition_sizes=[1] # Make sure to choose a small batch size to introduce parallelism to the plan. statement ok set datafusion.execution.batch_size = 2; # union_with_type_coercion -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found explain ( SELECT id, name FROM t1 @@ -278,52 +217,9 @@ UNION ALL EXCEPT SELECT id, name FROM t1 ) ----- -logical_plan -01)Union -02)--LeftAnti Join: t1.id = CAST(t2.id AS Int32), t1.name = t2.name -03)----Aggregate: groupBy=[[t1.id, t1.name]], aggr=[[]] -04)------TableScan: t1 projection=[id, name] -05)----TableScan: t2 projection=[id, name] -06)--Projection: CAST(t2.id AS Int32) AS id, t2.name -07)----LeftAnti Join: CAST(t2.id AS Int32) = t1.id, t2.name = t1.name -08)------Aggregate: groupBy=[[t2.id, t2.name]], aggr=[[]] -09)--------TableScan: t2 projection=[id, name] -10)------TableScan: t1 projection=[id, name] -physical_plan -01)UnionExec -02)--CoalesceBatchesExec: target_batch_size=2 -03)----HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(id@0, CAST(t2.id AS Int32)@2), (name@1, name@1)] -04)------AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] -05)--------CoalesceBatchesExec: target_batch_size=2 -06)----------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 -07)------------AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] -08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -09)----------------MemoryExec: partitions=1, partition_sizes=[1] -10)------CoalesceBatchesExec: target_batch_size=2 -11)--------RepartitionExec: partitioning=Hash([CAST(t2.id AS Int32)@2, name@1], 4), input_partitions=4 -12)----------ProjectionExec: expr=[id@0 as id, name@1 as name, CAST(id@0 AS Int32) as CAST(t2.id AS Int32)] -13)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -14)--------------MemoryExec: partitions=1, partition_sizes=[1] -15)--ProjectionExec: expr=[CAST(id@0 AS Int32) as id, name@1 as name] -16)----CoalesceBatchesExec: target_batch_size=2 -17)------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(CAST(t2.id AS Int32)@2, id@0), (name@1, name@1)], projection=[id@0, name@1] -18)--------CoalesceBatchesExec: target_batch_size=2 -19)----------RepartitionExec: partitioning=Hash([CAST(t2.id AS Int32)@2, name@1], 4), input_partitions=4 -20)------------ProjectionExec: expr=[id@0 as id, name@1 as name, CAST(id@0 AS Int32) as CAST(t2.id AS Int32)] -21)--------------AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] -22)----------------CoalesceBatchesExec: target_batch_size=2 -23)------------------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 -24)--------------------AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] -25)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -26)------------------------MemoryExec: partitions=1, partition_sizes=[1] -27)--------CoalesceBatchesExec: target_batch_size=2 -28)----------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 -29)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -30)--------------MemoryExec: partitions=1, partition_sizes=[1] - - -query IT rowsort + + +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found ( SELECT id, name FROM t1 EXCEPT @@ -335,12 +231,9 @@ UNION ALL EXCEPT SELECT id, name FROM t1 ) ----- -3 Alice -3 John # union_with_except_input -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found explain ( SELECT name FROM t1 @@ -353,43 +246,6 @@ UNION ALL EXCEPT SELECT name FROM t1 ) ----- -logical_plan -01)Union -02)--LeftAnti Join: t1.name = t2.name -03)----Aggregate: groupBy=[[t1.name]], aggr=[[]] -04)------TableScan: t1 projection=[name] -05)----TableScan: t2 projection=[name] -06)--LeftAnti Join: t2.name = t1.name -07)----Aggregate: groupBy=[[t2.name]], aggr=[[]] -08)------TableScan: t2 projection=[name] -09)----TableScan: t1 projection=[name] -physical_plan -01)InterleaveExec -02)--CoalesceBatchesExec: target_batch_size=2 -03)----HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(name@0, name@0)] -04)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] -05)--------CoalesceBatchesExec: target_batch_size=2 -06)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -07)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] -08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -09)----------------MemoryExec: partitions=1, partition_sizes=[1] -10)------CoalesceBatchesExec: target_batch_size=2 -11)--------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -12)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -13)------------MemoryExec: partitions=1, partition_sizes=[1] -14)--CoalesceBatchesExec: target_batch_size=2 -15)----HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(name@0, name@0)] -16)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] -17)--------CoalesceBatchesExec: target_batch_size=2 -18)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -19)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] -20)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -21)----------------MemoryExec: partitions=1, partition_sizes=[1] -22)------CoalesceBatchesExec: target_batch_size=2 -23)--------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -24)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -25)------------MemoryExec: partitions=1, partition_sizes=[1] # union_upcast_types query TT @@ -426,48 +282,23 @@ e 4144173353 b 4076864659 # union_with_hash_aggregate -query TT +query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found explain SELECT count(*) FROM ( SELECT distinct name FROM t1 UNION ALL SELECT distinct name FROM t2 ) GROUP BY name ----- -logical_plan -01)Projection: count(*) -02)--Aggregate: groupBy=[[t1.name]], aggr=[[count(Int64(1)) AS count(*)]] -03)----Union -04)------Aggregate: groupBy=[[t1.name]], aggr=[[]] -05)--------TableScan: t1 projection=[name] -06)------Aggregate: groupBy=[[t2.name]], aggr=[[]] -07)--------TableScan: t2 projection=[name] -physical_plan -01)ProjectionExec: expr=[count(*)@1 as count(*)] -02)--AggregateExec: mode=SinglePartitioned, gby=[name@0 as name], aggr=[count(*)] -03)----InterleaveExec -04)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] -05)--------CoalesceBatchesExec: target_batch_size=2 -06)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -07)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] -08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -09)----------------MemoryExec: partitions=1, partition_sizes=[1] -10)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] -11)--------CoalesceBatchesExec: target_batch_size=2 -12)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 -13)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] -14)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -15)----------------MemoryExec: partitions=1, partition_sizes=[1] ######## # Clean up after the test ######## -statement ok +statement error DataFusion error: Execution error: Table 't1' doesn't exist\. drop table t1; -statement ok +statement error DataFusion error: Execution error: Table 't2' doesn't exist\. drop table t2; statement ok diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.slt index 06733f7b1e40..698faf87c9b2 100644 --- a/datafusion/sqllogictest/test_files/unnest.slt +++ b/datafusion/sqllogictest/test_files/unnest.slt @@ -267,7 +267,7 @@ query error DataFusion error: Error during planning: unnest\(\) requires exactly select unnest(); ## Unnest empty expression in from clause -query error DataFusion error: SQL error: ParserError\("Expected an expression:, found: \)"\) +query error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \)"\) select * from unnest(); @@ -496,7 +496,7 @@ select unnest(column1) from (select * from (values([1,2,3]), ([4,5,6])) limit 1 5 6 -## FIXME: https://github.com/apache/datafusion/issues/11198 +## FIXME: https://github.com/apache/datafusion/issues/11198 query error DataFusion error: Error during planning: Projections require unique expression names but the expression "UNNEST\(Column\(Column \{ relation: Some\(Bare \{ table: "unnest_table" \}\), name: "column1" \}\)\)" at position 0 and "UNNEST\(Column\(Column \{ relation: Some\(Bare \{ table: "unnest_table" \}\), name: "column1" \}\)\)" at position 1 have the same name. Consider aliasing \("AS"\) one of them. select unnest(column1), unnest(column1) from unnest_table; @@ -556,4 +556,4 @@ physical_plan 05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 06)----------UnnestExec 07)------------ProjectionExec: expr=[column3@0 as unnest(recursive_unnest_table.column3), column3@0 as column3] -08)--------------MemoryExec: partitions=1, partition_sizes=[1] \ No newline at end of file +08)--------------MemoryExec: partitions=1, partition_sizes=[1] diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index 7f2e766aab91..d685ef93d5af 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -3413,7 +3413,7 @@ SELECT LIMIT 5 # window1 spec is defined multiple times -statement error DataFusion error: Error during planning: The window window1 is defined multiple times! +statement error DataFusion error: SQL error: ParserError\("Expected an expression, found: FROM"\) SELECT MAX(c12) OVER window1 as min1, FROM aggregate_test_100 From 7be2263793be7730615c52fec79ca3397eefb40f Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 19:12:55 +0300 Subject: [PATCH 02/15] clean imports and qualified imports --- datafusion/sql/src/statement.rs | 50 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index ee050381a73d..6df25086305d 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -46,18 +46,19 @@ use datafusion_expr::{ cast, col, Analyze, CreateCatalog, CreateCatalogSchema, CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody, CreateMemoryTable, CreateView, DescribeTable, DmlStatement, DropCatalogSchema, - DropFunction, DropTable, DropView, EmptyRelation, Explain, ExprSchemable, Filter, - LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, SetVariable, - Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, + DropFunction, DropTable, DropView, EmptyRelation, Explain, Expr, ExprSchemable, + Filter, LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, + SetVariable, Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, }; -use sqlparser::ast::{self, AssignmentTarget, CreateTable}; +use sqlparser::ast; use sqlparser::ast::{ - Assignment, ColumnDef, CreateTableOptions, Delete, DescribeAlias, Expr as SQLExpr, - Expr, FromTable, Ident, Insert, ObjectName, ObjectType, OneOrManyWithParens, Query, - SchemaName, SetExpr, ShowCreateObject, ShowStatementFilter, Statement, - TableConstraint, TableFactor, TableWithJoins, TransactionMode, UnaryOperator, Value, + Assignment, AssignmentTarget, ColumnDef, CreateTable, CreateTableOptions, Delete, + DescribeAlias, Expr as SQLExpr, FromTable, Ident, Insert, ObjectName, ObjectType, + OneOrManyWithParens, Query, SchemaName, SetExpr, ShowCreateObject, + ShowStatementFilter, Statement, TableConstraint, TableFactor, TableWithJoins, + TransactionMode, UnaryOperator, Value, }; use sqlparser::parser::ParserError::ParserError; @@ -954,7 +955,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { order_exprs: Vec, schema: &DFSchemaRef, planner_context: &mut PlannerContext, - ) -> Result>> { + ) -> Result>> { // Ask user to provide a schema if schema is empty. if !order_exprs.is_empty() && schema.fields().is_empty() { return plan_err!( @@ -1159,7 +1160,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { local: bool, hivevar: bool, variables: &OneOrManyWithParens, - value: Vec, + value: Vec, ) -> Result { if local { return not_impl_err!("LOCAL is not supported"); @@ -1218,7 +1219,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { fn delete_to_plan( &self, table_name: ObjectName, - predicate_expr: Option, + predicate_expr: Option, ) -> Result { // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(table_name.clone())?; @@ -1264,7 +1265,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { table: TableWithJoins, assignments: Vec, from: Option, - predicate_expr: Option, + predicate_expr: Option, ) -> Result { let (table_name, table_alias) = match &table.relation { TableFactor::Table { name, alias, .. } => (name.clone(), alias.clone()), @@ -1297,7 +1298,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { table_schema.field_with_unqualified_name(&col_name.value)?; Ok((col_name.value.clone(), assign.value.clone())) }) - .collect::>>()?; + .collect::>>()?; // Build scan, join with from table if it exists. let mut input_tables = vec![table]; @@ -1336,8 +1337,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &mut planner_context, )?; // Update placeholder's datatype to the type of the target column - if let datafusion_expr::Expr::Placeholder(placeholder) = &mut expr - { + if let Expr::Placeholder(placeholder) = &mut expr { placeholder.data_type = placeholder .data_type .take() @@ -1349,14 +1349,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { None => { // If the target table has an alias, use it to qualify the column name if let Some(alias) = &table_alias { - datafusion_expr::Expr::Column(Column::new( + Expr::Column(Column::new( Some(self.normalizer.normalize(alias.name.clone())), field.name(), )) } else { - datafusion_expr::Expr::Column(Column::from(( - qualifier, field, - ))) + Expr::Column(Column::from((qualifier, field))) } } }; @@ -1431,7 +1429,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { if let SetExpr::Values(ast::Values { rows, .. }) = (*source.body).clone() { for row in rows.iter() { for (idx, val) in row.iter().enumerate() { - if let ast::Expr::Value(Value::Placeholder(name)) = val { + if let SQLExpr::Value(Value::Placeholder(name)) = val { let name = name.replace('$', "").parse::().map_err(|_| { plan_datafusion_err!("Can't parse placeholder: {name}") @@ -1464,23 +1462,23 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { .map(|(i, value_index)| { let target_field = table_schema.field(i); let expr = match value_index { - Some(v) => datafusion_expr::Expr::Column(Column::from( - source.schema().qualified_field(v), - )) - .cast_to(target_field.data_type(), source.schema())?, + Some(v) => { + Expr::Column(Column::from(source.schema().qualified_field(v))) + .cast_to(target_field.data_type(), source.schema())? + } // The value is not specified. Fill in the default value for the column. None => table_source .get_column_default(target_field.name()) .cloned() .unwrap_or_else(|| { // If there is no default for the column, then the default is NULL - datafusion_expr::Expr::Literal(ScalarValue::Null) + Expr::Literal(ScalarValue::Null) }) .cast_to(target_field.data_type(), &DFSchema::empty())?, }; Ok(expr.alias(target_field.name())) }) - .collect::>>()?; + .collect::>>()?; let source = project(source, exprs)?; let op = if overwrite { From a36539cb3c78bd5b1b40e13749c0d12132d27b72 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 19:13:27 +0300 Subject: [PATCH 03/15] update df-cli cargo lock --- datafusion-cli/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 7da9cc427c37..e48c6b081e1a 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -3438,9 +3438,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.47.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +checksum = "749780d15ad1ee15fd74f5f84b0665560b6abb913de744c2b69155770f9601da" dependencies = [ "log", "sqlparser_derive", From 9ba27c01d9b7135e0b3431dc89642a9d5037eb00 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 19:52:35 +0300 Subject: [PATCH 04/15] fix trailing commas in slt tests --- .../sqllogictest/test_files/aggregate.slt | 48 ++-- .../sqllogictest/test_files/coalesce.slt | 30 ++- .../sqllogictest/test_files/csv_files.slt | 15 +- .../sqllogictest/test_files/encoding.slt | 32 ++- datafusion/sqllogictest/test_files/expr.slt | 130 ++++++----- .../sqllogictest/test_files/group_by.slt | 46 +++- datafusion/sqllogictest/test_files/math.slt | 216 +++++++++++------- .../sqllogictest/test_files/predicates.slt | 4 +- datafusion/sqllogictest/test_files/scalar.slt | 4 +- datafusion/sqllogictest/test_files/select.slt | 6 +- .../sqllogictest/test_files/strings.slt | 42 +++- datafusion/sqllogictest/test_files/struct.slt | 2 +- datafusion/sqllogictest/test_files/union.slt | 32 +-- datafusion/sqllogictest/test_files/window.slt | 4 +- 14 files changed, 386 insertions(+), 225 deletions(-) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index f15cf8accbbe..bc983c7f358b 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1966,7 +1966,7 @@ drop table t; # test count with largeutf8 statement ok -create table t (c string) as values +create table t (c string) as values (arrow_cast('a', 'LargeUtf8')), (arrow_cast('b', 'LargeUtf8')), (arrow_cast(null, 'LargeUtf8')), @@ -3634,7 +3634,7 @@ DROP TABLE min_bool; # Min_Max End # ################# -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok create table bool_aggregate_functions ( c1 boolean not null, c2 boolean not null, @@ -3643,7 +3643,7 @@ create table bool_aggregate_functions ( c5 boolean, c6 boolean, c7 boolean, - c8 boolean, + c8 boolean ) as values (true, true, false, false, true, true, null, null), @@ -3651,20 +3651,28 @@ as values (true, true, false, false, null, true, false, null); # query_bool_and -query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found +query BBBBBBBB SELECT bool_and(c1), bool_and(c2), bool_and(c3), bool_and(c4), bool_and(c5), bool_and(c6), bool_and(c7), bool_and(c8) FROM bool_aggregate_functions +---- +true false false false false true false NULL # query_bool_and_distinct -query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found +query BBBBBBBB SELECT bool_and(distinct c1), bool_and(distinct c2), bool_and(distinct c3), bool_and(distinct c4), bool_and(distinct c5), bool_and(distinct c6), bool_and(distinct c7), bool_and(distinct c8) FROM bool_aggregate_functions +---- +true false false false false true false NULL # query_bool_or -query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found +query BBBBBBBB SELECT bool_or(c1), bool_or(c2), bool_or(c3), bool_or(c4), bool_or(c5), bool_or(c6), bool_or(c7), bool_or(c8) FROM bool_aggregate_functions +---- +true true true false true true false NULL # query_bool_or_distinct -query error DataFusion error: Error during planning: table 'datafusion\.public\.bool_aggregate_functions' not found +query BBBBBBBB SELECT bool_or(distinct c1), bool_or(distinct c2), bool_or(distinct c3), bool_or(distinct c4), bool_or(distinct c5), bool_or(distinct c6), bool_or(distinct c7), bool_or(distinct c8) FROM bool_aggregate_functions +---- +true true true false true true false NULL # All supported timestamp types @@ -4129,7 +4137,7 @@ statement ok create table t (c1 decimal(10, 0), c2 int) as values (null, null), (null, null), (null, null); query RTIT -select +select sum(c1), arrow_typeof(sum(c1)), sum(c2), arrow_typeof(sum(c2)) from t; @@ -4763,7 +4771,7 @@ NULL NULL 3 NULL 1 4 0 8 0 # regr_*() basic tests query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4778,7 +4786,7 @@ from (values (1,2), (2,4), (3,6)); 2 0 3 1 2 4 2 8 4 query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -4796,7 +4804,7 @@ from aggregate_test_100; # regr_*() functions ignore NULLs query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4811,7 +4819,7 @@ from (values (1,NULL), (2,4), (3,6)); 2 0 2 1 2.5 5 0.5 2 1 query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4826,7 +4834,7 @@ from (values (1,NULL), (NULL,4), (3,6)); NULL NULL 1 NULL 3 6 0 0 0 query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4841,8 +4849,8 @@ from (values (1,NULL), (NULL,4), (NULL,NULL)); NULL NULL 0 NULL NULL NULL NULL NULL NULL query TRRRRRRRRR rowsort -select - column3, +select + column3, regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4866,7 +4874,7 @@ statement ok set datafusion.execution.batch_size = 1; query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -4884,7 +4892,7 @@ statement ok set datafusion.execution.batch_size = 2; query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -4902,7 +4910,7 @@ statement ok set datafusion.execution.batch_size = 3; query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -5056,7 +5064,7 @@ CREATE TABLE float_table ( # Test string_agg with largeutf8 statement ok -create table string_agg_large_utf8 (c string) as values +create table string_agg_large_utf8 (c string) as values (arrow_cast('a', 'LargeUtf8')), (arrow_cast('b', 'LargeUtf8')), (arrow_cast('c', 'LargeUtf8')) @@ -5111,7 +5119,7 @@ select count(*) from (select count(*) a, count(*) b from (select 1)); # UTF8 string matters for string to &[u8] conversion, add it to prevent regression statement ok -create table distinct_count_string_table as values +create table distinct_count_string_table as values (1, 'a', 'longstringtest_a', '台灣'), (2, 'b', 'longstringtest_b1', '日本'), (2, 'b', 'longstringtest_b2', '中國'), diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.slt index 5a0b4b0975b3..8671c49aa49e 100644 --- a/datafusion/sqllogictest/test_files/coalesce.slt +++ b/datafusion/sqllogictest/test_files/coalesce.slt @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. - # Test coalesce function query I select coalesce(1, 2, 3); @@ -247,7 +246,7 @@ statement ok create table t(c varchar) as values ('a'), (null); query TT -select +select coalesce(c, arrow_cast('b', 'Dictionary(Int32, Utf8)')), arrow_typeof(coalesce(c, arrow_cast('b', 'Dictionary(Int32, Utf8)'))) from t; @@ -260,12 +259,12 @@ drop table t; # test dict coercion with dict statement ok -create table t as values - (arrow_cast('foo', 'Dictionary(Int32, Utf8)')), +create table t as values + (arrow_cast('foo', 'Dictionary(Int32, Utf8)')), (null); query ?T -select +select coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, LargeUtf8)')), arrow_typeof(coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, LargeUtf8)'))) from t; @@ -274,7 +273,7 @@ foo Dictionary(Int64, LargeUtf8) bar Dictionary(Int64, LargeUtf8) query ?T -select +select coalesce(column1, arrow_cast('bar', 'Dictionary(Int32, LargeUtf8)')), arrow_typeof(coalesce(column1, arrow_cast('bar', 'Dictionary(Int32, LargeUtf8)'))) from t; @@ -283,7 +282,7 @@ foo Dictionary(Int32, LargeUtf8) bar Dictionary(Int32, LargeUtf8) query ?T -select +select coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, Utf8)')), arrow_typeof(coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, Utf8)'))) from t; @@ -358,10 +357,10 @@ SELECT COALESCE(c1, c2, '-1') FROM test; statement ok drop table test -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test( c1 BIGINT, - c2 BIGINT, + c2 BIGINT ) as VALUES (1, 2), (NULL, 2), @@ -369,14 +368,21 @@ CREATE TABLE test( (NULL, NULL); # coalesce sum with default value -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query I SELECT SUM(COALESCE(c1, c2, 0)) FROM test +---- +4 # coalesce mul with default value -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query I SELECT COALESCE(c1 * c2, 0) FROM test +---- +2 +0 +0 +0 -statement error DataFusion error: Execution error: Table 'test' doesn't exist\. +statement ok drop table test # coalesce date32 diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.slt index d1cdbaf705c2..ca3bebe79f27 100644 --- a/datafusion/sqllogictest/test_files/csv_files.slt +++ b/datafusion/sqllogictest/test_files/csv_files.slt @@ -164,10 +164,10 @@ physical_plan # ensure that correct quote character is used when writing to csv -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE table_with_necessary_quoting ( int_col INT, - string_col TEXT, + string_col TEXT ) AS VALUES (1, 'e|e|e'), (2, 'f|f|f'), @@ -175,12 +175,14 @@ CREATE TABLE table_with_necessary_quoting ( (4, 'h|h|h'); # quote is required because `|` is delimiter and part of the data -query error DataFusion error: Error during planning: table 'datafusion\.public\.table_with_necessary_quoting' not found +query IT COPY table_with_necessary_quoting TO 'test_files/scratch/csv_files/table_with_necessary_quoting.csv' STORED AS csv OPTIONS ('format.quote' '~', 'format.delimiter' '|', 'format.has_header' 'true'); +---- +4 # read the stored csv file with quote character statement ok @@ -193,8 +195,13 @@ OPTIONS ('format.quote' '~', 'format.delimiter' '|', 'format.has_header' 'true'); -query error DataFusion error: Object Store error: Object at location /home/mabdeen/dev/arrow\-datafusion/datafusion/sqllogictest/test_files/scratch/csv_files/table_with_necessary_quoting\.csv not found: No such file or directory \(os error 2\) +query TT select * from stored_table_with_necessary_quoting; +---- +1 e|e|e +2 f|f|f +3 g|g|g +4 h|h|h # Read CSV file with comments statement ok diff --git a/datafusion/sqllogictest/test_files/encoding.slt b/datafusion/sqllogictest/test_files/encoding.slt index b890401c6bfd..7a6ac5ca7121 100644 --- a/datafusion/sqllogictest/test_files/encoding.slt +++ b/datafusion/sqllogictest/test_files/encoding.slt @@ -15,12 +15,12 @@ # specific language governing permissions and limitations # under the License. -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test( num INT, bin_field BYTEA, base64_field TEXT, - hex_field TEXT, + hex_field TEXT ) as VALUES (0, 'abc', encode('abc', 'base64'), encode('abc', 'hex')), (1, 'qweqwe', encode('qweqwe', 'base64'), encode('qweqwe', 'hex')), @@ -31,27 +31,43 @@ CREATE TABLE test( query error DataFusion error: Error during planning: The encode function can only accept utf8 or binary\. select encode(12, 'hex') -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex select encode(bin_field, 'non_encoding') from test; query error DataFusion error: Error during planning: The decode function can only accept utf8 or binary\. select decode(12, 'hex') -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex select decode(hex_field, 'non_encoding') from test; query error select to_hex(hex_field) from test; # Arrays tests -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT encode(bin_field, 'hex') FROM test ORDER BY num; +---- +616263 +717765717765 +NULL -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT arrow_cast(decode(base64_field, 'base64'), 'Utf8') FROM test ORDER BY num; +---- +abc +qweqwe +NULL -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT arrow_cast(decode(hex_field, 'hex'), 'Utf8') FROM test ORDER BY num; +---- +abc +qweqwe +NULL -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T select to_hex(num) from test ORDER BY num; +---- +0 +1 +2 diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 6d3a205b4923..6a2d3cef09a2 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -504,7 +504,7 @@ NULL query T SELECT ltrim(' zzzytest ') ---- -zzzytest +zzzytest query T SELECT ltrim('zzzytest', 'xyz') @@ -682,7 +682,7 @@ tom query T SELECT trim(LEADING ' tom ') ---- -tom +tom query T SELECT trim(TRAILING ' tom ') @@ -697,7 +697,7 @@ tom query T SELECT trim(LEADING ' ' FROM ' tom ') ---- -tom +tom query T SELECT trim(TRAILING ' ' FROM ' tom ') @@ -803,7 +803,7 @@ NULL # test_random_expression query BB -SELECT +SELECT random() BETWEEN 0.0 AND 1.0, random() = random() ---- @@ -1802,7 +1802,10 @@ SELECT encode('','base64'); query ? SELECT decode('','base64'); ---- - + + + + query T SELECT encode('','hex'); @@ -1812,7 +1815,10 @@ SELECT encode('','hex'); query ? SELECT decode('','hex'); ---- - + + + + query T SELECT md5('tom'); @@ -2351,12 +2357,12 @@ NULL # query_binary_eq -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE t_source( column1 String, column2 String, column3 String, - column4 String, + column4 String ) AS VALUES ('one', 'one', 'one', 'one'), ('two', 'two', '', 'two'), @@ -2365,7 +2371,7 @@ CREATE TABLE t_source( ('three', 'three', 'three', ''); -statement error DataFusion error: Error during planning: table 'datafusion\.public\.t_source' not found +statement ok CREATE TABLE test as select arrow_cast(column1, 'Binary') as c1, @@ -2374,8 +2380,14 @@ select arrow_cast(column4, 'LargeBinary') as c4 from t_source; -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query BBBB SELECT sha256(c1)=digest('one', 'sha256'), sha256(c2)=sha256('two'), digest(c1, 'blake2b')=digest(c3, 'blake2b'), c2=c4 FROM test; +---- +true false true true +false true false true +NULL NULL NULL NULL +false false true true +false false true false ############# @@ -2446,20 +2458,20 @@ host3 3.3 # can have an aggregate function with an inner CASE WHEN query TR -select - t2.server_host as host, +select + t2.server_host as host, sum(( - case when t2.server_host is not null + case when t2.server_host is not null then t2.server_load2 end - )) + )) from ( - select + select struct(time,load1,load2,host)['c2'] as server_load2, struct(time,load1,load2,host)['c3'] as server_host from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; ---- host1 101 @@ -2468,19 +2480,19 @@ host3 303 # TODO: Issue tracked in https://github.com/apache/datafusion/issues/10364 query TR -select - t2.server['c3'] as host, +select + t2.server['c3'] as host, sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c2'] end - )) + )) from ( - select + select struct(time,load1,load2,host) as server from t1 - ) t2 - where t2.server['c3'] IS NOT NULL + ) t2 + where t2.server['c3'] IS NOT NULL group by t2.server['c3'] order by host; ---- host1 101 @@ -2489,22 +2501,22 @@ host3 303 # can have 2 projections with aggr(short_circuited), with different short-circuited expr query TRR -select - t2.server_host as host, +select + t2.server_host as host, sum(coalesce(server_load1)), sum(( - case when t2.server_host is not null + case when t2.server_host is not null then t2.server_load2 end - )) + )) from ( - select + select struct(time,load1,load2,host)['c1'] as server_load1, struct(time,load1,load2,host)['c2'] as server_load2, struct(time,load1,load2,host)['c3'] as server_host from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; ---- host1 1.1 101 @@ -2513,43 +2525,43 @@ host3 3.3 303 # TODO: Issue tracked in https://github.com/apache/datafusion/issues/10364 query error -select - t2.server['c3'] as host, +select + t2.server['c3'] as host, sum(coalesce(server['c1'])), sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c2'] end - )) + )) from ( - select + select struct(time,load1,load2,host) as server, from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; query TRR -select - t2.server_host as host, +select + t2.server_host as host, sum(( - case when t2.server_host is not null - then server_load1 + case when t2.server_host is not null + then server_load1 end - )), + )), sum(( - case when server_host is not null - then server_load2 + case when server_host is not null + then server_load2 end - )) + )) from ( - select + select struct(time,load1,load2,host)['c1'] as server_load1, struct(time,load1,load2,host)['c2'] as server_load2, struct(time,load1,load2,host)['c3'] as server_host from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; ---- host1 1.1 101 @@ -2558,24 +2570,24 @@ host3 3.3 303 # TODO: Issue tracked in https://github.com/apache/datafusion/issues/10364 query TRR -select - t2.server['c3'] as host, +select + t2.server['c3'] as host, sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c1'] end - )), + )), sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c2'] end - )) + )) from ( - select - struct(time,load1,load2,host) as server + select + struct(time,load1,load2,host) as server from t1 - ) t2 - where t2.server['c3'] IS NOT NULL + ) t2 + where t2.server['c3'] IS NOT NULL group by t2.server['c3'] order by host; ---- host1 1.1 101 diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt index dd4a3e602fd5..fd4e76d1b741 100644 --- a/datafusion/sqllogictest/test_files/group_by.slt +++ b/datafusion/sqllogictest/test_files/group_by.slt @@ -4486,10 +4486,10 @@ LIMIT 5 1 TUR 4 2022-01-03T10:00:00 TRY 100 # Create a table with timestamp data -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE src_table ( t1 TIMESTAMP, - c2 INT, + c2 INT ) AS VALUES ('2020-12-10T00:00:00.00Z', 0), ('2020-12-11T00:00:00.00Z', 1), @@ -4503,25 +4503,33 @@ CREATE TABLE src_table ( ('2020-12-19T00:00:00.00Z', 9); # Use src_table to create a partitioned file -query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found +query PI COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/0.csv' STORED AS CSV; +---- +10 -query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found +query PI COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/1.csv' STORED AS CSV; +---- +10 -query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found +query PI COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/2.csv' STORED AS CSV; +---- +10 -query error DataFusion error: Error during planning: table 'datafusion\.public\.src_table' not found +query PI COPY (SELECT * FROM src_table) TO 'test_files/scratch/group_by/timestamp_table/3.csv' STORED AS CSV; +---- +10 # Create a table from the generated CSV files: statement ok @@ -4534,18 +4542,26 @@ LOCATION 'test_files/scratch/group_by/timestamp_table' OPTIONS ('format.has_header' 'true'); # Group By using date_trunc -query error DataFusion error: Object Store error: Object at location /home/mabdeen/dev/arrow\-datafusion/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table not found: No such file or directory \(os error 2\) +query PI SELECT date_trunc('week', t1) as week, sum(c2) FROM timestamp_table GROUP BY date_trunc('week', t1) +---- +2020-12-14T00:00:00 156 +2020-12-07T00:00:00 24 # GROUP BY using LIMIT -query error DataFusion error: Object Store error: Object at location /home/mabdeen/dev/arrow\-datafusion/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table not found: No such file or directory \(os error 2\) +query IP SELECT c2, MAX(t1) FROM timestamp_table GROUP BY c2 ORDER BY MAX(t1) DESC LIMIT 4; +---- +9 2020-12-19T00:00:00 +8 2020-12-18T00:00:00 +7 2020-12-17T00:00:00 +6 2020-12-16T00:00:00 # Explain the GROUP BY with LIMIT to ensure the plan contains `lim=[4]` query TT @@ -4561,9 +4577,19 @@ logical_plan 02)--Sort: MAX(timestamp_table.t1) DESC NULLS FIRST, fetch=4 03)----Aggregate: groupBy=[[timestamp_table.c2]], aggr=[[MAX(timestamp_table.t1)]] 04)------TableScan: timestamp_table projection=[t1, c2] +physical_plan +01)GlobalLimitExec: skip=0, fetch=4 +02)--SortPreservingMergeExec: [MAX(timestamp_table.t1)@1 DESC], fetch=4 +03)----SortExec: TopK(fetch=4), expr=[MAX(timestamp_table.t1)@1 DESC], preserve_partitioning=[true] +04)------AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[MAX(timestamp_table.t1)], lim=[4] +05)--------CoalesceBatchesExec: target_batch_size=2 +06)----------RepartitionExec: partitioning=Hash([c2@0], 8), input_partitions=8 +07)------------AggregateExec: mode=Partial, gby=[c2@1 as c2], aggr=[MAX(timestamp_table.t1)], lim=[4] +08)--------------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=4 +09)----------------CsvExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/3.csv]]}, projection=[t1, c2], has_header=true # Clean up -statement error DataFusion error: Execution error: Table 'src_table' doesn't exist\. +statement ok DROP TABLE src_table; statement ok @@ -5115,7 +5141,7 @@ statement ok CREATE TABLE test_case_expr(a INT, b TEXT) AS VALUES (1,'hello'), (2,'world') query T -SELECT (CASE WHEN CONCAT(b, 'hello') = 'test' THEN 'good' ELSE 'bad' END) AS c +SELECT (CASE WHEN CONCAT(b, 'hello') = 'test' THEN 'good' ELSE 'bad' END) AS c FROM test_case_expr GROUP BY c; ---- bad diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index 2b03c1b075cf..ae1a5c78de7c 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -127,16 +127,16 @@ SELECT abs('foo'); statement ok CREATE TABLE test_nullable_integer( - c1 TINYINT, - c2 SMALLINT, - c3 INT, - c4 BIGINT, - c5 TINYINT UNSIGNED, - c6 SMALLINT UNSIGNED, - c7 INT UNSIGNED, - c8 BIGINT UNSIGNED, + c1 TINYINT, + c2 SMALLINT, + c3 INT, + c4 BIGINT, + c5 TINYINT UNSIGNED, + c6 SMALLINT UNSIGNED, + c7 INT UNSIGNED, + c8 BIGINT UNSIGNED, dataset TEXT - ) + ) AS VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'nulls'), (0, 0, 0, 0, 0, 0, 0, 0, 'zeros'), @@ -225,7 +225,7 @@ SELECT c8%0 FROM test_nullable_integer # abs: return type query TTTTTTTT rowsort -select +select arrow_typeof(abs(c1)), arrow_typeof(abs(c2)), arrow_typeof(abs(c3)), arrow_typeof(abs(c4)), arrow_typeof(abs(c5)), arrow_typeof(abs(c6)), arrow_typeof(abs(c7)), arrow_typeof(abs(c8)) from test_nullable_integer limit 1 @@ -271,164 +271,222 @@ statement ok drop table test_nullable_integer -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test_non_nullable_integer( - c1 TINYINT NOT NULL, - c2 SMALLINT NOT NULL, - c3 INT NOT NULL, - c4 BIGINT NOT NULL, - c5 TINYINT UNSIGNED NOT NULL, - c6 SMALLINT UNSIGNED NOT NULL, - c7 INT UNSIGNED NOT NULL, - c8 BIGINT UNSIGNED NOT NULL, + c1 TINYINT NOT NULL, + c2 SMALLINT NOT NULL, + c3 INT NOT NULL, + c4 BIGINT NOT NULL, + c5 TINYINT UNSIGNED NOT NULL, + c6 SMALLINT UNSIGNED NOT NULL, + c7 INT UNSIGNED NOT NULL, + c8 BIGINT UNSIGNED NOT NULL ); -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query I INSERT INTO test_non_nullable_integer VALUES(1, 1, 1, 1, 1, 1, 1, 1) +---- +1 -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query IIIIIIII select c1*0, c2*0, c3*0, c4*0, c5*0, c6*0, c7*0, c8*0 from test_non_nullable_integer +---- +0 0 0 0 0 0 0 0 -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c1/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c2/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c3/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c4/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c5/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c6/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c7/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c8/0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c1%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c2%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c3%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c4%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c5%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c6%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c7%0 FROM test_non_nullable_integer -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_integer' not found +query error DataFusion error: Arrow error: Divide by zero error SELECT c8%0 FROM test_non_nullable_integer -statement error DataFusion error: Execution error: Table 'test_non_nullable_integer' doesn't exist\. +statement ok drop table test_non_nullable_integer -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test_nullable_float( c1 float, - c2 double, + c2 double ) AS VALUES (-1.0, -1.0), - (1.0, 1.0), + (1.0, 1.0), (NULL, NULL), (0., 0.), ('NaN'::double, 'NaN'::double); -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found +query RR SELECT c1*0, c2*0 FROM test_nullable_float +---- +0 0 +0 0 +NULL NULL +0 0 +NaN NaN -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found +query RR SELECT c1/0, c2/0 FROM test_nullable_float +---- +-Infinity -Infinity +Infinity Infinity +NULL NULL +NaN NaN +NaN NaN -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found +query RR SELECT c1%0, c2%0 FROM test_nullable_float +---- +NaN NaN +NaN NaN +NULL NULL +NaN NaN +NaN NaN -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found +query RR SELECT c1%1, c2%1 FROM test_nullable_float +---- +0 0 +0 0 +NULL NULL +0 0 +NaN NaN # abs: return type -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found +query TT SELECT arrow_typeof(abs(c1)), arrow_typeof(abs(c2)) FROM test_nullable_float limit 1 +---- +Float32 Float64 # abs: floats -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_nullable_float' not found -SELECT abs(c1), abs(c2) from test_nullable_float +query RR +SELECT abs(c1), abs(c2) from test_nullable_float +---- +1 1 +1 1 +NULL NULL +0 0 +NaN NaN -statement error DataFusion error: Execution error: Table 'test_nullable_float' doesn't exist\. +statement ok drop table test_nullable_float -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test_non_nullable_float( c1 float NOT NULL, - c2 double NOT NULL, - ); + c2 double NOT NULL + ); -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found +query I INSERT INTO test_non_nullable_float VALUES (-1.0, -1.0), (1.0, 1.0), (0., 0.), ('NaN'::double, 'NaN'::double) +---- +4 -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found +query RR SELECT c1*0, c2*0 FROM test_non_nullable_float +---- +0 0 +0 0 +0 0 +NaN NaN -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found +query RR SELECT c1/0, c2/0 FROM test_non_nullable_float +---- +-Infinity -Infinity +Infinity Infinity +NaN NaN +NaN NaN -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found +query RR SELECT c1%0, c2%0 FROM test_non_nullable_float +---- +NaN NaN +NaN NaN +NaN NaN +NaN NaN -query error DataFusion error: Error during planning: table 'datafusion\.public\.test_non_nullable_float' not found +query RR SELECT c1%1, c2%1 FROM test_non_nullable_float +---- +0 0 +0 0 +0 0 +NaN NaN -statement error DataFusion error: Execution error: Table 'test_non_nullable_float' doesn't exist\. +statement ok drop table test_non_nullable_float statement ok CREATE TABLE test_nullable_decimal( c1 DECIMAL(10, 2), /* Decimal128 */ - c2 DECIMAL(38, 10), /* Decimal128 with max precision */ + c2 DECIMAL(38, 10), /* Decimal128 with max precision */ c3 DECIMAL(40, 2), /* Decimal256 */ - c4 DECIMAL(76, 10) /* Decimal256 with max precision */ - ) AS VALUES - (0, 0, 0, 0), + c4 DECIMAL(76, 10) /* Decimal256 with max precision */ + ) AS VALUES + (0, 0, 0, 0), (NULL, NULL, NULL, NULL); query I INSERT into test_nullable_decimal values ( - -99999999.99, - '-9999999999999999999999999999.9999999999', - '-99999999999999999999999999999999999999.99', + -99999999.99, + '-9999999999999999999999999999.9999999999', + '-99999999999999999999999999999999999999.99', '-999999999999999999999999999999999999999999999999999999999999999999.9999999999' - ), + ), ( - 99999999.99, - '9999999999999999999999999999.9999999999', - '99999999999999999999999999999999999999.99', + 99999999.99, + '9999999999999999999999999999.9999999999', + '99999999999999999999999999999999999999.99', '999999999999999999999999999999999999999999999999999999999999999999.9999999999' - ) + ) ---- 2 @@ -463,9 +521,9 @@ SELECT c1%0 FROM test_nullable_decimal WHERE c1 IS NOT NULL; # abs: return type query TTTT -SELECT - arrow_typeof(abs(c1)), - arrow_typeof(abs(c2)), +SELECT + arrow_typeof(abs(c1)), + arrow_typeof(abs(c2)), arrow_typeof(abs(c3)), arrow_typeof(abs(c4)) FROM test_nullable_decimal limit 1 @@ -482,11 +540,11 @@ SELECT abs(c1), abs(c2), abs(c3), abs(c4) FROM test_nullable_decimal NULL NULL NULL NULL statement ok -drop table test_nullable_decimal +drop table test_nullable_decimal statement ok -CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL); +CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL); query I INSERT INTO test_non_nullable_decimal VALUES(1) @@ -499,13 +557,13 @@ SELECT c1*0 FROM test_non_nullable_decimal 0 query error DataFusion error: Arrow error: Divide by zero error -SELECT c1/0 FROM test_non_nullable_decimal +SELECT c1/0 FROM test_non_nullable_decimal query error DataFusion error: Arrow error: Divide by zero error -SELECT c1%0 FROM test_non_nullable_decimal +SELECT c1%0 FROM test_non_nullable_decimal statement ok -drop table test_non_nullable_decimal +drop table test_non_nullable_decimal statement ok CREATE TABLE signed_integers( @@ -545,7 +603,7 @@ NULL NULL NULL # scalar maxes and/or negative 1 query III -select +select gcd(9223372036854775807, -9223372036854775808), -- i64::MAX, i64::MIN gcd(9223372036854775807, -1), -- i64::MAX, -1 gcd(-9223372036854775808, -1); -- i64::MIN, -1 diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.slt index 4ecdecf5f03a..8292f839e54d 100644 --- a/datafusion/sqllogictest/test_files/predicates.slt +++ b/datafusion/sqllogictest/test_files/predicates.slt @@ -39,7 +39,7 @@ CREATE EXTERNAL TABLE aggregate_test_100 ( c13 VARCHAR NOT NULL ) STORED AS CSV -LOCATION '../../testing/data/csv/aggregate_test_100.csv' +LOCATION '../../testing/data/csv/aggregate_test_100.csv' OPTIONS ('format.has_header' 'true'); statement ok @@ -692,7 +692,7 @@ CREATE TABLE IF NOT EXISTS partsupp ( ps_suppkey BIGINT, ps_availqty INTEGER, ps_supplycost DECIMAL(15, 2), - ps_comment VARCHAR, + ps_comment VARCHAR ) AS VALUES (63700, 7311, 100, 993.49, 'ven ideas. quickly even packages print. pending multipliers must have to are fluff'); diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 376350d530de..7e6c9e7caedc 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -1578,7 +1578,7 @@ false statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t1( a boolean, - b boolean, + b boolean ) as VALUES (true, true), (true, null), @@ -1806,7 +1806,7 @@ D false # test string_temporal_coercion query BBBBBBBBBB -select +select arrow_cast(to_timestamp('2020-01-01 01:01:11.1234567890Z'), 'Timestamp(Second, None)') == '2020-01-01T01:01:11', arrow_cast(to_timestamp('2020-01-02 01:01:11.1234567890Z'), 'Timestamp(Second, None)') == arrow_cast('2020-01-02T01:01:11', 'LargeUtf8'), arrow_cast(to_timestamp('2020-01-03 01:01:11.1234567890Z'), 'Time32(Second)') == '01:01:11', diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index d7c8a9314e9e..dfb66f1e11ec 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -101,7 +101,7 @@ statement error DataFusion error: SQL error: ParserError\("Expected: column name CREATE TABLE test ( c1 BIGINT NOT NULL, c2 BIGINT NOT NULL, - c3 BOOLEAN NOT NULL, + c3 BOOLEAN NOT NULL ) AS VALUES (0, 1, false), (0, 10, true), (0, 2, true), @@ -375,7 +375,7 @@ VALUES (1,2,3,4,5,6,7,8,9,10,11,12,13,NULL,'F',3.5) # Test non-literal expressions in VALUES query II -VALUES (1, CASE WHEN RANDOM() > 0.5 THEN 1 ELSE 1 END), +VALUES (1, CASE WHEN RANDOM() > 0.5 THEN 1 ELSE 1 END), (2, CASE WHEN RANDOM() > 0.5 THEN 2 ELSE 2 END); ---- 1 1 @@ -1452,7 +1452,7 @@ query II SELECT CASE WHEN B.x > 0 THEN A.x / B.x ELSE 0 END AS value1, CASE WHEN B.x > 0 AND B.y > 0 THEN A.x / B.x ELSE 0 END AS value3 -FROM t AS A, (SELECT * FROM t WHERE x = 0) AS B; +FROM t AS A, (SELECT * FROM t WHERE x = 0) AS B; ---- 0 0 0 0 diff --git a/datafusion/sqllogictest/test_files/strings.slt b/datafusion/sqllogictest/test_files/strings.slt index 0d632db013ff..cba6301c0c46 100644 --- a/datafusion/sqllogictest/test_files/strings.slt +++ b/datafusion/sqllogictest/test_files/strings.slt @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test( - s TEXT, + s TEXT ) as VALUES ('p1'), ('p1e1'), @@ -32,24 +32,52 @@ CREATE TABLE test( ; # LIKE -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT s FROM test WHERE s LIKE 'p1%'; +---- +p1 +p1e1 +p1m1e1 -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT s FROM test WHERE s LIKE '%m1%'; +---- +p1m1e1 +P1m1e1 +p2m1e1 # NOT LIKE -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT s FROM test WHERE s NOT LIKE 'p1%'; +---- +P1 +P1e1 +P1m1e1 +e1 +p2 +p2e1 +p2m1e1 # ILIKE -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT s FROM test WHERE s ILIKE 'p1%'; +---- +p1 +p1e1 +p1m1e1 +P1 +P1e1 +P1m1e1 # NOT ILIKE -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query T SELECT s FROM test WHERE s NOT ILIKE 'p1%'; +---- +e1 +p2 +p2e1 +p2m1e1 ## VARCHAR with length support diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt index c46da32d69df..f67b9d30fc71 100644 --- a/datafusion/sqllogictest/test_files/struct.slt +++ b/datafusion/sqllogictest/test_files/struct.slt @@ -24,7 +24,7 @@ CREATE TABLE values( a INT, b FLOAT, c VARCHAR, - n VARCHAR, + n VARCHAR ) AS VALUES (1, 1.1, 'a', NULL), (2, 2.2, 'b', NULL), diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index 26fdc4bebd9c..ee8060810d09 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -21,33 +21,33 @@ statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t1( - id INT, - name TEXT, + id INT, + name TEXT ) as VALUES - (1, 'Alex'), - (2, 'Bob'), + (1, 'Alex'), + (2, 'Bob'), (3, 'Alice') ; statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) CREATE TABLE t2( id TINYINT, - name TEXT, + name TEXT ) as VALUES - (1, 'Alex'), - (2, 'Bob'), + (1, 'Alex'), + (2, 'Bob'), (3, 'John') ; # union with EXCEPT(JOIN) query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found -( +( SELECT name FROM t1 EXCEPT SELECT name FROM t2 -) +) UNION ALL -( +( SELECT name FROM t2 EXCEPT SELECT name FROM t1 @@ -55,13 +55,13 @@ UNION ALL # union with type coercion query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found -( +( SELECT * FROM t1 EXCEPT SELECT * FROM t2 -) +) UNION ALL -( +( SELECT * FROM t2 EXCEPT SELECT * FROM t1 @@ -349,11 +349,11 @@ OPTIONS ('format.has_header' 'true'); query TT explain SELECT c1 FROM( -( +( SELECT c1 FROM t1 -) +) UNION ALL -( +( SELECT c1a FROM t2 )) ORDER BY c1 diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index d685ef93d5af..ead961073e5c 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -1766,7 +1766,7 @@ logical_plan 01)Projection: count(*) AS global_count 02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1)) AS count(*)]] 03)----SubqueryAlias: a -04)------Projection: +04)------Projection: 05)--------Sort: aggregate_test_100.c1 ASC NULLS LAST 06)----------Aggregate: groupBy=[[aggregate_test_100.c1]], aggr=[[]] 07)------------Projection: aggregate_test_100.c1 @@ -3415,7 +3415,7 @@ SELECT # window1 spec is defined multiple times statement error DataFusion error: SQL error: ParserError\("Expected an expression, found: FROM"\) SELECT - MAX(c12) OVER window1 as min1, + MAX(c12) OVER window1 as min1 FROM aggregate_test_100 WINDOW window1 AS (ORDER BY C12), window1 AS (ORDER BY C3) From 38af0e6c55eeaa935db51233a9b7e69a92d31480 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 19:53:27 +0300 Subject: [PATCH 05/15] update slt tests results --- datafusion/sqllogictest/test_files/expr.slt | 2 + .../sqllogictest/test_files/group_by.slt | 2 +- .../sqllogictest/test_files/predicates.slt | 40 +++- datafusion/sqllogictest/test_files/scalar.slt | 52 ++++- datafusion/sqllogictest/test_files/select.slt | 35 ++- datafusion/sqllogictest/test_files/struct.slt | 79 +++++-- datafusion/sqllogictest/test_files/union.slt | 205 ++++++++++++++++-- datafusion/sqllogictest/test_files/window.slt | 2 +- 8 files changed, 368 insertions(+), 49 deletions(-) diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 6a2d3cef09a2..d186dc569fab 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -1807,6 +1807,7 @@ SELECT decode('','base64'); + query T SELECT encode('','hex'); ---- @@ -1820,6 +1821,7 @@ SELECT decode('','hex'); + query T SELECT md5('tom'); ---- diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt index fd4e76d1b741..f542218d5f97 100644 --- a/datafusion/sqllogictest/test_files/group_by.slt +++ b/datafusion/sqllogictest/test_files/group_by.slt @@ -4547,8 +4547,8 @@ SELECT date_trunc('week', t1) as week, sum(c2) FROM timestamp_table GROUP BY date_trunc('week', t1) ---- -2020-12-14T00:00:00 156 2020-12-07T00:00:00 24 +2020-12-14T00:00:00 156 # GROUP BY using LIMIT query IP diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.slt index 8292f839e54d..d6b8579535c0 100644 --- a/datafusion/sqllogictest/test_files/predicates.slt +++ b/datafusion/sqllogictest/test_files/predicates.slt @@ -686,7 +686,7 @@ physical_plan ######## # TPCH Q19 - Pull predicates to inner join (simplified) ######## -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE IF NOT EXISTS partsupp ( ps_partkey BIGINT, ps_suppkey BIGINT, @@ -696,7 +696,7 @@ CREATE TABLE IF NOT EXISTS partsupp ( ) AS VALUES (63700, 7311, 100, 993.49, 'ven ideas. quickly even packages print. pending multipliers must have to are fluff'); -query error DataFusion error: Error during planning: table 'datafusion\.public\.partsupp' not found +query IRRI SELECT p_partkey, sum(l_extendedprice), @@ -719,9 +719,11 @@ OR AND ps_partkey = p_partkey ) GROUP BY p_partkey; +---- +63700 13309.6 0.1 1 -query error DataFusion error: Error during planning: table 'datafusion\.public\.partsupp' not found +query TT EXPLAIN SELECT p_partkey, sum(l_extendedprice), @@ -744,6 +746,38 @@ OR AND ps_partkey = p_partkey ) GROUP BY p_partkey; +---- +logical_plan +01)Aggregate: groupBy=[[part.p_partkey]], aggr=[[sum(lineitem.l_extendedprice), avg(lineitem.l_discount), count(DISTINCT partsupp.ps_suppkey)]] +02)--Projection: lineitem.l_extendedprice, lineitem.l_discount, part.p_partkey, partsupp.ps_suppkey +03)----Inner Join: part.p_partkey = partsupp.ps_partkey +04)------Projection: lineitem.l_extendedprice, lineitem.l_discount, part.p_partkey +05)--------Inner Join: lineitem.l_partkey = part.p_partkey +06)----------TableScan: lineitem projection=[l_partkey, l_extendedprice, l_discount] +07)----------Projection: part.p_partkey +08)------------Filter: part.p_brand = Utf8("Brand#12") OR part.p_brand = Utf8("Brand#23") +09)--------------TableScan: part projection=[p_partkey, p_brand], partial_filters=[part.p_brand = Utf8("Brand#12") OR part.p_brand = Utf8("Brand#23")] +10)------TableScan: partsupp projection=[ps_partkey, ps_suppkey] +physical_plan +01)AggregateExec: mode=SinglePartitioned, gby=[p_partkey@2 as p_partkey], aggr=[sum(lineitem.l_extendedprice), avg(lineitem.l_discount), count(DISTINCT partsupp.ps_suppkey)] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@2, ps_partkey@0)], projection=[l_extendedprice@0, l_discount@1, p_partkey@2, ps_suppkey@4] +04)------CoalesceBatchesExec: target_batch_size=8192 +05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_extendedprice@1, l_discount@2, p_partkey@3] +06)----------CoalesceBatchesExec: target_batch_size=8192 +07)------------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4 +08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +09)----------------CsvExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/lineitem.csv]]}, projection=[l_partkey, l_extendedprice, l_discount], has_header=true +10)----------CoalesceBatchesExec: target_batch_size=8192 +11)------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4 +12)--------------ProjectionExec: expr=[p_partkey@0 as p_partkey] +13)----------------CoalesceBatchesExec: target_batch_size=8192 +14)------------------FilterExec: p_brand@1 = Brand#12 OR p_brand@1 = Brand#23 +15)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +16)----------------------CsvExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/part.csv]]}, projection=[p_partkey, p_brand], has_header=true +17)------CoalesceBatchesExec: target_batch_size=8192 +18)--------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=1 +19)----------MemoryExec: partitions=1, partition_sizes=[1] # Inlist simplification diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 7e6c9e7caedc..7265c152378c 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -1575,7 +1575,7 @@ SELECT CASE WHEN 'cpu' != 'cpu' THEN true ELSE false END ---- false -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE t1( a boolean, b boolean @@ -1592,27 +1592,67 @@ CREATE TABLE t1( ; # csv query boolean eq neq -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query BBBBBB SELECT a, b, a = b as eq, b = true as eq_scalar, a != b as neq, a != true as neq_scalar FROM t1 +---- +true true true true false false +true NULL NULL NULL NULL false +true false false false true false +NULL true NULL true NULL NULL +NULL NULL NULL NULL NULL NULL +NULL false NULL false NULL NULL +false true false true true true +false NULL NULL NULL NULL true +false NULL NULL NULL NULL true # csv query boolean lt lt eq -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query BBBBBB SELECT a, b, a < b as lt, b = true as lt_scalar, a <= b as lt_eq, a <= true as lt_eq_scalar FROM t1 +---- +true true false true true true +true NULL NULL NULL NULL true +true false false false false true +NULL true NULL true NULL NULL +NULL NULL NULL NULL NULL NULL +NULL false NULL false NULL NULL +false true true true true true +false NULL NULL NULL NULL true +false NULL NULL NULL NULL true # csv query boolean gt gt eq -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query BBBBBB SELECT a, b, a > b as gt, b = true as gt_scalar, a >= b as gt_eq, a >= true as gt_eq_scalar FROM t1 +---- +true true false true true true +true NULL NULL NULL NULL true +true false true false true true +NULL true NULL true NULL NULL +NULL NULL NULL NULL NULL NULL +NULL false NULL false NULL NULL +false true false true false false +false NULL NULL NULL NULL false +false NULL NULL NULL NULL false # csv query boolean distinct from -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query BBBBBB SELECT a, b, a is distinct from b as df, b is distinct from true as df_scalar, a is not distinct from b as ndf, a is not distinct from true as ndf_scalar FROM t1 +---- +true true false false true true +true NULL true true false true +true false true true false true +NULL true true false false false +NULL NULL false true true false +NULL false true true false false +false true true false false false +false NULL true true false false +false NULL true true false false -statement error DataFusion error: Execution error: Table 't1' doesn't exist\. +statement ok drop table t1 # like nlike with null lt diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index dfb66f1e11ec..9165ee21cf19 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -97,7 +97,7 @@ One 1 Three 1 -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE test ( c1 BIGINT NOT NULL, c2 BIGINT NOT NULL, @@ -145,16 +145,43 @@ CREATE TABLE test ( # parallel_query_with_filter -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query II SELECT c1, c2 FROM test WHERE c1 > 0 AND c1 < 3; +---- +1 1 +1 10 +1 2 +1 3 +1 4 +1 5 +1 6 +1 7 +1 8 +1 9 +2 1 +2 10 +2 2 +2 3 +2 4 +2 5 +2 6 +2 7 +2 8 +2 9 ###### # Boolean literal ###### -query error DataFusion error: Error during planning: table 'datafusion\.public\.test' not found +query IB SELECT c1, c3 FROM test WHERE c1 > 2 AND c3 = true; +---- +3 true +3 true +3 true +3 true +3 true -statement error DataFusion error: Execution error: Table 'test' doesn't exist\. +statement ok drop table test; ###### diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt index f67b9d30fc71..a7384fd4d8ad 100644 --- a/datafusion/sqllogictest/test_files/struct.slt +++ b/datafusion/sqllogictest/test_files/struct.slt @@ -19,7 +19,7 @@ ## Struct Expressions Tests ############# -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE values( a INT, b FLOAT, @@ -65,8 +65,12 @@ select struct(1, 3.14, 'h')['c0'], struct(3, 2.55, 'b')['c1'], struct(2, 6.43, ' 1 2.55 a # struct[i] with columns -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query R select struct(a, b, c)['c1'] from values; +---- +1.1 +2.2 +3.3 # struct scalar function #1 query ? @@ -87,17 +91,32 @@ select struct(1, 3.14 as name1, 'e', true); {c0: 1, name1: 3.14, c2: e, c3: true} # struct scalar function with columns #1 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select struct(a, b, c) from values; +---- +{c0: 1, c1: 1.1, c2: a} +{c0: 2, c1: 2.2, c2: b} +{c0: 3, c1: 3.3, c2: c} # struct scalar function with columns and scalars -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select struct(a, 'foo') from values; +---- +{c0: 1, c1: foo} +{c0: 2, c1: foo} +{c0: 3, c1: foo} # explain struct scalar function with columns #1 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query TT explain select struct(a, b, c) from values; +---- +logical_plan +01)Projection: struct(values.a, values.b, values.c) +02)--TableScan: values projection=[a, b, c] +physical_plan +01)ProjectionExec: expr=[struct(a@0, b@1, c@2) as struct(values.a,values.b,values.c)] +02)--MemoryExec: partitions=1, partition_sizes=[1] # error on 0 arguments query error @@ -112,7 +131,7 @@ query error DataFusion error: Execution error: named_struct requires an even num select named_struct(1); # error on odd number of arguments #3 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query error DataFusion error: Execution error: named_struct requires an even number of arguments, got 1 instead select named_struct(values.a) from values; # error on odd number of arguments #4 @@ -128,38 +147,66 @@ query error DataFusion error: Execution error: named_struct even arguments must select named_struct('corret', 1, 0, 'wrong'); # error on even argument not a string literal #3 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query error DataFusion error: Execution error: named_struct even arguments must be string literals, got values\.a instead at position 0 select named_struct(values.a, 'a') from values; # error on even argument not a string literal #4 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query error DataFusion error: Execution error: named_struct even arguments must be string literals, got values\.c instead at position 0 select named_struct(values.c, 'c') from values; # named_struct with mixed scalar and array values #1 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select named_struct('scalar', 27, 'array', values.a, 'null', NULL) from values; +---- +{scalar: 27, array: 1, null: } +{scalar: 27, array: 2, null: } +{scalar: 27, array: 3, null: } -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select {'scalar': 27, 'array': values.a, 'null': NULL} from values; +---- +{scalar: 27, array: 1, null: } +{scalar: 27, array: 2, null: } +{scalar: 27, array: 3, null: } # named_struct with mixed scalar and array values #2 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select named_struct('array', values.a, 'scalar', 27, 'null', NULL) from values; +---- +{array: 1, scalar: 27, null: } +{array: 2, scalar: 27, null: } +{array: 3, scalar: 27, null: } -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select {'array': values.a, 'scalar': 27, 'null': NULL} from values; +---- +{array: 1, scalar: 27, null: } +{array: 2, scalar: 27, null: } +{array: 3, scalar: 27, null: } # named_struct with mixed scalar and array values #3 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select named_struct('null', NULL, 'array', values.a, 'scalar', 27) from values; +---- +{null: , array: 1, scalar: 27} +{null: , array: 2, scalar: 27} +{null: , array: 3, scalar: 27} # named_struct with mixed scalar and array values #4 -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select named_struct('null_array', values.n, 'array', values.a, 'scalar', 27, 'null', NULL) from values; +---- +{null_array: , array: 1, scalar: 27, null: } +{null_array: , array: 2, scalar: 27, null: } +{null_array: , array: 3, scalar: 27, null: } # named_struct arrays only -query error DataFusion error: Error during planning: table 'datafusion\.public\.values' not found +query ? select named_struct('field_a', a, 'field_b', b) from values; +---- +{field_a: 1, field_b: 1.1} +{field_a: 2, field_b: 2.2} +{field_a: 3, field_b: 3.3} # named_struct scalars only query ? @@ -167,7 +214,7 @@ select named_struct('field_a', 1, 'field_b', 2); ---- {field_a: 1, field_b: 2} -statement error DataFusion error: Execution error: Table 'values' doesn't exist\. +statement ok drop table values; query T diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index ee8060810d09..029effa35abf 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -19,7 +19,7 @@ ## UNION Tests ########## -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE t1( id INT, name TEXT @@ -29,7 +29,7 @@ CREATE TABLE t1( (3, 'Alice') ; -statement error DataFusion error: SQL error: ParserError\("Expected: column name or constraint definition, found: \)"\) +statement ok CREATE TABLE t2( id TINYINT, name TEXT @@ -40,7 +40,7 @@ CREATE TABLE t2( ; # union with EXCEPT(JOIN) -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query T ( SELECT name FROM t1 EXCEPT @@ -52,9 +52,12 @@ UNION ALL EXCEPT SELECT name FROM t1 ) +---- +John +Alice # union with type coercion -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query IT ( SELECT * FROM t1 EXCEPT @@ -66,6 +69,9 @@ UNION ALL EXCEPT SELECT * FROM t1 ) +---- +3 Alice +3 John # union all query I rowsort @@ -76,8 +82,10 @@ SELECT 2 as x 1 2 -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query I select count(*) from (select id from t1 union all select id from t2) +---- +6 # csv_union_all statement ok @@ -170,7 +178,7 @@ SELECT 1 UNION SELECT 2 2 # union_with_except_input -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query T ( SELECT name FROM t1 EXCEPT @@ -182,29 +190,82 @@ UNION ALL EXCEPT SELECT name FROM t1 ) +---- +John +Alice # nested_union -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query T SELECT name FROM t1 UNION (SELECT name from t2 UNION SELECT name || '_new' from t2) +---- +Bob +Alex_new +Alex +John +John_new +Alice +Bob_new # should be un-nested, with a single (logical) aggregate -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query TT EXPLAIN SELECT name FROM t1 UNION (SELECT name from t2 UNION SELECT name || '_new' from t2) +---- +logical_plan +01)Aggregate: groupBy=[[t1.name]], aggr=[[]] +02)--Union +03)----TableScan: t1 projection=[name] +04)----TableScan: t2 projection=[name] +05)----Projection: t2.name || Utf8("_new") AS name +06)------TableScan: t2 projection=[name] +physical_plan +01)AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 +05)--------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] +06)----------UnionExec +07)------------MemoryExec: partitions=1, partition_sizes=[1] +08)------------MemoryExec: partitions=1, partition_sizes=[1] +09)------------ProjectionExec: expr=[name@0 || _new as name] +10)--------------MemoryExec: partitions=1, partition_sizes=[1] # nested_union_all -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query T SELECT name FROM t1 UNION ALL (SELECT name from t2 UNION ALL SELECT name || '_new' from t2) +---- +Alex +Bob +Alice +Alex_new +Bob_new +John_new +Alex +Bob +John # Plan is unnested -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query TT EXPLAIN SELECT name FROM t1 UNION ALL (SELECT name from t2 UNION ALL SELECT name || '_new' from t2) +---- +logical_plan +01)Union +02)--TableScan: t1 projection=[name] +03)--TableScan: t2 projection=[name] +04)--Projection: t2.name || Utf8("_new") AS name +05)----TableScan: t2 projection=[name] +physical_plan +01)UnionExec +02)--MemoryExec: partitions=1, partition_sizes=[1] +03)--MemoryExec: partitions=1, partition_sizes=[1] +04)--ProjectionExec: expr=[name@0 || _new as name] +05)----MemoryExec: partitions=1, partition_sizes=[1] # Make sure to choose a small batch size to introduce parallelism to the plan. statement ok set datafusion.execution.batch_size = 2; # union_with_type_coercion -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query TT explain ( SELECT id, name FROM t1 @@ -217,9 +278,52 @@ UNION ALL EXCEPT SELECT id, name FROM t1 ) - - -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +---- +logical_plan +01)Union +02)--LeftAnti Join: t1.id = CAST(t2.id AS Int32), t1.name = t2.name +03)----Aggregate: groupBy=[[t1.id, t1.name]], aggr=[[]] +04)------TableScan: t1 projection=[id, name] +05)----TableScan: t2 projection=[id, name] +06)--Projection: CAST(t2.id AS Int32) AS id, t2.name +07)----LeftAnti Join: CAST(t2.id AS Int32) = t1.id, t2.name = t1.name +08)------Aggregate: groupBy=[[t2.id, t2.name]], aggr=[[]] +09)--------TableScan: t2 projection=[id, name] +10)------TableScan: t1 projection=[id, name] +physical_plan +01)UnionExec +02)--CoalesceBatchesExec: target_batch_size=2 +03)----HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(id@0, CAST(t2.id AS Int32)@2), (name@1, name@1)] +04)------AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] +05)--------CoalesceBatchesExec: target_batch_size=2 +06)----------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 +07)------------AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] +08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +09)----------------MemoryExec: partitions=1, partition_sizes=[1] +10)------CoalesceBatchesExec: target_batch_size=2 +11)--------RepartitionExec: partitioning=Hash([CAST(t2.id AS Int32)@2, name@1], 4), input_partitions=4 +12)----------ProjectionExec: expr=[id@0 as id, name@1 as name, CAST(id@0 AS Int32) as CAST(t2.id AS Int32)] +13)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +14)--------------MemoryExec: partitions=1, partition_sizes=[1] +15)--ProjectionExec: expr=[CAST(id@0 AS Int32) as id, name@1 as name] +16)----CoalesceBatchesExec: target_batch_size=2 +17)------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(CAST(t2.id AS Int32)@2, id@0), (name@1, name@1)], projection=[id@0, name@1] +18)--------CoalesceBatchesExec: target_batch_size=2 +19)----------RepartitionExec: partitioning=Hash([CAST(t2.id AS Int32)@2, name@1], 4), input_partitions=4 +20)------------ProjectionExec: expr=[id@0 as id, name@1 as name, CAST(id@0 AS Int32) as CAST(t2.id AS Int32)] +21)--------------AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] +22)----------------CoalesceBatchesExec: target_batch_size=2 +23)------------------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 +24)--------------------AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] +25)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +26)------------------------MemoryExec: partitions=1, partition_sizes=[1] +27)--------CoalesceBatchesExec: target_batch_size=2 +28)----------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 +29)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +30)--------------MemoryExec: partitions=1, partition_sizes=[1] + + +query IT ( SELECT id, name FROM t1 EXCEPT @@ -231,9 +335,12 @@ UNION ALL EXCEPT SELECT id, name FROM t1 ) +---- +3 Alice +3 John # union_with_except_input -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query TT explain ( SELECT name FROM t1 @@ -246,6 +353,43 @@ UNION ALL EXCEPT SELECT name FROM t1 ) +---- +logical_plan +01)Union +02)--LeftAnti Join: t1.name = t2.name +03)----Aggregate: groupBy=[[t1.name]], aggr=[[]] +04)------TableScan: t1 projection=[name] +05)----TableScan: t2 projection=[name] +06)--LeftAnti Join: t2.name = t1.name +07)----Aggregate: groupBy=[[t2.name]], aggr=[[]] +08)------TableScan: t2 projection=[name] +09)----TableScan: t1 projection=[name] +physical_plan +01)InterleaveExec +02)--CoalesceBatchesExec: target_batch_size=2 +03)----HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(name@0, name@0)] +04)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] +05)--------CoalesceBatchesExec: target_batch_size=2 +06)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +07)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] +08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +09)----------------MemoryExec: partitions=1, partition_sizes=[1] +10)------CoalesceBatchesExec: target_batch_size=2 +11)--------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +12)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +13)------------MemoryExec: partitions=1, partition_sizes=[1] +14)--CoalesceBatchesExec: target_batch_size=2 +15)----HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(name@0, name@0)] +16)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] +17)--------CoalesceBatchesExec: target_batch_size=2 +18)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +19)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] +20)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +21)----------------MemoryExec: partitions=1, partition_sizes=[1] +22)------CoalesceBatchesExec: target_batch_size=2 +23)--------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +24)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +25)------------MemoryExec: partitions=1, partition_sizes=[1] # union_upcast_types query TT @@ -282,23 +426,48 @@ e 4144173353 b 4076864659 # union_with_hash_aggregate -query error DataFusion error: Error during planning: table 'datafusion\.public\.t1' not found +query TT explain SELECT count(*) FROM ( SELECT distinct name FROM t1 UNION ALL SELECT distinct name FROM t2 ) GROUP BY name +---- +logical_plan +01)Projection: count(*) +02)--Aggregate: groupBy=[[t1.name]], aggr=[[count(Int64(1)) AS count(*)]] +03)----Union +04)------Aggregate: groupBy=[[t1.name]], aggr=[[]] +05)--------TableScan: t1 projection=[name] +06)------Aggregate: groupBy=[[t2.name]], aggr=[[]] +07)--------TableScan: t2 projection=[name] +physical_plan +01)ProjectionExec: expr=[count(*)@1 as count(*)] +02)--AggregateExec: mode=SinglePartitioned, gby=[name@0 as name], aggr=[count(*)] +03)----InterleaveExec +04)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] +05)--------CoalesceBatchesExec: target_batch_size=2 +06)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +07)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] +08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +09)----------------MemoryExec: partitions=1, partition_sizes=[1] +10)------AggregateExec: mode=FinalPartitioned, gby=[name@0 as name], aggr=[] +11)--------CoalesceBatchesExec: target_batch_size=2 +12)----------RepartitionExec: partitioning=Hash([name@0], 4), input_partitions=4 +13)------------AggregateExec: mode=Partial, gby=[name@0 as name], aggr=[] +14)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +15)----------------MemoryExec: partitions=1, partition_sizes=[1] ######## # Clean up after the test ######## -statement error DataFusion error: Execution error: Table 't1' doesn't exist\. +statement ok drop table t1; -statement error DataFusion error: Execution error: Table 't2' doesn't exist\. +statement ok drop table t2; statement ok diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index ead961073e5c..bb52b027a8e2 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -3413,7 +3413,7 @@ SELECT LIMIT 5 # window1 spec is defined multiple times -statement error DataFusion error: SQL error: ParserError\("Expected an expression, found: FROM"\) +statement error DataFusion error: Error during planning: The window window1 is defined multiple times! SELECT MAX(c12) OVER window1 as min1 FROM aggregate_test_100 From e7b60c055e09ad6101b7ce26ac06bb2938da3647 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 20:03:47 +0300 Subject: [PATCH 06/15] restore rowsort in slt tests --- datafusion/sqllogictest/test_files/expr.slt | 22 +-------- datafusion/sqllogictest/test_files/scalar.slt | 48 +++++++++---------- datafusion/sqllogictest/test_files/union.slt | 34 ++++++------- 3 files changed, 43 insertions(+), 61 deletions(-) diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index d186dc569fab..1b19a16ecbfc 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -1802,30 +1802,12 @@ SELECT encode('','base64'); query ? SELECT decode('','base64'); ---- - - - - - - -query T -SELECT encode('','hex'); ----- -(empty) + query ? SELECT decode('','hex'); ---- - - - - - - -query T -SELECT md5('tom'); ----- -34b7da764b21d298ef307d04d8152dc5 + query ? SELECT digest('tom','md5'); diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 7265c152378c..10785d209436 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -1592,49 +1592,49 @@ CREATE TABLE t1( ; # csv query boolean eq neq -query BBBBBB +query BBBBBB rowsort SELECT a, b, a = b as eq, b = true as eq_scalar, a != b as neq, a != true as neq_scalar FROM t1 ---- -true true true true false false -true NULL NULL NULL NULL false -true false false false true false -NULL true NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL false NULL false NULL NULL -false true false true true true +NULL true NULL true NULL NULL false NULL NULL NULL NULL true false NULL NULL NULL NULL true +false true false true true true +true NULL NULL NULL NULL false +true false false false true false +true true true true false false # csv query boolean lt lt eq -query BBBBBB +query BBBBBB rowsort SELECT a, b, a < b as lt, b = true as lt_scalar, a <= b as lt_eq, a <= true as lt_eq_scalar FROM t1 ---- -true true false true true true -true NULL NULL NULL NULL true -true false false false false true -NULL true NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL false NULL false NULL NULL -false true true true true true +NULL true NULL true NULL NULL false NULL NULL NULL NULL true false NULL NULL NULL NULL true +false true true true true true +true NULL NULL NULL NULL true +true false false false false true +true true false true true true # csv query boolean gt gt eq -query BBBBBB +query BBBBBB rowsort SELECT a, b, a > b as gt, b = true as gt_scalar, a >= b as gt_eq, a >= true as gt_eq_scalar FROM t1 ---- -true true false true true true -true NULL NULL NULL NULL true -true false true false true true -NULL true NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL false NULL false NULL NULL -false true false true false false +NULL true NULL true NULL NULL false NULL NULL NULL NULL false false NULL NULL NULL NULL false +false true false true false false +true NULL NULL NULL NULL true +true false true false true true +true true false true true true # csv query boolean distinct from -query BBBBBB +query BBBBBB rowsort SELECT a, b, a is distinct from b as df, b is distinct from true as df_scalar, @@ -1642,15 +1642,15 @@ SELECT a, b, a is not distinct from true as ndf_scalar FROM t1 ---- -true true false false true true -true NULL true true false true -true false true true false true -NULL true true false false false NULL NULL false true true false NULL false true true false false -false true true false false false +NULL true true false false false false NULL true true false false false NULL true true false false +false true true false false false +true NULL true true false true +true false true true false true +true true false false true true statement ok drop table t1 diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index 029effa35abf..56732b3f58ad 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -40,7 +40,7 @@ CREATE TABLE t2( ; # union with EXCEPT(JOIN) -query T +query T rowsort ( SELECT name FROM t1 EXCEPT @@ -53,11 +53,11 @@ UNION ALL SELECT name FROM t1 ) ---- -John Alice +John # union with type coercion -query IT +query IT rowsort ( SELECT * FROM t1 EXCEPT @@ -178,7 +178,7 @@ SELECT 1 UNION SELECT 2 2 # union_with_except_input -query T +query T rowsort ( SELECT name FROM t1 EXCEPT @@ -191,20 +191,20 @@ UNION ALL SELECT name FROM t1 ) ---- -John Alice +John # nested_union -query T +query T rowsort SELECT name FROM t1 UNION (SELECT name from t2 UNION SELECT name || '_new' from t2) ---- -Bob -Alex_new Alex -John -John_new +Alex_new Alice +Bob Bob_new +John +John_new # should be un-nested, with a single (logical) aggregate query TT @@ -230,18 +230,18 @@ physical_plan 10)--------------MemoryExec: partitions=1, partition_sizes=[1] # nested_union_all -query T +query T rowsort SELECT name FROM t1 UNION ALL (SELECT name from t2 UNION ALL SELECT name || '_new' from t2) ---- Alex -Bob -Alice -Alex_new -Bob_new -John_new Alex +Alex_new +Alice +Bob Bob +Bob_new John +John_new # Plan is unnested query TT @@ -323,7 +323,7 @@ physical_plan 30)--------------MemoryExec: partitions=1, partition_sizes=[1] -query IT +query IT rowsort ( SELECT id, name FROM t1 EXCEPT From 7950efd437bec365aa856e24362f49801403557c Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 20:15:11 +0300 Subject: [PATCH 07/15] fix slt tests --- datafusion/sqllogictest/test_files/expr.slt | 10 +++++++ .../sqllogictest/test_files/group_by.slt | 2 +- datafusion/sqllogictest/test_files/math.slt | 30 +++++++++---------- .../sqllogictest/test_files/strings.slt | 18 +++++------ 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 1b19a16ecbfc..9872aaa2002a 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -1804,11 +1804,21 @@ SELECT decode('','base64'); ---- +query T +SELECT encode('','hex'); +---- +(empty) + query ? SELECT decode('','hex'); ---- +query T +SELECT md5('tom'); +---- +34b7da764b21d298ef307d04d8152dc5 + query ? SELECT digest('tom','md5'); ---- diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt index f542218d5f97..7578d8f3c329 100644 --- a/datafusion/sqllogictest/test_files/group_by.slt +++ b/datafusion/sqllogictest/test_files/group_by.slt @@ -4542,7 +4542,7 @@ LOCATION 'test_files/scratch/group_by/timestamp_table' OPTIONS ('format.has_header' 'true'); # Group By using date_trunc -query PI +query PI rowsort SELECT date_trunc('week', t1) as week, sum(c2) FROM timestamp_table GROUP BY date_trunc('week', t1) diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index ae1a5c78de7c..aae0b1092a1d 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -288,7 +288,7 @@ INSERT INTO test_non_nullable_integer VALUES(1, 1, 1, 1, 1, 1, 1, 1) ---- 1 -query IIIIIIII +query IIIIIIII rowsort select c1*0, c2*0, c3*0, c4*0, c5*0, c6*0, c7*0, c8*0 from test_non_nullable_integer ---- 0 0 0 0 0 0 0 0 @@ -356,16 +356,16 @@ CREATE TABLE test_nullable_float( (0., 0.), ('NaN'::double, 'NaN'::double); -query RR +query RR rowsort SELECT c1*0, c2*0 FROM test_nullable_float ---- 0 0 0 0 -NULL NULL 0 0 +NULL NULL NaN NaN -query RR +query RR rowsort SELECT c1/0, c2/0 FROM test_nullable_float ---- -Infinity -Infinity @@ -374,38 +374,38 @@ NULL NULL NaN NaN NaN NaN -query RR +query RR rowsort SELECT c1%0, c2%0 FROM test_nullable_float ---- +NULL NULL NaN NaN NaN NaN -NULL NULL NaN NaN NaN NaN -query RR +query RR rowsort SELECT c1%1, c2%1 FROM test_nullable_float ---- 0 0 0 0 -NULL NULL 0 0 +NULL NULL NaN NaN # abs: return type -query TT +query TT rowsort SELECT arrow_typeof(abs(c1)), arrow_typeof(abs(c2)) FROM test_nullable_float limit 1 ---- Float32 Float64 # abs: floats -query RR +query RR rowsort SELECT abs(c1), abs(c2) from test_nullable_float ---- +0 0 1 1 1 1 NULL NULL -0 0 NaN NaN statement ok @@ -427,7 +427,7 @@ INSERT INTO test_non_nullable_float VALUES ---- 4 -query RR +query RR rowsort SELECT c1*0, c2*0 FROM test_non_nullable_float ---- 0 0 @@ -435,7 +435,7 @@ SELECT c1*0, c2*0 FROM test_non_nullable_float 0 0 NaN NaN -query RR +query RR rowsort SELECT c1/0, c2/0 FROM test_non_nullable_float ---- -Infinity -Infinity @@ -443,7 +443,7 @@ Infinity Infinity NaN NaN NaN NaN -query RR +query RR rowsort SELECT c1%0, c2%0 FROM test_non_nullable_float ---- NaN NaN @@ -451,7 +451,7 @@ NaN NaN NaN NaN NaN NaN -query RR +query RR rowsort SELECT c1%1, c2%1 FROM test_non_nullable_float ---- 0 0 diff --git a/datafusion/sqllogictest/test_files/strings.slt b/datafusion/sqllogictest/test_files/strings.slt index cba6301c0c46..30fb2d750d95 100644 --- a/datafusion/sqllogictest/test_files/strings.slt +++ b/datafusion/sqllogictest/test_files/strings.slt @@ -32,22 +32,22 @@ CREATE TABLE test( ; # LIKE -query T +query T rowsort SELECT s FROM test WHERE s LIKE 'p1%'; ---- p1 p1e1 p1m1e1 -query T +query T rowsort SELECT s FROM test WHERE s LIKE '%m1%'; ---- -p1m1e1 P1m1e1 +p1m1e1 p2m1e1 # NOT LIKE -query T +query T rowsort SELECT s FROM test WHERE s NOT LIKE 'p1%'; ---- P1 @@ -60,18 +60,18 @@ p2m1e1 # ILIKE -query T +query T rowsort SELECT s FROM test WHERE s ILIKE 'p1%'; ---- -p1 -p1e1 -p1m1e1 P1 P1e1 P1m1e1 +p1 +p1e1 +p1m1e1 # NOT ILIKE -query T +query T rowsort SELECT s FROM test WHERE s NOT ILIKE 'p1%'; ---- e1 From fb9dc36b298461a05bcde89c9d689f3802c12f22 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 21:32:00 +0300 Subject: [PATCH 08/15] rerun CI From 2fb8c3d2a828ce0af4ed235adb178c3fff6d4fc8 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sat, 13 Jul 2024 22:05:31 +0300 Subject: [PATCH 09/15] reset unchanged slt files --- .../sqllogictest/test_files/aggregate.slt | 28 ++--- .../sqllogictest/test_files/coalesce.slt | 13 ++- datafusion/sqllogictest/test_files/expr.slt | 110 +++++++++--------- .../sqllogictest/test_files/group_by.slt | 2 +- datafusion/sqllogictest/test_files/math.slt | 82 ++++++------- .../sqllogictest/test_files/predicates.slt | 2 +- datafusion/sqllogictest/test_files/scalar.slt | 2 +- datafusion/sqllogictest/test_files/select.slt | 4 +- datafusion/sqllogictest/test_files/union.slt | 28 ++--- datafusion/sqllogictest/test_files/window.slt | 2 +- 10 files changed, 137 insertions(+), 136 deletions(-) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index bc983c7f358b..a0140b1c5292 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1966,7 +1966,7 @@ drop table t; # test count with largeutf8 statement ok -create table t (c string) as values +create table t (c string) as values (arrow_cast('a', 'LargeUtf8')), (arrow_cast('b', 'LargeUtf8')), (arrow_cast(null, 'LargeUtf8')), @@ -4137,7 +4137,7 @@ statement ok create table t (c1 decimal(10, 0), c2 int) as values (null, null), (null, null), (null, null); query RTIT -select +select sum(c1), arrow_typeof(sum(c1)), sum(c2), arrow_typeof(sum(c2)) from t; @@ -4771,7 +4771,7 @@ NULL NULL 3 NULL 1 4 0 8 0 # regr_*() basic tests query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4786,7 +4786,7 @@ from (values (1,2), (2,4), (3,6)); 2 0 3 1 2 4 2 8 4 query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -4804,7 +4804,7 @@ from aggregate_test_100; # regr_*() functions ignore NULLs query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4819,7 +4819,7 @@ from (values (1,NULL), (2,4), (3,6)); 2 0 2 1 2.5 5 0.5 2 1 query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4834,7 +4834,7 @@ from (values (1,NULL), (NULL,4), (3,6)); NULL NULL 1 NULL 3 6 0 0 0 query RRRRRRRRR -select +select regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4849,8 +4849,8 @@ from (values (1,NULL), (NULL,4), (NULL,NULL)); NULL NULL 0 NULL NULL NULL NULL NULL NULL query TRRRRRRRRR rowsort -select - column3, +select + column3, regr_slope(column2, column1), regr_intercept(column2, column1), regr_count(column2, column1), @@ -4874,7 +4874,7 @@ statement ok set datafusion.execution.batch_size = 1; query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -4892,7 +4892,7 @@ statement ok set datafusion.execution.batch_size = 2; query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -4910,7 +4910,7 @@ statement ok set datafusion.execution.batch_size = 3; query RRRRRRRRR -select +select regr_slope(c12, c11), regr_intercept(c12, c11), regr_count(c12, c11), @@ -5064,7 +5064,7 @@ CREATE TABLE float_table ( # Test string_agg with largeutf8 statement ok -create table string_agg_large_utf8 (c string) as values +create table string_agg_large_utf8 (c string) as values (arrow_cast('a', 'LargeUtf8')), (arrow_cast('b', 'LargeUtf8')), (arrow_cast('c', 'LargeUtf8')) @@ -5119,7 +5119,7 @@ select count(*) from (select count(*) a, count(*) b from (select 1)); # UTF8 string matters for string to &[u8] conversion, add it to prevent regression statement ok -create table distinct_count_string_table as values +create table distinct_count_string_table as values (1, 'a', 'longstringtest_a', '台灣'), (2, 'b', 'longstringtest_b1', '日本'), (2, 'b', 'longstringtest_b2', '中國'), diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.slt index 8671c49aa49e..d16b79734c62 100644 --- a/datafusion/sqllogictest/test_files/coalesce.slt +++ b/datafusion/sqllogictest/test_files/coalesce.slt @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. + # Test coalesce function query I select coalesce(1, 2, 3); @@ -246,7 +247,7 @@ statement ok create table t(c varchar) as values ('a'), (null); query TT -select +select coalesce(c, arrow_cast('b', 'Dictionary(Int32, Utf8)')), arrow_typeof(coalesce(c, arrow_cast('b', 'Dictionary(Int32, Utf8)'))) from t; @@ -259,12 +260,12 @@ drop table t; # test dict coercion with dict statement ok -create table t as values - (arrow_cast('foo', 'Dictionary(Int32, Utf8)')), +create table t as values + (arrow_cast('foo', 'Dictionary(Int32, Utf8)')), (null); query ?T -select +select coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, LargeUtf8)')), arrow_typeof(coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, LargeUtf8)'))) from t; @@ -273,7 +274,7 @@ foo Dictionary(Int64, LargeUtf8) bar Dictionary(Int64, LargeUtf8) query ?T -select +select coalesce(column1, arrow_cast('bar', 'Dictionary(Int32, LargeUtf8)')), arrow_typeof(coalesce(column1, arrow_cast('bar', 'Dictionary(Int32, LargeUtf8)'))) from t; @@ -282,7 +283,7 @@ foo Dictionary(Int32, LargeUtf8) bar Dictionary(Int32, LargeUtf8) query ?T -select +select coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, Utf8)')), arrow_typeof(coalesce(column1, arrow_cast('bar', 'Dictionary(Int64, Utf8)'))) from t; diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 9872aaa2002a..b08d329d4a86 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -504,7 +504,7 @@ NULL query T SELECT ltrim(' zzzytest ') ---- -zzzytest +zzzytest query T SELECT ltrim('zzzytest', 'xyz') @@ -682,7 +682,7 @@ tom query T SELECT trim(LEADING ' tom ') ---- -tom +tom query T SELECT trim(TRAILING ' tom ') @@ -697,7 +697,7 @@ tom query T SELECT trim(LEADING ' ' FROM ' tom ') ---- -tom +tom query T SELECT trim(TRAILING ' ' FROM ' tom ') @@ -803,7 +803,7 @@ NULL # test_random_expression query BB -SELECT +SELECT random() BETWEEN 0.0 AND 1.0, random() = random() ---- @@ -1802,7 +1802,7 @@ SELECT encode('','base64'); query ? SELECT decode('','base64'); ---- - + query T SELECT encode('','hex'); @@ -1812,7 +1812,7 @@ SELECT encode('','hex'); query ? SELECT decode('','hex'); ---- - + query T SELECT md5('tom'); @@ -2452,20 +2452,20 @@ host3 3.3 # can have an aggregate function with an inner CASE WHEN query TR -select - t2.server_host as host, +select + t2.server_host as host, sum(( - case when t2.server_host is not null + case when t2.server_host is not null then t2.server_load2 end - )) + )) from ( - select + select struct(time,load1,load2,host)['c2'] as server_load2, struct(time,load1,load2,host)['c3'] as server_host from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; ---- host1 101 @@ -2474,19 +2474,19 @@ host3 303 # TODO: Issue tracked in https://github.com/apache/datafusion/issues/10364 query TR -select - t2.server['c3'] as host, +select + t2.server['c3'] as host, sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c2'] end - )) + )) from ( - select + select struct(time,load1,load2,host) as server from t1 - ) t2 - where t2.server['c3'] IS NOT NULL + ) t2 + where t2.server['c3'] IS NOT NULL group by t2.server['c3'] order by host; ---- host1 101 @@ -2495,22 +2495,22 @@ host3 303 # can have 2 projections with aggr(short_circuited), with different short-circuited expr query TRR -select - t2.server_host as host, +select + t2.server_host as host, sum(coalesce(server_load1)), sum(( - case when t2.server_host is not null + case when t2.server_host is not null then t2.server_load2 end - )) + )) from ( - select + select struct(time,load1,load2,host)['c1'] as server_load1, struct(time,load1,load2,host)['c2'] as server_load2, struct(time,load1,load2,host)['c3'] as server_host from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; ---- host1 1.1 101 @@ -2519,43 +2519,43 @@ host3 3.3 303 # TODO: Issue tracked in https://github.com/apache/datafusion/issues/10364 query error -select - t2.server['c3'] as host, +select + t2.server['c3'] as host, sum(coalesce(server['c1'])), sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c2'] end - )) + )) from ( - select + select struct(time,load1,load2,host) as server, from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; query TRR -select - t2.server_host as host, +select + t2.server_host as host, sum(( - case when t2.server_host is not null - then server_load1 + case when t2.server_host is not null + then server_load1 end - )), + )), sum(( - case when server_host is not null - then server_load2 + case when server_host is not null + then server_load2 end - )) + )) from ( - select + select struct(time,load1,load2,host)['c1'] as server_load1, struct(time,load1,load2,host)['c2'] as server_load2, struct(time,load1,load2,host)['c3'] as server_host from t1 - ) t2 - where server_host IS NOT NULL + ) t2 + where server_host IS NOT NULL group by server_host order by host; ---- host1 1.1 101 @@ -2564,24 +2564,24 @@ host3 3.3 303 # TODO: Issue tracked in https://github.com/apache/datafusion/issues/10364 query TRR -select - t2.server['c3'] as host, +select + t2.server['c3'] as host, sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c1'] end - )), + )), sum(( - case when t2.server['c3'] is not null + case when t2.server['c3'] is not null then t2.server['c2'] end - )) + )) from ( - select - struct(time,load1,load2,host) as server + select + struct(time,load1,load2,host) as server from t1 - ) t2 - where t2.server['c3'] IS NOT NULL + ) t2 + where t2.server['c3'] IS NOT NULL group by t2.server['c3'] order by host; ---- host1 1.1 101 diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt index 7578d8f3c329..b2be65a609e3 100644 --- a/datafusion/sqllogictest/test_files/group_by.slt +++ b/datafusion/sqllogictest/test_files/group_by.slt @@ -5141,7 +5141,7 @@ statement ok CREATE TABLE test_case_expr(a INT, b TEXT) AS VALUES (1,'hello'), (2,'world') query T -SELECT (CASE WHEN CONCAT(b, 'hello') = 'test' THEN 'good' ELSE 'bad' END) AS c +SELECT (CASE WHEN CONCAT(b, 'hello') = 'test' THEN 'good' ELSE 'bad' END) AS c FROM test_case_expr GROUP BY c; ---- bad diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index aae0b1092a1d..6ff804c3065d 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -127,16 +127,16 @@ SELECT abs('foo'); statement ok CREATE TABLE test_nullable_integer( - c1 TINYINT, - c2 SMALLINT, - c3 INT, - c4 BIGINT, - c5 TINYINT UNSIGNED, - c6 SMALLINT UNSIGNED, - c7 INT UNSIGNED, - c8 BIGINT UNSIGNED, + c1 TINYINT, + c2 SMALLINT, + c3 INT, + c4 BIGINT, + c5 TINYINT UNSIGNED, + c6 SMALLINT UNSIGNED, + c7 INT UNSIGNED, + c8 BIGINT UNSIGNED, dataset TEXT - ) + ) AS VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'nulls'), (0, 0, 0, 0, 0, 0, 0, 0, 'zeros'), @@ -225,7 +225,7 @@ SELECT c8%0 FROM test_nullable_integer # abs: return type query TTTTTTTT rowsort -select +select arrow_typeof(abs(c1)), arrow_typeof(abs(c2)), arrow_typeof(abs(c3)), arrow_typeof(abs(c4)), arrow_typeof(abs(c5)), arrow_typeof(abs(c6)), arrow_typeof(abs(c7)), arrow_typeof(abs(c8)) from test_nullable_integer limit 1 @@ -273,13 +273,13 @@ drop table test_nullable_integer statement ok CREATE TABLE test_non_nullable_integer( - c1 TINYINT NOT NULL, - c2 SMALLINT NOT NULL, - c3 INT NOT NULL, - c4 BIGINT NOT NULL, - c5 TINYINT UNSIGNED NOT NULL, - c6 SMALLINT UNSIGNED NOT NULL, - c7 INT UNSIGNED NOT NULL, + c1 TINYINT NOT NULL, + c2 SMALLINT NOT NULL, + c3 INT NOT NULL, + c4 BIGINT NOT NULL, + c5 TINYINT UNSIGNED NOT NULL, + c6 SMALLINT UNSIGNED NOT NULL, + c7 INT UNSIGNED NOT NULL, c8 BIGINT UNSIGNED NOT NULL ); @@ -351,7 +351,7 @@ CREATE TABLE test_nullable_float( c2 double ) AS VALUES (-1.0, -1.0), - (1.0, 1.0), + (1.0, 1.0), (NULL, NULL), (0., 0.), ('NaN'::double, 'NaN'::double); @@ -400,7 +400,7 @@ Float32 Float64 # abs: floats query RR rowsort -SELECT abs(c1), abs(c2) from test_nullable_float +SELECT abs(c1), abs(c2) from test_nullable_float ---- 0 0 1 1 @@ -416,7 +416,7 @@ statement ok CREATE TABLE test_non_nullable_float( c1 float NOT NULL, c2 double NOT NULL - ); + ); query I INSERT INTO test_non_nullable_float VALUES @@ -466,27 +466,27 @@ drop table test_non_nullable_float statement ok CREATE TABLE test_nullable_decimal( c1 DECIMAL(10, 2), /* Decimal128 */ - c2 DECIMAL(38, 10), /* Decimal128 with max precision */ + c2 DECIMAL(38, 10), /* Decimal128 with max precision */ c3 DECIMAL(40, 2), /* Decimal256 */ - c4 DECIMAL(76, 10) /* Decimal256 with max precision */ - ) AS VALUES - (0, 0, 0, 0), + c4 DECIMAL(76, 10) /* Decimal256 with max precision */ + ) AS VALUES + (0, 0, 0, 0), (NULL, NULL, NULL, NULL); query I INSERT into test_nullable_decimal values ( - -99999999.99, - '-9999999999999999999999999999.9999999999', - '-99999999999999999999999999999999999999.99', + -99999999.99, + '-9999999999999999999999999999.9999999999', + '-99999999999999999999999999999999999999.99', '-999999999999999999999999999999999999999999999999999999999999999999.9999999999' - ), + ), ( - 99999999.99, - '9999999999999999999999999999.9999999999', - '99999999999999999999999999999999999999.99', + 99999999.99, + '9999999999999999999999999999.9999999999', + '99999999999999999999999999999999999999.99', '999999999999999999999999999999999999999999999999999999999999999999.9999999999' - ) + ) ---- 2 @@ -521,9 +521,9 @@ SELECT c1%0 FROM test_nullable_decimal WHERE c1 IS NOT NULL; # abs: return type query TTTT -SELECT - arrow_typeof(abs(c1)), - arrow_typeof(abs(c2)), +SELECT + arrow_typeof(abs(c1)), + arrow_typeof(abs(c2)), arrow_typeof(abs(c3)), arrow_typeof(abs(c4)) FROM test_nullable_decimal limit 1 @@ -540,11 +540,11 @@ SELECT abs(c1), abs(c2), abs(c3), abs(c4) FROM test_nullable_decimal NULL NULL NULL NULL statement ok -drop table test_nullable_decimal +drop table test_nullable_decimal statement ok -CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL); +CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL); query I INSERT INTO test_non_nullable_decimal VALUES(1) @@ -557,13 +557,13 @@ SELECT c1*0 FROM test_non_nullable_decimal 0 query error DataFusion error: Arrow error: Divide by zero error -SELECT c1/0 FROM test_non_nullable_decimal +SELECT c1/0 FROM test_non_nullable_decimal query error DataFusion error: Arrow error: Divide by zero error -SELECT c1%0 FROM test_non_nullable_decimal +SELECT c1%0 FROM test_non_nullable_decimal statement ok -drop table test_non_nullable_decimal +drop table test_non_nullable_decimal statement ok CREATE TABLE signed_integers( @@ -603,7 +603,7 @@ NULL NULL NULL # scalar maxes and/or negative 1 query III -select +select gcd(9223372036854775807, -9223372036854775808), -- i64::MAX, i64::MIN gcd(9223372036854775807, -1), -- i64::MAX, -1 gcd(-9223372036854775808, -1); -- i64::MIN, -1 diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.slt index d6b8579535c0..4695e37aa560 100644 --- a/datafusion/sqllogictest/test_files/predicates.slt +++ b/datafusion/sqllogictest/test_files/predicates.slt @@ -39,7 +39,7 @@ CREATE EXTERNAL TABLE aggregate_test_100 ( c13 VARCHAR NOT NULL ) STORED AS CSV -LOCATION '../../testing/data/csv/aggregate_test_100.csv' +LOCATION '../../testing/data/csv/aggregate_test_100.csv' OPTIONS ('format.has_header' 'true'); statement ok diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 10785d209436..5daa9333fb36 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -1846,7 +1846,7 @@ D false # test string_temporal_coercion query BBBBBBBBBB -select +select arrow_cast(to_timestamp('2020-01-01 01:01:11.1234567890Z'), 'Timestamp(Second, None)') == '2020-01-01T01:01:11', arrow_cast(to_timestamp('2020-01-02 01:01:11.1234567890Z'), 'Timestamp(Second, None)') == arrow_cast('2020-01-02T01:01:11', 'LargeUtf8'), arrow_cast(to_timestamp('2020-01-03 01:01:11.1234567890Z'), 'Time32(Second)') == '01:01:11', diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index 9165ee21cf19..03426dec874f 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -402,7 +402,7 @@ VALUES (1,2,3,4,5,6,7,8,9,10,11,12,13,NULL,'F',3.5) # Test non-literal expressions in VALUES query II -VALUES (1, CASE WHEN RANDOM() > 0.5 THEN 1 ELSE 1 END), +VALUES (1, CASE WHEN RANDOM() > 0.5 THEN 1 ELSE 1 END), (2, CASE WHEN RANDOM() > 0.5 THEN 2 ELSE 2 END); ---- 1 1 @@ -1479,7 +1479,7 @@ query II SELECT CASE WHEN B.x > 0 THEN A.x / B.x ELSE 0 END AS value1, CASE WHEN B.x > 0 AND B.y > 0 THEN A.x / B.x ELSE 0 END AS value3 -FROM t AS A, (SELECT * FROM t WHERE x = 0) AS B; +FROM t AS A, (SELECT * FROM t WHERE x = 0) AS B; ---- 0 0 0 0 diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index 56732b3f58ad..31b16f975e9e 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -21,11 +21,11 @@ statement ok CREATE TABLE t1( - id INT, + id INT, name TEXT ) as VALUES - (1, 'Alex'), - (2, 'Bob'), + (1, 'Alex'), + (2, 'Bob'), (3, 'Alice') ; @@ -34,20 +34,20 @@ CREATE TABLE t2( id TINYINT, name TEXT ) as VALUES - (1, 'Alex'), - (2, 'Bob'), + (1, 'Alex'), + (2, 'Bob'), (3, 'John') ; # union with EXCEPT(JOIN) query T rowsort -( +( SELECT name FROM t1 EXCEPT SELECT name FROM t2 -) +) UNION ALL -( +( SELECT name FROM t2 EXCEPT SELECT name FROM t1 @@ -58,13 +58,13 @@ John # union with type coercion query IT rowsort -( +( SELECT * FROM t1 EXCEPT SELECT * FROM t2 -) +) UNION ALL -( +( SELECT * FROM t2 EXCEPT SELECT * FROM t1 @@ -518,11 +518,11 @@ OPTIONS ('format.has_header' 'true'); query TT explain SELECT c1 FROM( -( +( SELECT c1 FROM t1 -) +) UNION ALL -( +( SELECT c1a FROM t2 )) ORDER BY c1 diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index bb52b027a8e2..6d2c334b47ac 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -1766,7 +1766,7 @@ logical_plan 01)Projection: count(*) AS global_count 02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1)) AS count(*)]] 03)----SubqueryAlias: a -04)------Projection: +04)------Projection: 05)--------Sort: aggregate_test_100.c1 ASC NULLS LAST 06)----------Aggregate: groupBy=[[aggregate_test_100.c1]], aggr=[[]] 07)------------Projection: aggregate_test_100.c1 From 4fc036a9112528ec96926df93b1301465829bbcc Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sun, 14 Jul 2024 00:07:16 +0300 Subject: [PATCH 10/15] Revert "clean imports and qualified imports" This reverts commit 7be2263793be7730615c52fec79ca3397eefb40f. --- datafusion/sql/src/statement.rs | 50 +++++++++++++++++---------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 6df25086305d..ee050381a73d 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -46,19 +46,18 @@ use datafusion_expr::{ cast, col, Analyze, CreateCatalog, CreateCatalogSchema, CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody, CreateMemoryTable, CreateView, DescribeTable, DmlStatement, DropCatalogSchema, - DropFunction, DropTable, DropView, EmptyRelation, Explain, Expr, ExprSchemable, - Filter, LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, - SetVariable, Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, + DropFunction, DropTable, DropView, EmptyRelation, Explain, ExprSchemable, Filter, + LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, SetVariable, + Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, }; -use sqlparser::ast; +use sqlparser::ast::{self, AssignmentTarget, CreateTable}; use sqlparser::ast::{ - Assignment, AssignmentTarget, ColumnDef, CreateTable, CreateTableOptions, Delete, - DescribeAlias, Expr as SQLExpr, FromTable, Ident, Insert, ObjectName, ObjectType, - OneOrManyWithParens, Query, SchemaName, SetExpr, ShowCreateObject, - ShowStatementFilter, Statement, TableConstraint, TableFactor, TableWithJoins, - TransactionMode, UnaryOperator, Value, + Assignment, ColumnDef, CreateTableOptions, Delete, DescribeAlias, Expr as SQLExpr, + Expr, FromTable, Ident, Insert, ObjectName, ObjectType, OneOrManyWithParens, Query, + SchemaName, SetExpr, ShowCreateObject, ShowStatementFilter, Statement, + TableConstraint, TableFactor, TableWithJoins, TransactionMode, UnaryOperator, Value, }; use sqlparser::parser::ParserError::ParserError; @@ -955,7 +954,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { order_exprs: Vec, schema: &DFSchemaRef, planner_context: &mut PlannerContext, - ) -> Result>> { + ) -> Result>> { // Ask user to provide a schema if schema is empty. if !order_exprs.is_empty() && schema.fields().is_empty() { return plan_err!( @@ -1160,7 +1159,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { local: bool, hivevar: bool, variables: &OneOrManyWithParens, - value: Vec, + value: Vec, ) -> Result { if local { return not_impl_err!("LOCAL is not supported"); @@ -1219,7 +1218,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { fn delete_to_plan( &self, table_name: ObjectName, - predicate_expr: Option, + predicate_expr: Option, ) -> Result { // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(table_name.clone())?; @@ -1265,7 +1264,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { table: TableWithJoins, assignments: Vec, from: Option, - predicate_expr: Option, + predicate_expr: Option, ) -> Result { let (table_name, table_alias) = match &table.relation { TableFactor::Table { name, alias, .. } => (name.clone(), alias.clone()), @@ -1298,7 +1297,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { table_schema.field_with_unqualified_name(&col_name.value)?; Ok((col_name.value.clone(), assign.value.clone())) }) - .collect::>>()?; + .collect::>>()?; // Build scan, join with from table if it exists. let mut input_tables = vec![table]; @@ -1337,7 +1336,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &mut planner_context, )?; // Update placeholder's datatype to the type of the target column - if let Expr::Placeholder(placeholder) = &mut expr { + if let datafusion_expr::Expr::Placeholder(placeholder) = &mut expr + { placeholder.data_type = placeholder .data_type .take() @@ -1349,12 +1349,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { None => { // If the target table has an alias, use it to qualify the column name if let Some(alias) = &table_alias { - Expr::Column(Column::new( + datafusion_expr::Expr::Column(Column::new( Some(self.normalizer.normalize(alias.name.clone())), field.name(), )) } else { - Expr::Column(Column::from((qualifier, field))) + datafusion_expr::Expr::Column(Column::from(( + qualifier, field, + ))) } } }; @@ -1429,7 +1431,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { if let SetExpr::Values(ast::Values { rows, .. }) = (*source.body).clone() { for row in rows.iter() { for (idx, val) in row.iter().enumerate() { - if let SQLExpr::Value(Value::Placeholder(name)) = val { + if let ast::Expr::Value(Value::Placeholder(name)) = val { let name = name.replace('$', "").parse::().map_err(|_| { plan_datafusion_err!("Can't parse placeholder: {name}") @@ -1462,23 +1464,23 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { .map(|(i, value_index)| { let target_field = table_schema.field(i); let expr = match value_index { - Some(v) => { - Expr::Column(Column::from(source.schema().qualified_field(v))) - .cast_to(target_field.data_type(), source.schema())? - } + Some(v) => datafusion_expr::Expr::Column(Column::from( + source.schema().qualified_field(v), + )) + .cast_to(target_field.data_type(), source.schema())?, // The value is not specified. Fill in the default value for the column. None => table_source .get_column_default(target_field.name()) .cloned() .unwrap_or_else(|| { // If there is no default for the column, then the default is NULL - Expr::Literal(ScalarValue::Null) + datafusion_expr::Expr::Literal(ScalarValue::Null) }) .cast_to(target_field.data_type(), &DFSchema::empty())?, }; Ok(expr.alias(target_field.name())) }) - .collect::>>()?; + .collect::>>()?; let source = project(source, exprs)?; let op = if overwrite { From 17638048a78a6d757945a90825cc420173fa5f72 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sun, 14 Jul 2024 22:46:05 +0300 Subject: [PATCH 11/15] update non-windows systems stack size --- datafusion/sqllogictest/bin/sqllogictests.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 560328ee8619..6320a60e9ba3 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -51,10 +51,14 @@ pub fn main() { .unwrap(); } -#[tokio::main] #[cfg(not(target_family = "windows"))] -pub async fn main() -> Result<()> { - run_tests().await +fn main() -> Result<()> { + tokio::runtime::Builder::new_multi_thread() + .thread_stack_size(2 * 1024 * 1024 + 512 * 1024) + .enable_all() + .build() + .unwrap() + .block_on(run_tests()) } /// Sets up an empty directory at test_files/scratch/ From 3a7cfe6a7a1138034cb1d1f6f1c04b5631a52a65 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sun, 14 Jul 2024 22:48:02 +0300 Subject: [PATCH 12/15] update windows stack size --- datafusion/sqllogictest/bin/sqllogictests.rs | 22 +++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 6320a60e9ba3..646fcad38d4f 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -34,25 +34,17 @@ const TEST_DIRECTORY: &str = "test_files/"; const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_"; #[cfg(target_family = "windows")] -pub fn main() { - // Tests from `tpch/tpch.slt` fail with stackoverflow with the default stack size. - thread::Builder::new() - .stack_size(2 * 1024 * 1024) // 2 MB - .spawn(move || { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap() - .block_on(async { run_tests().await }) - .unwrap() - }) +pub fn main() -> Result<()> { + tokio::runtime::Builder::new_multi_thread() + .thread_stack_size(2 * 1024 * 1024 + 512 * 1024) + .enable_all() + .build() .unwrap() - .join() - .unwrap(); + .block_on(run_tests()) } #[cfg(not(target_family = "windows"))] -fn main() -> Result<()> { +pub fn main() -> Result<()> { tokio::runtime::Builder::new_multi_thread() .thread_stack_size(2 * 1024 * 1024 + 512 * 1024) .enable_all() From d04948d9765440f3bd59300aa5dc7db32fc0862b Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sun, 14 Jul 2024 22:57:53 +0300 Subject: [PATCH 13/15] remove windows-only unused import --- datafusion/sqllogictest/bin/sqllogictests.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 646fcad38d4f..999291569620 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -18,8 +18,6 @@ use std::ffi::OsStr; use std::fs; use std::path::{Path, PathBuf}; -#[cfg(target_family = "windows")] -use std::thread; use clap::Parser; use datafusion_sqllogictest::{DataFusion, TestContext}; From 6cf09d39cee95f8c616d04f4515dc0bb2568e791 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sun, 14 Jul 2024 23:18:22 +0300 Subject: [PATCH 14/15] use same test main for all systems --- datafusion/sqllogictest/bin/sqllogictests.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 999291569620..8c8ed2e58743 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -30,21 +30,11 @@ use datafusion_common_runtime::SpawnedTask; const TEST_DIRECTORY: &str = "test_files/"; const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_"; +const STACK_SIZE: usize = 2 * 1024 * 1024 + 512 * 1024; // 2.5 MBs, the default 2 MBs is currently too small -#[cfg(target_family = "windows")] pub fn main() -> Result<()> { tokio::runtime::Builder::new_multi_thread() - .thread_stack_size(2 * 1024 * 1024 + 512 * 1024) - .enable_all() - .build() - .unwrap() - .block_on(run_tests()) -} - -#[cfg(not(target_family = "windows"))] -pub fn main() -> Result<()> { - tokio::runtime::Builder::new_multi_thread() - .thread_stack_size(2 * 1024 * 1024 + 512 * 1024) + .thread_stack_size(STACK_SIZE) .enable_all() .build() .unwrap() From 0586d5ead8295695ef82a12fdc670ee161c16b15 Mon Sep 17 00:00:00 2001 From: Mohamed Abdeen Date: Sun, 14 Jul 2024 23:50:15 +0300 Subject: [PATCH 15/15] Reapply "clean imports and qualified imports" This reverts commit 4fc036a9112528ec96926df93b1301465829bbcc. --- datafusion/sql/src/statement.rs | 50 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index ee050381a73d..6df25086305d 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -46,18 +46,19 @@ use datafusion_expr::{ cast, col, Analyze, CreateCatalog, CreateCatalogSchema, CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody, CreateMemoryTable, CreateView, DescribeTable, DmlStatement, DropCatalogSchema, - DropFunction, DropTable, DropView, EmptyRelation, Explain, ExprSchemable, Filter, - LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, SetVariable, - Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, + DropFunction, DropTable, DropView, EmptyRelation, Explain, Expr, ExprSchemable, + Filter, LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, + SetVariable, Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, }; -use sqlparser::ast::{self, AssignmentTarget, CreateTable}; +use sqlparser::ast; use sqlparser::ast::{ - Assignment, ColumnDef, CreateTableOptions, Delete, DescribeAlias, Expr as SQLExpr, - Expr, FromTable, Ident, Insert, ObjectName, ObjectType, OneOrManyWithParens, Query, - SchemaName, SetExpr, ShowCreateObject, ShowStatementFilter, Statement, - TableConstraint, TableFactor, TableWithJoins, TransactionMode, UnaryOperator, Value, + Assignment, AssignmentTarget, ColumnDef, CreateTable, CreateTableOptions, Delete, + DescribeAlias, Expr as SQLExpr, FromTable, Ident, Insert, ObjectName, ObjectType, + OneOrManyWithParens, Query, SchemaName, SetExpr, ShowCreateObject, + ShowStatementFilter, Statement, TableConstraint, TableFactor, TableWithJoins, + TransactionMode, UnaryOperator, Value, }; use sqlparser::parser::ParserError::ParserError; @@ -954,7 +955,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { order_exprs: Vec, schema: &DFSchemaRef, planner_context: &mut PlannerContext, - ) -> Result>> { + ) -> Result>> { // Ask user to provide a schema if schema is empty. if !order_exprs.is_empty() && schema.fields().is_empty() { return plan_err!( @@ -1159,7 +1160,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { local: bool, hivevar: bool, variables: &OneOrManyWithParens, - value: Vec, + value: Vec, ) -> Result { if local { return not_impl_err!("LOCAL is not supported"); @@ -1218,7 +1219,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { fn delete_to_plan( &self, table_name: ObjectName, - predicate_expr: Option, + predicate_expr: Option, ) -> Result { // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(table_name.clone())?; @@ -1264,7 +1265,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { table: TableWithJoins, assignments: Vec, from: Option, - predicate_expr: Option, + predicate_expr: Option, ) -> Result { let (table_name, table_alias) = match &table.relation { TableFactor::Table { name, alias, .. } => (name.clone(), alias.clone()), @@ -1297,7 +1298,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { table_schema.field_with_unqualified_name(&col_name.value)?; Ok((col_name.value.clone(), assign.value.clone())) }) - .collect::>>()?; + .collect::>>()?; // Build scan, join with from table if it exists. let mut input_tables = vec![table]; @@ -1336,8 +1337,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &mut planner_context, )?; // Update placeholder's datatype to the type of the target column - if let datafusion_expr::Expr::Placeholder(placeholder) = &mut expr - { + if let Expr::Placeholder(placeholder) = &mut expr { placeholder.data_type = placeholder .data_type .take() @@ -1349,14 +1349,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { None => { // If the target table has an alias, use it to qualify the column name if let Some(alias) = &table_alias { - datafusion_expr::Expr::Column(Column::new( + Expr::Column(Column::new( Some(self.normalizer.normalize(alias.name.clone())), field.name(), )) } else { - datafusion_expr::Expr::Column(Column::from(( - qualifier, field, - ))) + Expr::Column(Column::from((qualifier, field))) } } }; @@ -1431,7 +1429,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { if let SetExpr::Values(ast::Values { rows, .. }) = (*source.body).clone() { for row in rows.iter() { for (idx, val) in row.iter().enumerate() { - if let ast::Expr::Value(Value::Placeholder(name)) = val { + if let SQLExpr::Value(Value::Placeholder(name)) = val { let name = name.replace('$', "").parse::().map_err(|_| { plan_datafusion_err!("Can't parse placeholder: {name}") @@ -1464,23 +1462,23 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { .map(|(i, value_index)| { let target_field = table_schema.field(i); let expr = match value_index { - Some(v) => datafusion_expr::Expr::Column(Column::from( - source.schema().qualified_field(v), - )) - .cast_to(target_field.data_type(), source.schema())?, + Some(v) => { + Expr::Column(Column::from(source.schema().qualified_field(v))) + .cast_to(target_field.data_type(), source.schema())? + } // The value is not specified. Fill in the default value for the column. None => table_source .get_column_default(target_field.name()) .cloned() .unwrap_or_else(|| { // If there is no default for the column, then the default is NULL - datafusion_expr::Expr::Literal(ScalarValue::Null) + Expr::Literal(ScalarValue::Null) }) .cast_to(target_field.data_type(), &DFSchema::empty())?, }; Ok(expr.alias(target_field.name())) }) - .collect::>>()?; + .collect::>>()?; let source = project(source, exprs)?; let op = if overwrite {