From 43eea6a0372f9bc986cc04e257a68f4b8ae7375e Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 10 Aug 2024 12:34:46 -0400 Subject: [PATCH] Update to DataFrusion 38 and sqlparser 0.45 --- .github/workflows/build_test.yml | 6 +- Cargo.lock | 95 ++++++++++----- Cargo.toml | 18 +-- vegafusion-common/src/data/scalar.rs | 5 +- vegafusion-common/src/datatypes.rs | 4 +- vegafusion-datafusion-udfs/src/udafs/mod.rs | 8 +- .../src/udfs/math/isfinite.rs | 5 +- .../src/expression/compiler/binary.rs | 6 +- .../data/vl_selection_test.rs | 3 +- .../builtin_functions/date_time/date_parts.rs | 3 +- .../src/expression/compiler/call.rs | 63 ++++------ .../src/expression/compiler/member.rs | 8 +- .../src/expression/compiler/mod.rs | 111 +++++++----------- vegafusion-runtime/src/transform/aggregate.rs | 2 +- vegafusion-runtime/src/transform/bin.rs | 4 +- vegafusion-runtime/src/transform/pivot.rs | 3 +- vegafusion-runtime/src/transform/timeunit.rs | 3 +- vegafusion-runtime/src/transform/window.rs | 2 +- vegafusion-sql/Cargo.toml | 2 +- vegafusion-sql/src/compile/expr.rs | 11 +- .../src/connection/datafusion_conn.rs | 2 +- vegafusion-sql/src/dataframe/mod.rs | 50 +++----- .../src/dialect/transforms/date_part_tz.rs | 2 +- vegafusion-sql/tests/expected/select.toml | 20 ++-- vegafusion-sql/tests/test_aggregate.rs | 3 +- vegafusion-sql/tests/test_select.rs | 67 ++++------- vegafusion-sql/tests/test_stack.rs | 4 +- 27 files changed, 233 insertions(+), 277 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index d5dc6bee0..bcb407e42 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -319,7 +319,7 @@ jobs: ls -la python -m pip install vegafusion-*.whl python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl - python -m pip install pytest vega-datasets polars-lts-cpu duckdb==0.9.2 "vl-convert-python>=1.0.1rc1" scikit-image pandas==2.0 + python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" - name: Test lazy imports working-directory: python/vegafusion/ run: python checks/check_lazy_imports.py @@ -353,7 +353,7 @@ jobs: ls -la python -m pip install vegafusion-*.whl python -m pip install vegafusion_python_embed-*macosx_10_*_x86_64.whl - python -m pip install pytest vega-datasets polars-lts-cpu duckdb==0.9.2 vl-convert-python scikit-image pandas==2.0 + python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2" python -m pip install pyarrow==10.0 altair==5.1.2 - name: Test vegafusion working-directory: python/vegafusion/ @@ -389,7 +389,7 @@ jobs: python -m pip install $vegafusion python -m pip install $vegafusion_python_embed - python -m pip install pytest vega-datasets polars[timezone] duckdb==0.9.2 vl-convert-python scikit-image + python -m pip install pytest vega-datasets polars[timezone] "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2" - name: Test vegafusion working-directory: python/vegafusion/ run: pytest diff --git a/Cargo.lock b/Cargo.lock index 800b66d15..15a9a737b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,9 +1236,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11" +checksum = "05fb4eeeb7109393a0739ac5b8fd892f95ccef691421491c85544f7997366f68" dependencies = [ "ahash", "arrow", @@ -1256,6 +1256,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", @@ -1287,9 +1288,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3" +checksum = "741aeac15c82f239f2fc17deccaab19873abbd62987be20023689b15fa72fa09" dependencies = [ "ahash", "arrow", @@ -1309,18 +1310,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e4a44d8ef1b1e85d32234e6012364c411c3787859bb3bba893b0332cb03dfd" +checksum = "6e8ddfb8d8cb51646a30da0122ecfffb81ca16919ae9a3495a9e7468bdcd52b8" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95" +checksum = "282122f90b20e8f98ebfa101e4bf20e718fd2684cf81bef4e8c6366571c64404" dependencies = [ "arrow", "chrono", @@ -1339,9 +1340,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628" +checksum = "5478588f733df0dfd87a62671c7478f590952c95fa2fa5c137e3ff2929491e22" dependencies = [ "ahash", "arrow", @@ -1349,6 +1350,7 @@ dependencies = [ "chrono", "datafusion-common", "paste", + "serde_json", "sqlparser", "strum 0.26.1", "strum_macros 0.26.1", @@ -1356,9 +1358,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959" +checksum = "f4afd261cea6ac9c3ca1192fd5e9f940596d8e9208c5b1333f4961405db53185" dependencies = [ "arrow", "base64 0.22.1", @@ -1369,21 +1371,39 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "hashbrown 0.14.3", "hex", "itertools 0.12.1", "log", "md-5", + "rand", "regex", "sha2", "unicode-segmentation", "uuid", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "38.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b36a6c4838ab94b5bf8f7a96ce6ce059d805c5d1dcaa6ace49e034eb65cd999" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", + "paste", + "sqlparser", +] + [[package]] name = "datafusion-functions-array" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55" +checksum = "d5fdd200a6233f48d3362e7ccb784f926f759100e44ae2137a5e2dcb986a59c4" dependencies = [ "arrow", "arrow-array", @@ -1401,9 +1421,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a" +checksum = "54f2820938810e8a2d71228fd6f59f33396aebc5f5f687fcbf14de5aab6a7e1a" dependencies = [ "arrow", "async-trait", @@ -1412,6 +1432,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.3", + "indexmap 2.2.3", "itertools 0.12.1", "log", "regex-syntax", @@ -1419,9 +1440,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96" +checksum = "9adf8eb12716f52ddf01e09eb6c94d3c9b291e062c05c91b839a448bddba2ff8" dependencies = [ "ahash", "arrow", @@ -1431,37 +1452,45 @@ dependencies = [ "arrow-schema", "arrow-string", "base64 0.22.1", - "blake2", - "blake3", "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", "half", "hashbrown 0.14.3", "hex", "indexmap 2.2.3", "itertools 0.12.1", "log", - "md-5", "paste", "petgraph", - "rand", "regex", - "sha2", - "unicode-segmentation", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "38.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5472c3230584c150197b3f2c23f2392b9dc54dbfb62ad41e7e36447cfce4be" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", ] [[package]] name = "datafusion-physical-plan" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9" +checksum = "18ae750c38389685a8b62e5b899bbbec488950755ad6d218f3662d35b800c4fe" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "async-trait", "chrono", @@ -1469,7 +1498,9 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.3", @@ -1485,9 +1516,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db73393e42f35e165d31399192fbf41691967d428ebed47875ad34239fbcfc16" +checksum = "e6f4d2df0e7ba676fe9c0b7cbc0768ffc8f736600b58d305c6c70555c1259bd4" dependencies = [ "arrow", "chrono", @@ -1500,9 +1531,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356" +checksum = "befc67a3cdfbfa76853f43b10ac27337821bb98e519ab6baf431fcc0bcfcafdb" dependencies = [ "arrow", "arrow-array", @@ -3818,9 +3849,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" +checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" dependencies = [ "log", "sqlparser_derive", diff --git a/Cargo.toml b/Cargo.toml index a2d5995c4..4d9850d12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ [workspace.dependencies] arrow = { version = "51.0.0", default-features = false } -sqlparser = { version = "0.44.0" } +sqlparser = { version = "0.45.0" } chrono = { version = "0.4.31", default-features = false } reqwest = { version = "0.11.22", default-features = false } tokio = { version = "1.36.0" } @@ -26,28 +26,28 @@ prost-types = { version = "0.12.3" } object_store = { version= "0.9.1" } [workspace.dependencies.datafusion] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-common] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-expr] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-proto] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-physical-expr] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-optimizer] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-functions] -version = "37.1.0" +version = "38.0.0" [workspace.dependencies.datafusion-functions-array] -version = "37.1.0" +version = "38.0.0" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index d0ffb6d13..11bd551ad 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -1,13 +1,14 @@ use crate::error::{Result, VegaFusionError}; -use arrow::array::{new_empty_array, Array, ArrayRef, ListArray}; +use arrow::array::{Array, ArrayRef, ListArray}; use datafusion_common::DataFusionError; use arrow::datatypes::DataType; -use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; #[cfg(feature = "json")] use { + arrow::array::new_empty_array, + datafusion_common::utils::array_into_list_array, serde_json::{Map, Value}, std::ops::Deref, std::sync::Arc, diff --git a/vegafusion-common/src/datatypes.rs b/vegafusion-common/src/datatypes.rs index 76a295fce..75e325611 100644 --- a/vegafusion-common/src/datatypes.rs +++ b/vegafusion-common/src/datatypes.rs @@ -1,8 +1,8 @@ use crate::error::{Result, ResultWithContext}; use arrow::datatypes::DataType; use datafusion_common::DFSchema; -use datafusion_expr::{coalesce, lit, Expr, ExprSchemable, TryCast}; -use datafusion_functions::datetime::expr_fn::to_timestamp_millis; +use datafusion_expr::{lit, Expr, ExprSchemable, TryCast}; +use datafusion_functions::{datetime::expr_fn::to_timestamp_millis, expr_fn::coalesce}; pub fn is_numeric_datatype(dtype: &DataType) -> bool { matches!( diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index 8afefd60a..f9dbfe0d1 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -146,8 +146,8 @@ lazy_static! { Arc::new(DataType::Float64), Volatility::Immutable, // Accumulator factory - Arc::new(|dtype| Ok(Box::new(PercentileContAccumulator { - data_type: dtype.clone(), + Arc::new(|accum_args| Ok(Box::new(PercentileContAccumulator { + data_type: accum_args.data_type.clone(), all_values: Default::default(), percentile: 0.25, }))), @@ -165,8 +165,8 @@ lazy_static! { Arc::new(DataType::Float64), Volatility::Immutable, // Accumulator factory - Arc::new(|dtype| Ok(Box::new(PercentileContAccumulator { - data_type: dtype.clone(), + Arc::new(|accum_args| Ok(Box::new(PercentileContAccumulator { + data_type: accum_args.data_type.clone(), all_values: Default::default(), percentile: 0.75, }))), diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs index 2769c9105..edf44dcad 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs @@ -1,10 +1,11 @@ -use datafusion_physical_expr::udf::ScalarUDF; use std::any::Any; use std::sync::Arc; use vegafusion_common::arrow::array::{BooleanArray, Float32Array, Float64Array}; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; +use vegafusion_common::datafusion_expr::{ + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, +}; /// `isFinite(value)` /// diff --git a/vegafusion-runtime/src/expression/compiler/binary.rs b/vegafusion-runtime/src/expression/compiler/binary.rs index be74713bd..734c998c6 100644 --- a/vegafusion-runtime/src/expression/compiler/binary.rs +++ b/vegafusion-runtime/src/expression/compiler/binary.rs @@ -1,6 +1,8 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; use datafusion_expr::expr::BinaryExpr; -use datafusion_expr::{coalesce, concat, lit, Expr, Operator}; +use datafusion_expr::{lit, Expr, Operator}; +use datafusion_functions::expr_fn::coalesce; +use datafusion_functions::string::expr_fn::concat; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_common::datatypes::{ cast_to, data_type, is_null_literal, is_numeric_datatype, is_string_datatype, to_numeric, @@ -110,7 +112,7 @@ pub fn compile_binary( // plus is string concatenation let lhs_string = to_string(lhs, schema)?; let rhs_string = to_string(rhs, schema)?; - concat(&[lhs_string, rhs_string]) + concat(vec![lhs_string, rhs_string]) } else { // Both sides are non-strings, use regular numeric plus operation // Use result of to_numeric to handle booleans diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index ec7d6c047..4c8d43045 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -5,7 +5,8 @@ use std::convert::TryFrom; use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_expr::expr::Case; -use datafusion_expr::{ceil, expr, lit, Between, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Between, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_functions::expr_fn::ceil; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs index 1a6cade0f..625d22c42 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -1,6 +1,7 @@ use crate::expression::compiler::call::TzTransformFn; use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_functions::expr_fn::floor; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::datafusion_common::DFSchema; diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 4c6b70461..bd2acdc4a 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -6,11 +6,13 @@ use crate::expression::compiler::builtin_functions::date_time::datetime::{ use crate::expression::compiler::builtin_functions::type_checking::isvalid::is_valid_fn; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; -use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, ScalarUDF}; +use datafusion_expr::{expr, Expr, ScalarFunctionDefinition, ScalarUDF}; use datafusion_functions::expr_fn::isnan; +use datafusion_functions::math::{ + abs, acos, asin, atan, ceil, cos, exp, floor, ln, power, round, sin, sqrt, tan, +}; use std::collections::HashMap; use std::ops::Deref; -use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::data::table::VegaFusionTable; @@ -77,16 +79,9 @@ pub enum VegaFusionCallable { /// produces a new expression. UtcTransform(TzTransformFn), - /// Runtime function that is build in to DataFusion - BuiltinScalarFunction { - function: BuiltinScalarFunction, - /// If Some, all arguments should be cast to provided type - cast: Option, - }, - /// A custom runtime function that's not built into DataFusion ScalarUDF { - udf: ScalarUDF, + udf: Arc, /// If Some, all arguments should be cast to provided type cast: Option, }, @@ -138,14 +133,7 @@ pub fn compile_call( VegaFusionCallable::ScalarUDF { udf, cast } => { let args = compile_scalar_arguments(node, config, schema, cast)?; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(udf.clone())), - args, - })) - } - VegaFusionCallable::BuiltinScalarFunction { function, cast } => { - let args = compile_scalar_arguments(node, config, schema, cast)?; - Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(*function), + func_def: ScalarFunctionDefinition::UDF(udf.clone()), args, })) } @@ -225,31 +213,32 @@ pub fn default_callables() -> HashMap { let mut callables: HashMap = HashMap::new(); callables.insert("if".to_string(), VegaFusionCallable::Macro(Arc::new(if_fn))); - // Numeric functions built into DataFusion with names that match Vega. - // Cast arguments to Float64 - for fun_name in &[ - "abs", "acos", "asin", "atan", "ceil", "cos", "exp", "floor", "round", "sin", "sqrt", - "tan", "pow", + // Numeric functions built into DataFusion with mapping to Vega names + for (fun_name, udf) in [ + ("abs", abs()), + ("acos", acos()), + ("asin", asin()), + ("atan", atan()), + ("ceil", ceil()), + ("cos", cos()), + ("exp", exp()), + ("floor", floor()), + ("round", round()), + ("sin", sin()), + ("sqrt", sqrt()), + ("tan", tan()), + ("pow", power()), + ("log", ln()), // Vega log is DataFusion ln ] { - let function = BuiltinScalarFunction::from_str(fun_name).unwrap(); callables.insert( fun_name.to_string(), - VegaFusionCallable::BuiltinScalarFunction { - function, + VegaFusionCallable::ScalarUDF { + udf, cast: Some(DataType::Float64), }, ); } - // DataFusion ln is Vega log - callables.insert( - "log".to_string(), - VegaFusionCallable::BuiltinScalarFunction { - function: BuiltinScalarFunction::Ln, - cast: Some(DataType::Float64), - }, - ); - callables.insert( "isNaN".to_string(), VegaFusionCallable::UnaryTransform(Arc::new(isnan)), @@ -278,7 +267,7 @@ pub fn default_callables() -> HashMap { callables.insert( "span".to_string(), VegaFusionCallable::ScalarUDF { - udf: ScalarUDF::from(SpanUDF::new()), + udf: Arc::new(ScalarUDF::from(SpanUDF::new())), cast: None, }, ); @@ -286,7 +275,7 @@ pub fn default_callables() -> HashMap { callables.insert( "indexof".to_string(), VegaFusionCallable::ScalarUDF { - udf: ScalarUDF::from(IndexOfUDF::new()), + udf: Arc::new(ScalarUDF::from(IndexOfUDF::new())), cast: None, }, ); diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index 0b153a6a8..860843111 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -2,7 +2,8 @@ use crate::expression::compiler::builtin_functions::array::length::length_transf use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; -use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; +use datafusion_functions::expr_fn::substring; use std::convert::TryFrom; use std::sync::Arc; use vegafusion_common::arrow::array::Int64Array; @@ -90,10 +91,7 @@ pub fn compile_member( } else if matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { if let Some(index) = index { // SQL substr function is 1-indexed so add one - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Substr), - args: vec![compiled_object, lit((index + 1) as i64), lit(1i64)], - }) + substring(compiled_object, lit((index + 1) as i64), lit(1i64)) } else { return Err(VegaFusionError::compilation(format!( "Non-numeric element index: {property_string}" diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index b77420a06..fdafa7176 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -60,15 +60,15 @@ mod test_compile { use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; + use datafusion_functions::expr_fn::{coalesce, concat}; + use datafusion_functions_array::expr_fn::make_array; use vegafusion_core::expression::parser::parse; use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Case, TryCast}; - use datafusion_expr::{ - concat, expr, lit, BuiltinScalarFunction, Expr, Operator, ScalarFunctionDefinition, - }; + use datafusion_expr::{expr, lit, Expr, Operator, ScalarFunctionDefinition}; use std::collections::HashMap; use std::convert::TryFrom; @@ -174,16 +174,13 @@ mod test_compile { println!("expr: {result_expr:?}"); // unary not should cast numeric value to boolean - let expected_expr = !Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), - args: vec![ - Expr::TryCast(TryCast { - expr: Box::new(lit(32.0)), - data_type: DataType::Boolean, - }), - lit(false), - ], - }); + let expected_expr = coalesce(vec![ + Expr::TryCast(TryCast { + expr: Box::new(lit(32.0)), + data_type: DataType::Boolean, + }), + lit(false), + ]); assert_eq!(result_expr, expected_expr); @@ -200,20 +197,16 @@ mod test_compile { let expr = parse("32? 7: 9").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); println!("expr: {result_expr:?}"); - let expected_expr = Expr::Case(Case { expr: None, when_then_expr: vec![( - Box::new(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), - args: vec![ - Expr::TryCast(TryCast { - expr: Box::new(lit(32.0)), - data_type: DataType::Boolean, - }), - lit(false), - ], - })), + Box::new(coalesce(vec![ + Expr::TryCast(TryCast { + expr: Box::new(lit(32.0)), + data_type: DataType::Boolean, + }), + lit(false), + ])), Box::new(lit(7.0)), )], else_expr: Some(Box::new(lit(9.0))), @@ -259,16 +252,13 @@ mod test_compile { let expected_expr = Expr::Case(Case { expr: None, when_then_expr: vec![( - Box::new(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), - args: vec![ - Expr::TryCast(TryCast { - expr: Box::new(lit(5.0)), - data_type: DataType::Boolean, - }), - lit(false), - ], - })), + Box::new(coalesce(vec![ + Expr::TryCast(TryCast { + expr: Box::new(lit(5.0)), + data_type: DataType::Boolean, + }), + lit(false), + ])), Box::new(lit(55.0)), )], else_expr: Some(Box::new(lit(5.0))), @@ -332,7 +322,7 @@ mod test_compile { let expr = parse("'2' + '4'").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = concat(&[lit("2"), lit("4")]); + let expected_expr = concat(vec![lit("2"), lit("4")]); println!("expr: {result_expr:?}"); assert_eq!(result_expr, expected_expr); @@ -391,11 +381,7 @@ mod test_compile { fn test_compile_array_numeric() { let expr = parse("[1, 2, 3]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - - let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), - args: vec![lit(1.0), lit(2.0), lit(3.0)], - }); + let expected_expr = make_array(vec![lit(1.0), lit(2.0), lit(3.0)]); println!("expr: {result_expr:?}"); assert_eq!(result_expr, expected_expr); @@ -415,10 +401,7 @@ mod test_compile { let expr = parse("[]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), - args: vec![], - }); + let expected_expr = make_array(vec![]); println!("expr: {result_expr:?}"); assert_eq!(result_expr, expected_expr); @@ -436,23 +419,12 @@ mod test_compile { let expr = parse("[[1, 2], [3, 4], [5, 6]]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), - args: vec![ - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), - args: vec![lit(1.0), lit(2.0)], - }), - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), - args: vec![lit(3.0), lit(4.0)], - }), - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), - args: vec![lit(5.0), lit(6.0)], - }), - ], - }); + let expected_expr = make_array(vec![ + make_array(vec![lit(1.0), lit(2.0)]), + make_array(vec![lit(3.0), lit(4.0)]), + make_array(vec![lit(5.0), lit(6.0)]), + ]); + println!("expr: {result_expr:?}"); assert_eq!(result_expr, expected_expr); @@ -602,16 +574,13 @@ mod test_compile { let expected_expr = Expr::Case(Case { expr: None, when_then_expr: vec![( - Box::new(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), - args: vec![ - Expr::TryCast(TryCast { - expr: Box::new(lit(32.0)), - data_type: DataType::Boolean, - }), - lit(false), - ], - })), + Box::new(coalesce(vec![ + Expr::TryCast(TryCast { + expr: Box::new(lit(32.0)), + data_type: DataType::Boolean, + }), + lit(false), + ])), Box::new(lit(7.0)), )], else_expr: Some(Box::new(lit(9.0))), diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index 9124c3c05..cb77369ef 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -128,7 +128,7 @@ pub fn make_aggr_expr_for_named_col( ) -> Result { let column = if let Some(col_name) = col_name { let col_name = unescape_field(&col_name); - if schema.index_of_column_by_name(None, &col_name).is_err() { + if schema.index_of_column_by_name(None, &col_name).is_none() { // No column with specified name, short circuit to return default value return if matches!(op, AggregateOp::Sum | AggregateOp::Count) { // return zero for sum and count diff --git a/vegafusion-runtime/src/transform/bin.rs b/vegafusion-runtime/src/transform/bin.rs index a0d4b6e78..834e56a3f 100644 --- a/vegafusion-runtime/src/transform/bin.rs +++ b/vegafusion-runtime/src/transform/bin.rs @@ -9,8 +9,8 @@ use datafusion_expr::lit; use datafusion_common::scalar::ScalarValue; use datafusion_common::utils::array_into_list_array; use datafusion_common::DFSchema; -use datafusion_expr::{floor, when, Expr}; -use datafusion_functions::expr_fn::abs; +use datafusion_expr::{when, Expr}; +use datafusion_functions::expr_fn::{abs, floor}; use float_cmp::approx_eq; use std::ops::{Add, Div, Mul, Sub}; use std::sync::Arc; diff --git a/vegafusion-runtime/src/transform/pivot.rs b/vegafusion-runtime/src/transform/pivot.rs index e9ee57381..c53d098e6 100644 --- a/vegafusion-runtime/src/transform/pivot.rs +++ b/vegafusion-runtime/src/transform/pivot.rs @@ -2,7 +2,8 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::aggregate::make_agg_expr_for_col_expr; use crate::transform::TransformTrait; use async_trait::async_trait; -use datafusion_expr::{coalesce, expr::Sort, lit, min, when, Expr}; +use datafusion_expr::{expr::Sort, lit, min, when, Expr}; +use datafusion_functions::expr_fn::coalesce; use std::sync::Arc; use vegafusion_common::arrow::array::StringArray; use vegafusion_common::arrow::datatypes::DataType; diff --git a/vegafusion-runtime/src/transform/timeunit.rs b/vegafusion-runtime/src/transform/timeunit.rs index ad99aa591..d33ae7f1d 100644 --- a/vegafusion-runtime/src/transform/timeunit.rs +++ b/vegafusion-runtime/src/transform/timeunit.rs @@ -2,6 +2,7 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; use datafusion_common::DFSchema; +use datafusion_functions::expr_fn::floor; use std::collections::HashSet; use std::ops::{Add, Div, Mul, Sub}; use std::sync::Arc; @@ -11,7 +12,7 @@ use vegafusion_core::proto::gen::transforms::{TimeUnit, TimeUnitTimeZone, TimeUn use vegafusion_core::task_graph::task_value::TaskValue; use datafusion_expr::expr::Cast; -use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use itertools::Itertools; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::datatypes::{cast_to, is_numeric_datatype}; diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index 5f25e4a42..33665277d 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -44,7 +44,7 @@ impl TransformTrait for Window { .schema_df()? .fields() .iter() - .map(|f| flat_col(f.field().name())) + .map(|f| flat_col(f.name())) .collect(); if order_by.is_empty() { diff --git a/vegafusion-sql/Cargo.toml b/vegafusion-sql/Cargo.toml index b1a7c3f54..f17a3a5cf 100644 --- a/vegafusion-sql/Cargo.toml +++ b/vegafusion-sql/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" description = "VegaFusion SQL dialect generation and connection implementations" [features] -datafusion-conn = [ "datafusion", "tempfile", "reqwest", "reqwest-retry", "reqwest-middleware", "vegafusion-datafusion-udfs", "object_store", "url", "vegafusion-common/object_store",] +datafusion-conn = [ "datafusion", "tempfile", "reqwest", "reqwest-retry", "reqwest-middleware", "vegafusion-datafusion-udfs", "object_store", "url", "vegafusion-common/object_store", "vegafusion-common/prettyprint",] pyarrow = [ "pyo3", "datafusion-common/pyarrow", "vegafusion-common/pyarrow", "vegafusion-dataframe/pyarrow",] [dependencies] diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index f7bdf9c0a..c0e728530 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -439,6 +439,7 @@ impl ToSqlExpr for Expr { partition_by, order_by, window_frame: sql_window_frame, + window_name: None, }); let sql_fun = SqlFunction { @@ -643,9 +644,8 @@ mod tests { use arrow::datatypes::DataType; use datafusion_common::DFSchema; use datafusion_expr::expr::Cast; - use datafusion_expr::{ - expr, lit, Between, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, - }; + use datafusion_expr::{lit, Between, Expr}; + use datafusion_functions::expr_fn::sin; use datafusion_functions::string::expr_fn::upper; use vegafusion_common::column::flat_col; @@ -664,10 +664,7 @@ mod tests { #[test] pub fn test2() { - let df_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Sin), - args: vec![lit(1.2)], - }) + flat_col("B"); + let df_expr = sin(lit(1.2)) + flat_col("B"); let dialect: Dialect = Dialect::datafusion(); let sql_expr = df_expr.to_sql(&dialect, &schema()).unwrap(); diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index a0a5dc865..735548399 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -385,7 +385,7 @@ impl SqlConnection for DataFusionConnection { .schema() .fields() .iter() - .map(|f| f.field().as_ref().clone().with_nullable(true)) + .map(|f| f.as_ref().clone().with_nullable(true)) .collect(); let expected_fields: Vec<_> = schema .fields diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index a204682c6..68e3bfa5d 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -6,14 +6,13 @@ use crate::dialect::{Dialect, ValuesMode}; use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use datafusion_common::{Column, DFSchema, OwnedTableReference, ScalarValue}; +use datafusion_common::{Column, DFSchema, ScalarValue, TableReference}; use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{ - expr, is_null, lit, max, min, when, AggregateFunction, BuiltInWindowFunction, - BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, WindowFrame, - WindowFunctionDefinition, + expr, is_null, lit, max, min, when, AggregateFunction, BuiltInWindowFunction, Expr, + ExprSchemable, WindowFrame, WindowFunctionDefinition, }; -use datafusion_functions::expr_fn::abs; +use datafusion_functions::expr_fn::{abs, coalesce}; use sqlparser::ast::{ Cte, Expr as SqlExpr, GroupByExpr, Ident, NullTreatment, Query, Select, SelectItem, SetExpr, Statement, TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, @@ -594,7 +593,7 @@ impl SqlDataFrame { .map(|col| { let col = Expr::Column(Column { relation: if self.dialect().joinaggregate_fully_qualified { - Some(OwnedTableReference::bare(inner_name.clone())) + Some(TableReference::bare(inner_name.clone())) } else { None }, @@ -618,7 +617,7 @@ impl SqlDataFrame { } else { let expr = Expr::Column(Column { relation: if self.dialect().joinaggregate_fully_qualified { - Some(OwnedTableReference::bare(self.parent_name())) + Some(TableReference::bare(self.parent_name())) } else { None }, @@ -896,11 +895,10 @@ impl SqlDataFrame { // Build partitioning column expressions let partition_by: Vec<_> = groupby.iter().map(|group| flat_col(group)).collect(); - - let numeric_field = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), - args: vec![to_numeric(flat_col(field), &self.schema_df()?)?, lit(0.0)], - }); + let numeric_field = coalesce(vec![ + to_numeric(flat_col(field), &self.schema_df()?)?, + lit(0.0), + ]); if let StackMode::Zero = mode { // Build window expression @@ -1150,13 +1148,7 @@ impl SqlDataFrame { let col_name = f.name(); if col_name == field { - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn( - BuiltinScalarFunction::Coalesce, - ), - args: vec![flat_col(field), lit(value.clone())], - }) - .alias(col_name) + coalesce(vec![flat_col(field), lit(value.clone())]).alias(col_name) } else { flat_col(col_name) } @@ -1197,13 +1189,7 @@ impl SqlDataFrame { .iter() .map(|col_name| { if col_name == field { - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn( - BuiltinScalarFunction::Coalesce, - ), - args: vec![flat_col(field), lit(value.clone())], - }) - .alias(col_name) + coalesce(vec![flat_col(field), lit(value.clone())]).alias(col_name) } else { flat_col(col_name) } @@ -1217,22 +1203,16 @@ impl SqlDataFrame { .iter() .map(|col_name| { let expr = if col_name == field { - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn( - BuiltinScalarFunction::Coalesce, - ), - args: vec![flat_col(field), lit(value.clone())], - }) - .alias(col_name) + coalesce(vec![flat_col(field), lit(value.clone())]).alias(col_name) } else if col_name == key { Expr::Column(Column { - relation: Some(OwnedTableReference::bare("_key")), + relation: Some(TableReference::bare("_key")), name: col_name.clone(), }) .alias(col_name) } else if groupby.contains(col_name) { Expr::Column(Column { - relation: Some(OwnedTableReference::bare("_groups")), + relation: Some(TableReference::bare("_groups")), name: col_name.clone(), }) .alias(col_name) diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index f4d98ace3..53d6bb8c6 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -65,7 +65,7 @@ pub fn part_to_date_time_field(part: &str) -> Result { Ok(match part.to_ascii_lowercase().as_str() { "year" | "years" => SqlDateTimeField::Year, "month" | "months " => SqlDateTimeField::Month, - "week" | "weeks" => SqlDateTimeField::Week, + "week" | "weeks" => SqlDateTimeField::Week(None), "day" | "days" => SqlDateTimeField::Day, "date" => SqlDateTimeField::Date, "hour" | "hours" => SqlDateTimeField::Hour, diff --git a/vegafusion-sql/tests/expected/select.toml b/vegafusion-sql/tests/expected/select.toml index df2139879..26d747680 100644 --- a/vegafusion-sql/tests/expected/select.toml +++ b/vegafusion-sql/tests/expected/select.toml @@ -332,34 +332,34 @@ result = ''' [scalar_math_functions] athena = """ -WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log10("c") AS "log", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, -1.8 AS `b`, 0.1 AS `c` UNION ALL SELECT 1 AS `a`, -1.0 AS `b`, 0.2 AS `c` UNION ALL SELECT 2 AS `a`, 0.0 AS `b`, 0.4 AS `c` UNION ALL SELECT 3 AS `a`, 1.0 AS `b`, 0.6 AS `c` UNION ALL SELECT 4 AS `a`, 1.8 AS `b`, 0.8 AS `c` UNION ALL SELECT 5 AS `a`, NULL AS `b`, NULL AS `c`), values1 AS (SELECT `a`, abs(`b`) AS `abs`, acos(`c`) AS `acos`, asin(`c`) AS `asin`, atan(`c`) AS `atan`, atan2(`c`, `a`) AS `atan2`, CEIL(`b`) AS `ceil`, cos(`b`) AS `cos`, exp(`b`) AS `exp`, FLOOR(`b`) AS `floor`, ln(`c`) AS `ln`, log10(`c`) AS `log`, log10(`c`) AS `log10`, log(`c`, 2) AS `log2`, pow(`b`, `a`) AS `power`, round(`b`) AS `round`, sin(`b`) AS `sin`, sqrt(`c`) AS `sqrt`, tan(`b`) AS `tan` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, -1.8 AS `b`, 0.1 AS `c` UNION ALL SELECT 1 AS `a`, -1.0 AS `b`, 0.2 AS `c` UNION ALL SELECT 2 AS `a`, 0.0 AS `b`, 0.4 AS `c` UNION ALL SELECT 3 AS `a`, 1.0 AS `b`, 0.6 AS `c` UNION ALL SELECT 4 AS `a`, 1.8 AS `b`, 0.8 AS `c` UNION ALL SELECT 5 AS `a`, NULL AS `b`, NULL AS `c`), values1 AS (SELECT `a`, abs(`b`) AS `abs`, acos(`c`) AS `acos`, asin(`c`) AS `asin`, atan(`c`) AS `atan`, atan2(`c`, `a`) AS `atan2`, CEIL(`b`) AS `ceil`, cos(`b`) AS `cos`, exp(`b`) AS `exp`, FLOOR(`b`) AS `floor`, ln(`c`) AS `ln`, log10(`c`) AS `log10`, log(`c`, 2) AS `log2`, pow(`b`, `a`) AS `power`, round(`b`) AS `round`, sin(`b`) AS `sin`, sqrt(`c`) AS `sqrt`, tan(`b`) AS `tan` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ -WITH values0 AS (SELECT 0 AS "a", -1.8 AS "b", 0.1 AS "c" UNION ALL SELECT 1 AS "a", -1.0 AS "b", 0.2 AS "c" UNION ALL SELECT 2 AS "a", 0.0 AS "b", 0.4 AS "c" UNION ALL SELECT 3 AS "a", 1.0 AS "b", 0.6 AS "c" UNION ALL SELECT 4 AS "a", 1.8 AS "b", 0.8 AS "c" UNION ALL SELECT 5 AS "a", NULL AS "b", NULL AS "c"), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log10("c") AS "log", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", -1.8 AS "b", 0.1 AS "c" UNION ALL SELECT 1 AS "a", -1.0 AS "b", 0.2 AS "c" UNION ALL SELECT 2 AS "a", 0.0 AS "b", 0.4 AS "c" UNION ALL SELECT 3 AS "a", 1.0 AS "b", 0.6 AS "c" UNION ALL SELECT 4 AS "a", 1.8 AS "b", 0.8 AS "c" UNION ALL SELECT 5 AS "a", NULL AS "b", NULL AS "c"), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ databricks = """ -WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS `_values` (`a`, `b`, `c`)), values1 AS (SELECT `a`, abs(`b`) AS `abs`, acos(`c`) AS `acos`, asin(`c`) AS `asin`, atan(`c`) AS `atan`, atan2(`c`, `a`) AS `atan2`, CEIL(`b`) AS `ceil`, cos(`b`) AS `cos`, exp(`b`) AS `exp`, FLOOR(`b`) AS `floor`, ln(`c`) AS `ln`, log10(`c`) AS `log`, log10(`c`) AS `log10`, log2(`c`) AS `log2`, pow(`b`, `a`) AS `power`, round(`b`) AS `round`, sin(`b`) AS `sin`, sqrt(`c`) AS `sqrt`, tan(`b`) AS `tan` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS `_values` (`a`, `b`, `c`)), values1 AS (SELECT `a`, abs(`b`) AS `abs`, acos(`c`) AS `acos`, asin(`c`) AS `asin`, atan(`c`) AS `atan`, atan2(`c`, `a`) AS `atan2`, CEIL(`b`) AS `ceil`, cos(`b`) AS `cos`, exp(`b`) AS `exp`, FLOOR(`b`) AS `floor`, ln(`c`) AS `ln`, log10(`c`) AS `log10`, log2(`c`) AS `log2`, pow(`b`, `a`) AS `power`, round(`b`) AS `round`, sin(`b`) AS `sin`, sqrt(`c`) AS `sqrt`, tan(`b`) AS `tan` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ datafusion = """ -WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log("c") AS "log", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ duckdb = """ -WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", pow(2.718281828459045, "b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log("c") AS "log", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", pow(2.718281828459045, "b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log10("c") AS "log10", log2("c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ mysql = """ -WITH values0 AS (SELECT * FROM (VALUES ROW(0, -1.8, 0.1), ROW(1, -1.0, 0.2), ROW(2, 0.0, 0.4), ROW(3, 1.0, 0.6), ROW(4, 1.8, 0.8), ROW(5, NULL, NULL)) AS `_values` (`a`, `b`, `c`)), values1 AS (SELECT `a`, abs(`b`) AS `abs`, acos(`c`) AS `acos`, asin(`c`) AS `asin`, atan(`c`) AS `atan`, atan2(`c`, `a`) AS `atan2`, CEIL(`b`) AS `ceil`, cos(`b`) AS `cos`, exp(`b`) AS `exp`, FLOOR(`b`) AS `floor`, ln(`c`) AS `ln`, log10(`c`) AS `log`, log10(`c`) AS `log10`, log2(`c`) AS `log2`, pow(`b`, `a`) AS `power`, round(`b`) AS `round`, sin(`b`) AS `sin`, sqrt(`c`) AS `sqrt`, tan(`b`) AS `tan` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC +WITH values0 AS (SELECT * FROM (VALUES ROW(0, -1.8, 0.1), ROW(1, -1.0, 0.2), ROW(2, 0.0, 0.4), ROW(3, 1.0, 0.6), ROW(4, 1.8, 0.8), ROW(5, NULL, NULL)) AS `_values` (`a`, `b`, `c`)), values1 AS (SELECT `a`, abs(`b`) AS `abs`, acos(`c`) AS `acos`, asin(`c`) AS `asin`, atan(`c`) AS `atan`, atan2(`c`, `a`) AS `atan2`, CEIL(`b`) AS `ceil`, cos(`b`) AS `cos`, exp(`b`) AS `exp`, FLOOR(`b`) AS `floor`, ln(`c`) AS `ln`, log10(`c`) AS `log10`, log2(`c`) AS `log2`, pow(`b`, `a`) AS `power`, round(`b`) AS `round`, sin(`b`) AS `sin`, sqrt(`c`) AS `sqrt`, tan(`b`) AS `tan` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC """ postgres = """ -WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log("c") AS "log", log(10, "c") AS "log10", log(2, "c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL)) AS "_values" ("a", "b", "c")), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log(10, "c") AS "log10", log(2, "c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ redshift = """ -WITH values0 AS (SELECT 0 AS "a", -1.8 AS "b", 0.1 AS "c" UNION ALL SELECT 1 AS "a", -1.0 AS "b", 0.2 AS "c" UNION ALL SELECT 2 AS "a", 0.0 AS "b", 0.4 AS "c" UNION ALL SELECT 3 AS "a", 1.0 AS "b", 0.6 AS "c" UNION ALL SELECT 4 AS "a", 1.8 AS "b", 0.8 AS "c" UNION ALL SELECT 5 AS "a", NULL AS "b", NULL AS "c"), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln(CAST("c" AS DOUBLE PRECISION)) AS "ln", log(CAST("c" AS DOUBLE PRECISION)) AS "log", log(CAST("c" AS DOUBLE PRECISION)) AS "log10", ln(CAST("c" AS DOUBLE PRECISION)) / ln(2) AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", -1.8 AS "b", 0.1 AS "c" UNION ALL SELECT 1 AS "a", -1.0 AS "b", 0.2 AS "c" UNION ALL SELECT 2 AS "a", 0.0 AS "b", 0.4 AS "c" UNION ALL SELECT 3 AS "a", 1.0 AS "b", 0.6 AS "c" UNION ALL SELECT 4 AS "a", 1.8 AS "b", 0.8 AS "c" UNION ALL SELECT 5 AS "a", NULL AS "b", NULL AS "c"), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln(CAST("c" AS DOUBLE PRECISION)) AS "ln", log(CAST("c" AS DOUBLE PRECISION)) AS "log10", ln(CAST("c" AS DOUBLE PRECISION)) / ln(2) AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ snowflake = """ -WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL))), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log(10, "c") AS "log", log(10, "c") AS "log10", log(2, "c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (0, -1.8, 0.1), (1, -1.0, 0.2), (2, 0.0, 0.4), (3, 1.0, 0.6), (4, 1.8, 0.8), (5, NULL, NULL))), values1 AS (SELECT "a", abs("b") AS "abs", acos("c") AS "acos", asin("c") AS "asin", atan("c") AS "atan", atan2("c", "a") AS "atan2", CEIL("b") AS "ceil", cos("b") AS "cos", exp("b") AS "exp", FLOOR("b") AS "floor", ln("c") AS "ln", log(10, "c") AS "log10", log(2, "c") AS "log2", pow("b", "a") AS "power", round("b") AS "round", sin("b") AS "sin", sqrt("c") AS "sqrt", tan("b") AS "tan" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ result = ''' +---+-----+--------------------+---------------------+---------------------+---------------------+------+---------------------+---------------------+-------+---------------------+----------------------+----------------------+---------------------+---------+-------+---------------------+---------------------+--------------------+ diff --git a/vegafusion-sql/tests/test_aggregate.rs b/vegafusion-sql/tests/test_aggregate.rs index a49477680..e61df58d9 100644 --- a/vegafusion-sql/tests/test_aggregate.rs +++ b/vegafusion-sql/tests/test_aggregate.rs @@ -2,7 +2,7 @@ extern crate lazy_static; mod utils; -use datafusion_expr::{avg, count, expr, lit, max, min, round, sum, AggregateFunction, Expr}; +use datafusion_expr::{avg, count, expr, lit, max, min, sum, AggregateFunction, Expr}; use rstest::rstest; use rstest_reuse::{self, *}; use serde_json::json; @@ -131,6 +131,7 @@ mod test_variance_aggs { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions::expr_fn::round; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); diff --git a/vegafusion-sql/tests/test_select.rs b/vegafusion-sql/tests/test_select.rs index 444681056..1d5311e75 100644 --- a/vegafusion-sql/tests/test_select.rs +++ b/vegafusion-sql/tests/test_select.rs @@ -485,28 +485,16 @@ mod test_non_finite_numbers { #[cfg(test)] mod test_scalar_math_functions { use crate::*; - use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, Expr}; use datafusion_functions::math::expr_fn::{abs, acos, asin, tan}; use vegafusion_common::column::flat_col; - fn make_scalar_fn1(fun: BuiltinScalarFunction, arg: &str, alias: &str) -> Expr { - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(fun), - args: vec![flat_col(arg)], - }) - .alias(alias) - } - - fn make_scalar_fn2(fun: BuiltinScalarFunction, arg1: &str, arg2: &str, alias: &str) -> Expr { - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(fun), - args: vec![flat_col(arg1), flat_col(arg2)], - }) - .alias(alias) - } - #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions::expr_fn::{ + atan, atan2, ceil, cos, exp, floor, ln, log10, log2, power, round, sin, sqrt, + }; + println!("{dialect_name}"); let (conn, _evaluable) = TOKIO_RUNTIME.block_on(make_connection(dialect_name)); @@ -527,20 +515,19 @@ mod test_scalar_math_functions { abs(flat_col("b")).alias("abs"), acos(flat_col("c")).alias("acos"), asin(flat_col("c")).alias("asin"), - make_scalar_fn1(BuiltinScalarFunction::Atan, "c", "atan"), - make_scalar_fn2(BuiltinScalarFunction::Atan2, "c", "a", "atan2"), - make_scalar_fn1(BuiltinScalarFunction::Ceil, "b", "ceil"), - make_scalar_fn1(BuiltinScalarFunction::Cos, "b", "cos"), - make_scalar_fn1(BuiltinScalarFunction::Exp, "b", "exp"), - make_scalar_fn1(BuiltinScalarFunction::Floor, "b", "floor"), - make_scalar_fn1(BuiltinScalarFunction::Ln, "c", "ln"), - make_scalar_fn1(BuiltinScalarFunction::Log, "c", "log"), - make_scalar_fn1(BuiltinScalarFunction::Log10, "c", "log10"), - make_scalar_fn1(BuiltinScalarFunction::Log2, "c", "log2"), - make_scalar_fn2(BuiltinScalarFunction::Power, "b", "a", "power"), - make_scalar_fn1(BuiltinScalarFunction::Round, "b", "round"), - make_scalar_fn1(BuiltinScalarFunction::Sin, "b", "sin"), - make_scalar_fn1(BuiltinScalarFunction::Sqrt, "c", "sqrt"), + atan(flat_col("c")).alias("atan"), + atan2(flat_col("c"), flat_col("a")).alias("atan2"), + ceil(flat_col("b")).alias("ceil"), + cos(flat_col("b")).alias("cos"), + exp(flat_col("b")).alias("exp"), + floor(flat_col("b")).alias("floor"), + ln(flat_col("c")).alias("ln"), + log10(flat_col("c")).alias("log10"), + log2(flat_col("c")).alias("log2"), + power(flat_col("b"), flat_col("a")).alias("power"), + round(vec![flat_col("b")]).alias("round"), + sin(flat_col("b")).alias("sin"), + sqrt(flat_col("c")).alias("sqrt"), tan(flat_col("b")).alias("tan"), ]) .await; @@ -1468,12 +1455,14 @@ mod test_timestamp_to_utc_timestamp { #[cfg(test)] mod test_string_ops { use crate::*; - use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; - use datafusion_functions::string::expr_fn::{lower, upper}; + use datafusion_expr::{expr, lit, Expr}; + use datafusion_functions::string::expr_fn::{concat, lower, upper}; use vegafusion_common::column::flat_col; #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions::expr_fn::substring; + println!("{dialect_name}"); let (conn, evaluable) = TOKIO_RUNTIME.block_on(make_connection(dialect_name)); @@ -1491,16 +1480,8 @@ mod test_string_ops { flat_col("a"), flat_col("b"), flat_col("c"), - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Substr), - args: vec![flat_col("b"), lit(2), lit(2)], - }) - .alias("b_substr"), - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Concat), - args: vec![flat_col("b"), lit(" "), flat_col("c")], - }) - .alias("bc_concat"), + substring(flat_col("b"), lit(2), lit(2)).alias("b_substr"), + concat(vec![flat_col("b"), lit(" "), flat_col("c")]).alias("bc_concat"), upper(flat_col("b")).alias("b_upper"), lower(flat_col("b")).alias("b_lower"), ]) diff --git a/vegafusion-sql/tests/test_stack.rs b/vegafusion-sql/tests/test_stack.rs index fcd9660c2..513459c37 100644 --- a/vegafusion-sql/tests/test_stack.rs +++ b/vegafusion-sql/tests/test_stack.rs @@ -2,7 +2,7 @@ extern crate lazy_static; mod utils; -use datafusion_expr::{expr, lit, round, Expr}; +use datafusion_expr::{expr, lit, Expr}; use rstest::rstest; use rstest_reuse::{self, *}; use serde_json::json; @@ -107,6 +107,8 @@ mod test_mode_normalized { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions::expr_fn::round; + println!("{dialect_name}"); let (conn, evaluable) = TOKIO_RUNTIME.block_on(make_connection(dialect_name)); let df = stack_data(conn);