Update to DataFrusion 38 and sqlparser 0.45

vega · Aug 10, 2024 · 43eea6a · 43eea6a
1 parent 7c30cb0
commit 43eea6a
Show file tree

Hide file tree

Showing 27 changed files with 233 additions and 277 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -319,7 +319,7 @@ jobs:
           ls -la
           python -m pip install vegafusion-*.whl
           python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl
-          python -m pip install pytest vega-datasets polars-lts-cpu duckdb==0.9.2 "vl-convert-python>=1.0.1rc1" scikit-image pandas==2.0
+          python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2"
       - name: Test lazy imports
         working-directory: python/vegafusion/
         run: python checks/check_lazy_imports.py
@@ -353,7 +353,7 @@ jobs:
           ls -la
           python -m pip install vegafusion-*.whl
           python -m pip install vegafusion_python_embed-*macosx_10_*_x86_64.whl
-          python -m pip install pytest vega-datasets polars-lts-cpu duckdb==0.9.2 vl-convert-python scikit-image pandas==2.0
+          python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2"
           python -m pip install pyarrow==10.0 altair==5.1.2
       - name: Test vegafusion
         working-directory: python/vegafusion/
@@ -389,7 +389,7 @@ jobs:
 
           python -m pip install $vegafusion
           python -m pip install $vegafusion_python_embed
-          python -m pip install pytest vega-datasets polars[timezone] duckdb==0.9.2 vl-convert-python scikit-image
+          python -m pip install pytest vega-datasets polars[timezone] "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2"
       - name: Test vegafusion
         working-directory: python/vegafusion/
         run: pytest

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -15,7 +15,7 @@ members = [
 
 [workspace.dependencies]
 arrow = { version = "51.0.0", default-features = false }
-sqlparser = { version = "0.44.0" }
+sqlparser = { version = "0.45.0" }
 chrono = { version = "0.4.31", default-features = false }
 reqwest = { version = "0.11.22", default-features = false }
 tokio = { version = "1.36.0" }
@@ -26,28 +26,28 @@ prost-types = { version = "0.12.3" }
 object_store = { version= "0.9.1" }
 
 [workspace.dependencies.datafusion]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-common]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-expr]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-proto]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-physical-expr]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-optimizer]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-functions]
-version = "37.1.0"
+version = "38.0.0"
 
 [workspace.dependencies.datafusion-functions-array]
-version = "37.1.0"
+version = "38.0.0"
 
 [profile.release]
 ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization

diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs
@@ -1,13 +1,14 @@
 use crate::error::{Result, VegaFusionError};
-use arrow::array::{new_empty_array, Array, ArrayRef, ListArray};
+use arrow::array::{Array, ArrayRef, ListArray};
 use datafusion_common::DataFusionError;
 
 use arrow::datatypes::DataType;
-use datafusion_common::utils::array_into_list_array;
 pub use datafusion_common::ScalarValue;
 
 #[cfg(feature = "json")]
 use {
+    arrow::array::new_empty_array,
+    datafusion_common::utils::array_into_list_array,
     serde_json::{Map, Value},
     std::ops::Deref,
     std::sync::Arc,

diff --git a/vegafusion-common/src/datatypes.rs b/vegafusion-common/src/datatypes.rs
@@ -1,8 +1,8 @@
 use crate::error::{Result, ResultWithContext};
 use arrow::datatypes::DataType;
 use datafusion_common::DFSchema;
-use datafusion_expr::{coalesce, lit, Expr, ExprSchemable, TryCast};
-use datafusion_functions::datetime::expr_fn::to_timestamp_millis;
+use datafusion_expr::{lit, Expr, ExprSchemable, TryCast};
+use datafusion_functions::{datetime::expr_fn::to_timestamp_millis, expr_fn::coalesce};
 
 pub fn is_numeric_datatype(dtype: &DataType) -> bool {
     matches!(

diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs
@@ -146,8 +146,8 @@ lazy_static! {
         Arc::new(DataType::Float64),
         Volatility::Immutable,
         // Accumulator factory
-        Arc::new(|dtype| Ok(Box::new(PercentileContAccumulator {
-            data_type: dtype.clone(),
+        Arc::new(|accum_args| Ok(Box::new(PercentileContAccumulator {
+            data_type: accum_args.data_type.clone(),
             all_values: Default::default(),
             percentile: 0.25,
         }))),
@@ -165,8 +165,8 @@ lazy_static! {
         Arc::new(DataType::Float64),
         Volatility::Immutable,
         // Accumulator factory
-        Arc::new(|dtype| Ok(Box::new(PercentileContAccumulator {
-            data_type: dtype.clone(),
+        Arc::new(|accum_args| Ok(Box::new(PercentileContAccumulator {
+            data_type: accum_args.data_type.clone(),
             all_values: Default::default(),
             percentile: 0.75,
         }))),

diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs
@@ -1,10 +1,11 @@
-use datafusion_physical_expr::udf::ScalarUDF;
 use std::any::Any;
 use std::sync::Arc;
 use vegafusion_common::arrow::array::{BooleanArray, Float32Array, Float64Array};
 use vegafusion_common::arrow::datatypes::DataType;
 use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue};
-use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility};
+use vegafusion_common::datafusion_expr::{
+    ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
 
 /// `isFinite(value)`
 ///