Fix regression in length function (#227)

* make length function return f64 to work around DataFusion type inference issue * Bump version to 1.0.0-rc2 * Fix more length tests
vega · Jan 17, 2023 · 010c346 · 010c346
1 parent cd0eca3
commit 010c346
Show file tree

Hide file tree

Showing 21 changed files with 50 additions and 43 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/javascript/vegafusion-chart-editor/package-lock.json b/javascript/vegafusion-chart-editor/package-lock.json
diff --git a/javascript/vegafusion-chart-editor/package.json b/javascript/vegafusion-chart-editor/package.json
@@ -1,6 +1,6 @@
 {
   "name": "create-wasm-app",
-  "version": "1.0.0-rc1",
+  "version": "1.0.0-rc2",
   "description": "create an app to consume rust-generated wasm packages",
   "main": "index.js",
   "bin": {

diff --git a/javascript/vegafusion-embed/package-lock.json b/javascript/vegafusion-embed/package-lock.json
diff --git a/javascript/vegafusion-embed/package.json b/javascript/vegafusion-embed/package.json
@@ -1,6 +1,6 @@
 {
   "name": "vegafusion-embed",
-  "version": "1.0.0-rc1",
+  "version": "1.0.0-rc2",
   "description": "Library to embed vegafusion visualizations",
   "keywords": [
     "vega",

diff --git a/python/vegafusion-jupyter/package-lock.json b/python/vegafusion-jupyter/package-lock.json
diff --git a/python/vegafusion-jupyter/package.json b/python/vegafusion-jupyter/package.json
@@ -1,6 +1,6 @@
 {
   "name": "vegafusion-jupyter",
-  "version": "1.0.0-rc1",
+  "version": "1.0.0-rc2",
   "description": "Altair Jupyter Widget library that relies on VegaFusion for serverside calculations",
   "keywords": [
     "jupyter",

diff --git a/python/vegafusion-jupyter/vegafusion_jupyter/_frontend.py b/python/vegafusion-jupyter/vegafusion_jupyter/_frontend.py
@@ -2,4 +2,4 @@
 Information about the frontend package of the widgets.
 """    
 module_name = "vegafusion-jupyter"
-module_version = "^1.0.0-rc1"
+module_version = "^1.0.0-rc2"
diff --git a/python/vegafusion-jupyter/vegafusion_jupyter/_version.py b/python/vegafusion-jupyter/vegafusion_jupyter/_version.py
@@ -1 +1 @@
-__version__ = '1.0.0-rc1'
+__version__ = '1.0.0-rc2'
diff --git a/python/vegafusion/setup.cfg b/python/vegafusion/setup.cfg
@@ -4,7 +4,7 @@ universal = 0
 [metadata]
 name = vegafusion
 description = Core tools for using VegaFusion from Python
-version = 1.0.0-rc1
+version = 1.0.0-rc2
 long_description = file: README.md
 long_description_content_type = text/markdown
 keywords = vega, altair, vegafusion, arrow

diff --git a/python/vegafusion/vegafusion/_version.py b/python/vegafusion/vegafusion/_version.py
@@ -1 +1 @@
-__version__ = '1.0.0-rc1'
+__version__ = '1.0.0-rc2'
diff --git a/vegafusion-core/Cargo.toml b/vegafusion-core/Cargo.toml
@@ -2,7 +2,7 @@
 name = "vegafusion-core"
 license = "BSD-3-Clause"
 edition = "2021"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [features]
 tonic_support = [ "tonic", "tonic-build",]

diff --git a/vegafusion-python-embed/Cargo.toml b/vegafusion-python-embed/Cargo.toml
@@ -2,7 +2,7 @@
 name = "vegafusion-python-embed"
 license = "BSD-3-Clause"
 edition = "2021"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [lib]
 name = "vegafusion_embed"
@@ -26,12 +26,12 @@ version = "1.0.79"
 [dependencies.vegafusion-core]
 path = "../vegafusion-core"
 features = [ "pyo3",]
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [dependencies.vegafusion-rt-datafusion]
 path = "../vegafusion-rt-datafusion"
 features = [ "pyarrow",]
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [dependencies.tokio]
 version = "1.18.1"

diff --git a/vegafusion-rt-datafusion/Cargo.toml b/vegafusion-rt-datafusion/Cargo.toml
@@ -6,7 +6,7 @@ harness = false
 name = "vegafusion-rt-datafusion"
 license = "BSD-3-Clause"
 edition = "2021"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [features]
 pyarrow = [ "vegafusion-core/pyarrow",]
@@ -67,7 +67,7 @@ features = [ "bundled",]
 [dependencies.vegafusion-core]
 path = "../vegafusion-core"
 features = [ "sqlgen",]
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [dependencies.serde]
 version = "1.0.137"

diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/array/length.rs
@@ -1,4 +1,4 @@
-use datafusion::arrow::array::{new_null_array, Array, Int32Array, ListArray};
+use datafusion::arrow::array::{new_null_array, Array, Float64Array, ListArray};
 use datafusion::arrow::compute::kernels;
 use datafusion::arrow::datatypes::DataType;
 use datafusion::physical_plan::udf::ScalarUDF;
@@ -21,48 +21,54 @@ pub fn make_length_udf() -> ScalarUDF {
             ColumnarValue::Scalar(value) => {
                 match value {
                     ScalarValue::List(Some(arr), _) => {
-                        ColumnarValue::Scalar(ScalarValue::from(arr.len() as i32))
+                        ColumnarValue::Scalar(ScalarValue::from(arr.len() as f64))
                     }
                     ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => {
-                        ColumnarValue::Scalar(ScalarValue::from(s.len() as i32))
+                        ColumnarValue::Scalar(ScalarValue::from(s.len() as f64))
                     }
                     _ => {
-                        // Scalar i32 null
-                        ColumnarValue::Scalar(ScalarValue::try_from(&DataType::Int32).unwrap())
+                        // Scalar f64 null
+                        ColumnarValue::Scalar(ScalarValue::try_from(&DataType::Float64).unwrap())
                     }
                 }
             }
             ColumnarValue::Array(array) => {
                 match array.data_type() {
                     DataType::Utf8 | DataType::LargeUtf8 => {
                         // String length
-                        ColumnarValue::Array(kernels::length::length(array.as_ref()).unwrap())
+                        ColumnarValue::Array(
+                            kernels::cast(
+                                &kernels::length::length(array.as_ref()).unwrap(),
+                                &DataType::Float64,
+                            )
+                            .unwrap(),
+                        )
                     }
                     DataType::FixedSizeList(_, n) => {
                         // Use scalar length
-                        ColumnarValue::Scalar(ScalarValue::from(*n))
+                        ColumnarValue::Scalar(ScalarValue::from(*n as f64))
                     }
                     DataType::List(_) => {
                         let array = array.as_any().downcast_ref::<ListArray>().unwrap();
                         let offsets = array.value_offsets();
-                        let mut length_builder = Int32Array::builder(array.len());
+                        let mut length_builder = Float64Array::builder(array.len());
 
                         for i in 0..array.len() {
-                            length_builder.append_value(offsets[i + 1] - offsets[i]);
+                            length_builder.append_value((offsets[i + 1] - offsets[i]) as f64);
                         }
 
                         ColumnarValue::Array(Arc::new(length_builder.finish()))
                     }
                     _ => {
-                        // Array of i32 nulls
-                        ColumnarValue::Array(new_null_array(&DataType::Int32, array.len()))
+                        // Array of f64
+                        ColumnarValue::Array(new_null_array(&DataType::Float64, array.len()))
                     }
                 }
             }
         })
     });
 
-    let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int32)));
+    let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64)));
     ScalarUDF::new(
         "len",
         &Signature::any(1, Volatility::Immutable),

diff --git a/vegafusion-rt-datafusion/src/expression/compiler/mod.rs b/vegafusion-rt-datafusion/src/expression/compiler/mod.rs
@@ -700,7 +700,7 @@ mod test_compile {
         println!("expr: {:?}", result_expr);
 
         let result_value = result_expr.eval_to_scalar().unwrap();
-        let expected = ScalarValue::from(3);
+        let expected = ScalarValue::from(3.0);
 
         println!("value: {:?}", result_value);
         assert_eq!(result_value, expected);
@@ -713,7 +713,7 @@ mod test_compile {
         println!("expr: {:?}", result_expr);
 
         let result_value = result_expr.eval_to_scalar().unwrap();
-        let expected = ScalarValue::from(3);
+        let expected = ScalarValue::from(3.0);
 
         println!("value: {:?}", result_value);
         assert_eq!(result_value, expected);

diff --git a/vegafusion-rt-datafusion/tests/test_expression_evaluation.rs b/vegafusion-rt-datafusion/tests/test_expression_evaluation.rs
@@ -212,7 +212,8 @@ mod test_member_access {
         case("[1, 2, 3][1]"),
         case("({foo: {bar: 10}}).foo"),
         case("({foo: {bar: 10}}).foo.bar"),
-        case("({foo: {bar: 10}})['foo']['bar']")
+        case("({foo: {bar: 10}})['foo']['bar']"),
+        case("{'a': [1, 2]}['a'][length({'a': [1, 2]}['a']) - 1.0]")
     )]
     fn test(expr: &str) {
         check_scalar_evaluation(expr, &config_a())

diff --git a/vegafusion-server/Cargo.toml b/vegafusion-server/Cargo.toml
@@ -4,7 +4,7 @@ path = "src/main.rs"
 
 [package]
 name = "vegafusion-server"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 edition = "2021"
 license = "BSD-3-Clause"
 
@@ -26,11 +26,11 @@ prost-build = "0.11.4"
 [dependencies.vegafusion-core]
 path = "../vegafusion-core"
 features = [ "tonic_support",]
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [dependencies.vegafusion-rt-datafusion]
 path = "../vegafusion-rt-datafusion"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [dependencies.tokio]
 version = "1.18.1"

diff --git a/vegafusion-wasm/Cargo.toml b/vegafusion-wasm/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "vegafusion-wasm"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 edition = "2021"
 
 [lib]
@@ -22,7 +22,7 @@ wasm-bindgen-test = "0.3.13"
 
 [dependencies.vegafusion-core]
 path = "../vegafusion-core"
-version = "1.0.0-rc1"
+version = "1.0.0-rc2"
 
 [dependencies.serde]
 version = "1.0.137"

diff --git a/vegafusion-wasm/package-lock.json b/vegafusion-wasm/package-lock.json
diff --git a/vegafusion-wasm/package.json b/vegafusion-wasm/package.json
@@ -1,6 +1,6 @@
 {
   "name": "vegafusion-wasm",
-  "version": "1.0.0-rc1",
+  "version": "1.0.0-rc2",
   "author": {
     "name": "Jon Mease",
     "email": "[email protected]",
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		__version__ = '1.0.0-rc1'
		__version__ = '1.0.0-rc2'