update to DataFusion main

Cargo.toml

@@ -13,6 +13,30 @@ members = [
     "vegafusion-jni",
 ]
 
+[workspace.dependencies]
+arrow = { version = "46.0.0", default_features = false }
+sqlparser = { version = "0.37.0" }
+chrono = { version = "0.4.31", default_features = false }
+reqwest = { version = "=0.11.13", default-features = false }
+tokio = { version = "1.32.0" }
+pyo3 = { version = "0.19" }
+
+# # Once datafusion 32.0.0 is released
+#datafusion = { version = "32.0.0" }
+#datafusion-common = { version = "32.0.0", default_features = false}
+#datafusion-expr = { version = "32.0.0" }
+#datafusion-proto = { version = "32.0.0" }
+#datafusion-physical-expr = { version = "32.0.0" }
+#datafusion-optimizer = { version = "32.0.0" }
+
+# # Git dependencies for development
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "9fa020708bab6e7293f3731d807a80284c1204bd" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "9fa020708bab6e7293f3731d807a80284c1204bd", default_features = false}
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "9fa020708bab6e7293f3731d807a80284c1204bd"  }
+datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev = "9fa020708bab6e7293f3731d807a80284c1204bd"  }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "9fa020708bab6e7293f3731d807a80284c1204bd"  }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "9fa020708bab6e7293f3731d807a80284c1204bd"  }
+
 [profile.release]
 ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization
 opt-level = 3

vegafusion-common/Cargo.toml

@@ -14,11 +14,11 @@ prettyprint = [ "arrow/prettyprint",]
 thiserror = "^1.0.29"
 
 [dependencies.chrono]
-version = "0.4.23"
+workspace = true
 optional = true
 
 [dependencies.sqlparser]
-version = "0.37.0"
+workspace = true
 optional = true
 
 [dependencies.serde_json]
@@ -27,23 +27,26 @@ default_features = false
 optional = true
 
 [dependencies.arrow]
-version = "46.0.0"
-default_features = false
+workspace = true
 features = [ "ipc",]
 
 [dependencies.datafusion-common]
-version = "31.0.0"
+git = "https://github.com/apache/arrow-datafusion.git"
+rev = "9fa020708bab6e7293f3731d807a80284c1204bd"
 default_features = false
 
 [dependencies.datafusion-expr]
-version = "31.0.0"
+git = "https://github.com/apache/arrow-datafusion.git"
+rev = "9fa020708bab6e7293f3731d807a80284c1204bd"
 
 [dependencies.datafusion-proto]
-version = "31.0.0"
+git = "https://github.com/apache/arrow-datafusion.git"
+rev = "9fa020708bab6e7293f3731d807a80284c1204bd"
 optional = true
+default-features = false
 
 [dependencies.pyo3]
-version = "0.19"
+workspace = true
 optional = true
 
 [dependencies.jni]

vegafusion-common/src/data/scalar.rs

@@ -69,7 +69,7 @@ impl ScalarValueHelpers for ScalarValue {
                         .iter()
                         .map(ScalarValue::from_json)
                         .collect::<Result<Vec<ScalarValue>>>()?;
-                    let dtype = elements[0].get_datatype();
+                    let dtype = elements[0].data_type();
                     (elements, dtype)
                 };
 

vegafusion-common/src/data/table.rs

@@ -201,7 +201,7 @@ impl VegaFusionTable {
             }
         }
 
-        let dtype = elements[0].get_datatype();
+        let dtype = elements[0].data_type();
         Ok(ScalarValue::List(
             Some(elements),
             Arc::new(Field::new("item", dtype, true)),

vegafusion-core/Cargo.toml

@@ -26,7 +26,7 @@ rand = "0.8.5"
 json-patch = "1.0.0"
 
 [dependencies.sqlparser]
-version = "0.37.0"
+workspace = true
 optional = true
 
 [dependencies.serde_json]
@@ -39,11 +39,11 @@ features = [ "json", "sqlparser",]
 version = "1.4.0"
 
 [dependencies.datafusion-common]
-version = "31.0.0"
+workspace = true
 default_features = false
 
 [dependencies.pyo3]
-version = "0.19"
+workspace = true
 optional = true
 
 [dependencies.serde]

vegafusion-dataframe/Cargo.toml

@@ -13,11 +13,11 @@ path = "../vegafusion-common"
 version = "1.4.0"
 
 [dependencies.datafusion-common]
-version = "31.0.0"
+workspace = true
 
 [dependencies.datafusion-expr]
-version = "31.0.0"
+workspace = true
 
 [dependencies.arrow]
-version = "46.0.0"
+workspace = true
 default_features = false

vegafusion-datafusion-udfs/Cargo.toml

@@ -8,7 +8,7 @@ description = "Custom DataFusion UDFs used by VegaFusion"
 [dependencies]
 ordered-float = "3.6.0"
 lazy_static = "^1.4.0"
-chrono = "0.4.23"
+chrono = { workspace = true }
 chrono-tz = "0.8.1"
 regex = "^1.5.5"
 
@@ -17,4 +17,4 @@ path = "../vegafusion-common"
 version = "1.4.0"
 
 [dependencies.datafusion-physical-expr]
-version = "31.0.0"
+workspace = true

vegafusion-datafusion-udfs/src/udfs/array/indexof.rs

@@ -37,7 +37,7 @@ fn make_indexof_udf() -> ScalarUDF {
         let arg = &args[1];
         Ok(match arg {
             ColumnarValue::Scalar(value) => {
-                let value_dtype = value.get_datatype();
+                let value_dtype = value.data_type();
                 if is_numeric_datatype(&value_dtype) && is_numeric_datatype(&array_dtype) {
                     let indices = build_notnan_index_map(array.as_slice());
                     if let Ok(value) = value.to_f64() {