Merge branch 'main' into feature/11268-scalar-funciton-map-v3

apache · Jul 9, 2024 · f9bc626 · f9bc626
2 parents e288b9f + e65c3e9
commit f9bc626
Show file tree

Hide file tree

Showing 170 changed files with 6,473 additions and 1,720 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -581,4 +581,4 @@ jobs:
         run: cargo msrv verify
       - name: Check datafusion-cli
         working-directory: datafusion-cli
-        run: cargo msrv verify
+        run: cargo msrv verify
diff --git a/Cargo.toml b/Cargo.toml
@@ -53,7 +53,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/datafusion"
 rust-version = "1.76"
-version = "39.0.0"
+version = "40.0.0"
 
 [workspace.dependencies]
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
@@ -64,45 +64,45 @@ version = "39.0.0"
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
-arrow = { version = "52.0.0", features = [
+arrow = { version = "52.1.0", features = [
     "prettyprint",
 ] }
-arrow-array = { version = "52.0.0", default-features = false, features = [
+arrow-array = { version = "52.1.0", default-features = false, features = [
     "chrono-tz",
 ] }
-arrow-buffer = { version = "52.0.0", default-features = false }
-arrow-flight = { version = "52.0.0", features = [
+arrow-buffer = { version = "52.1.0", default-features = false }
+arrow-flight = { version = "52.1.0", features = [
     "flight-sql-experimental",
 ] }
-arrow-ipc = { version = "52.0.0", default-features = false, features = [
+arrow-ipc = { version = "52.1.0", default-features = false, features = [
     "lz4",
 ] }
-arrow-ord = { version = "52.0.0", default-features = false }
-arrow-schema = { version = "52.0.0", default-features = false }
-arrow-string = { version = "52.0.0", default-features = false }
+arrow-ord = { version = "52.1.0", default-features = false }
+arrow-schema = { version = "52.1.0", default-features = false }
+arrow-string = { version = "52.1.0", default-features = false }
 async-trait = "0.1.73"
 bigdecimal = "=0.4.1"
 bytes = "1.4"
 chrono = { version = "0.4.34", default-features = false }
 ctor = "0.2.0"
 dashmap = "5.5.0"
-datafusion = { path = "datafusion/core", version = "39.0.0", default-features = false }
-datafusion-common = { path = "datafusion/common", version = "39.0.0", default-features = false }
-datafusion-common-runtime = { path = "datafusion/common-runtime", version = "39.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "39.0.0" }
-datafusion-expr = { path = "datafusion/expr", version = "39.0.0" }
-datafusion-functions = { path = "datafusion/functions", version = "39.0.0" }
-datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "39.0.0" }
-datafusion-functions-array = { path = "datafusion/functions-array", version = "39.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "39.0.0", default-features = false }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "39.0.0", default-features = false }
-datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "39.0.0", default-features = false }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "39.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "39.0.0" }
-datafusion-proto-common = { path = "datafusion/proto-common", version = "39.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "39.0.0" }
-datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "39.0.0" }
-datafusion-substrait = { path = "datafusion/substrait", version = "39.0.0" }
+datafusion = { path = "datafusion/core", version = "40.0.0", default-features = false }
+datafusion-common = { path = "datafusion/common", version = "40.0.0", default-features = false }
+datafusion-common-runtime = { path = "datafusion/common-runtime", version = "40.0.0" }
+datafusion-execution = { path = "datafusion/execution", version = "40.0.0" }
+datafusion-expr = { path = "datafusion/expr", version = "40.0.0" }
+datafusion-functions = { path = "datafusion/functions", version = "40.0.0" }
+datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "40.0.0" }
+datafusion-functions-array = { path = "datafusion/functions-array", version = "40.0.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "40.0.0", default-features = false }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "40.0.0", default-features = false }
+datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "40.0.0", default-features = false }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "40.0.0" }
+datafusion-proto = { path = "datafusion/proto", version = "40.0.0" }
+datafusion-proto-common = { path = "datafusion/proto-common", version = "40.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "40.0.0" }
+datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "40.0.0" }
+datafusion-substrait = { path = "datafusion/substrait", version = "40.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
@@ -114,7 +114,7 @@ log = "^0.4"
 num_cpus = "1.13.0"
 object_store = { version = "0.10.1", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "52.0.0", default-features = false, features = [
+parquet = { version = "52.1.0", default-features = false, features = [
     "arrow",
     "async",
     "object_store",

diff --git a/ci/scripts/rust_example.sh b/ci/scripts/rust_example.sh
@@ -19,7 +19,6 @@
 
 set -ex
 cd datafusion-examples/examples/
-cargo fmt --all -- --check
 cargo check --examples
 
 files=$(ls .)

diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "datafusion-cli"
 description = "Command Line Client for DataFusion query engine."
-version = "39.0.0"
+version = "40.0.0"
 authors = ["Apache DataFusion <[email protected]>"]
 edition = "2021"
 keywords = ["arrow", "datafusion", "query", "sql"]
@@ -30,12 +30,12 @@ rust-version = "1.76"
 readme = "README.md"
 
 [dependencies]
-arrow = { version = "52.0.0" }
+arrow = { version = "52.1.0" }
 async-trait = "0.1.41"
 aws-config = "0.55"
 aws-credential-types = "0.55"
 clap = { version = "3", features = ["derive", "cargo"] }
-datafusion = { path = "../datafusion/core", version = "39.0.0", features = [
+datafusion = { path = "../datafusion/core", version = "40.0.0", features = [
     "avro",
     "crypto_expressions",
     "datetime_expressions",
@@ -51,7 +51,7 @@ futures = "0.3"
 mimalloc = { version = "0.1", default-features = false }
 object_store = { version = "0.10.1", features = ["aws", "gcp", "http"] }
 parking_lot = { version = "0.12" }
-parquet = { version = "52.0.0", default-features = false }
+parquet = { version = "52.1.0", default-features = false }
 regex = "1.8"
 rustyline = "11.0"
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }

diff --git a/datafusion-cli/src/helper.rs b/datafusion-cli/src/helper.rs
@@ -252,7 +252,7 @@ mod tests {
     fn unescape_readline_input() -> Result<()> {
         let validator = CliHelper::default();
 
-        // shoule be valid
+        // should be valid
         let result = readline_direct(
              Cursor::new(
                  r"create external table test stored as csv location 'data.csv' options ('format.delimiter' ',');"
@@ -326,7 +326,7 @@ mod tests {
     fn sql_dialect() -> Result<()> {
         let mut validator = CliHelper::default();
 
-        // shoule be invalid in generic dialect
+        // should be invalid in generic dialect
         let result =
             readline_direct(Cursor::new(r"select 1 # 2;".as_bytes()), &validator)?;
         assert!(

diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
@@ -36,6 +36,9 @@ cd datafusion
 # Download test data
 git submodule update --init
 
+# Change to the examples directory
+cd datafusion-examples/examples
+
 # Run the `dataframe` example:
 # ... use the equivalent for other examples
 cargo run --example dataframe

diff --git a/datafusion-examples/examples/advanced_udaf.rs b/datafusion-examples/examples/advanced_udaf.rs
@@ -92,7 +92,7 @@ impl AggregateUDFImpl for GeoMeanUdaf {
     }
 
     /// This is the description of the state. accumulator's state() must match the types here.
-    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<arrow_schema::Field>> {
+    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<Field>> {
         Ok(vec![
             Field::new("prod", args.return_type.clone(), true),
             Field::new("n", DataType::UInt32, true),
@@ -339,7 +339,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
         Ok(())
     }
 
-    /// Generate output, as specififed by `emit_to` and update the intermediate state
+    /// Generate output, as specified by `emit_to` and update the intermediate state
     fn evaluate(&mut self, emit_to: datafusion_expr::EmitTo) -> Result<ArrayRef> {
         let counts = emit_to.take_needed(&mut self.counts);
         let prods = emit_to.take_needed(&mut self.prods);

diff --git a/datafusion-examples/examples/advanced_udwf.rs b/datafusion-examples/examples/advanced_udwf.rs
@@ -75,7 +75,7 @@ impl WindowUDFImpl for SmoothItUdf {
         Ok(DataType::Float64)
     }
 
-    /// Create a `PartitionEvalutor` to evaluate this function on a new
+    /// Create a `PartitionEvaluator` to evaluate this function on a new
     /// partition.
     fn partition_evaluator(&self) -> Result<Box<dyn PartitionEvaluator>> {
         Ok(Box::new(MyPartitionEvaluator::new()))

diff --git a/datafusion-examples/examples/catalog.rs b/datafusion-examples/examples/catalog.rs
@@ -83,7 +83,7 @@ async fn main() -> Result<()> {
     // register our catalog in the context
     ctx.register_catalog("dircat", Arc::new(catalog));
     {
-        // catalog was passed down into our custom catalog list since we overide the ctx's default
+        // catalog was passed down into our custom catalog list since we override the ctx's default
         let catalogs = catlist.catalogs.read().unwrap();
         assert!(catalogs.contains_key("dircat"));
     };

diff --git a/datafusion-examples/examples/expr_api.rs b/datafusion-examples/examples/expr_api.rs
@@ -46,7 +46,7 @@ use datafusion_expr::{AggregateExt, ColumnarValue, ExprSchemable, Operator};
 ///
 /// The code in this example shows how to:
 /// 1. Create [`Expr`]s using different APIs: [`main`]`
-/// 2. Use the fluent API to easly create complex [`Expr`]s:  [`expr_fn_demo`]
+/// 2. Use the fluent API to easily create complex [`Expr`]s:  [`expr_fn_demo`]
 /// 3. Evaluate [`Expr`]s against data: [`evaluate_demo`]
 /// 4. Simplify expressions: [`simplify_demo`]
 /// 5. Analyze predicates for boundary ranges: [`range_analysis_demo`]