chore: Upgrade to arrow/parquet 54.1.0 and fix clippy/ci (#14415)…

… (#14453) * chore: Fixed CI * chore * chore: Fixed clippy * chore Co-authored-by: Alex Huang <[email protected]>
apache · Feb 3, 2025 · e3ea7d1 · e3ea7d1
1 parent 1a29bd3
commit e3ea7d1
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 763 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -77,21 +77,21 @@ version = "45.0.0"
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
-arrow = { version = "54.0.0", features = [
+arrow = { version = "54.1.0", features = [
     "prettyprint",
 ] }
-arrow-array = { version = "54.0.0", default-features = false, features = [
+arrow-array = { version = "54.1.0", default-features = false, features = [
     "chrono-tz",
 ] }
-arrow-buffer = { version = "54.0.0", default-features = false }
-arrow-flight = { version = "54.0.0", features = [
+arrow-buffer = { version = "54.1.0", default-features = false }
+arrow-flight = { version = "54.1.0", features = [
     "flight-sql-experimental",
 ] }
-arrow-ipc = { version = "54.0.0", default-features = false, features = [
+arrow-ipc = { version = "54.1.0", default-features = false, features = [
     "lz4",
 ] }
-arrow-ord = { version = "54.0.0", default-features = false }
-arrow-schema = { version = "54.0.0", default-features = false }
+arrow-ord = { version = "54.1.0", default-features = false }
+arrow-schema = { version = "54.1.0", default-features = false }
 async-trait = "0.1.73"
 bigdecimal = "0.4.7"
 bytes = "1.4"
@@ -133,7 +133,7 @@ itertools = "0.14"
 log = "^0.4"
 object_store = { version = "0.11.0", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "54.0.0", default-features = false, features = [
+parquet = { version = "54.1.0", default-features = false, features = [
     "arrow",
     "async",
     "object_store",

diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.81.0"
 readme = "README.md"
 
 [dependencies]
-arrow = { version = "54.0.0" }
+arrow = { version = "54.1.0" }
 async-trait = "0.1.0"
 aws-config = "1.5.0"
 aws-credential-types = "1.2.0"
@@ -57,7 +57,7 @@ home = "=0.5.11"
 mimalloc = { version = "0.1", default-features = false }
 object_store = { version = "0.11.0", features = ["aws", "gcp", "http"] }
 parking_lot = { version = "0.12" }
-parquet = { version = "54.0.0", default-features = false }
+parquet = { version = "54.1.0", default-features = false }
 regex = "1.8"
 rustyline = "15.0"
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }

diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/core/src/datasource/physical_plan/arrow_file.rs
@@ -309,10 +309,8 @@ impl FileOpener for ArrowOpener {
                     for (dict_block, dict_result) in
                         footer.dictionaries().iter().flatten().zip(dict_results)
                     {
-                        decoder.read_dictionary(
-                            dict_block,
-                            &Buffer::from_bytes(dict_result.into()),
-                        )?;
+                        decoder
+                            .read_dictionary(dict_block, &Buffer::from(dict_result))?;
                     }
 
                     // filter recordbatches according to range
@@ -348,10 +346,7 @@ impl FileOpener for ArrowOpener {
                             .zip(recordbatch_results)
                             .filter_map(move |(block, data)| {
                                 decoder
-                                    .read_record_batch(
-                                        &block,
-                                        &Buffer::from_bytes(data.into()),
-                                    )
+                                    .read_record_batch(&block, &Buffer::from(data))
                                     .transpose()
                             }),
                     )