diff --git a/native/Cargo.lock b/native/Cargo.lock index c9301c6e3..861718902 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -114,9 +114,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba" +checksum = "a9ba0d7248932f4e2a12fb37f0a2e3ec82b3bdedbac2a1dce186e036843b8f8c" dependencies = [ "arrow-arith", "arrow-array", @@ -135,9 +135,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a" +checksum = "d60afcdc004841a5c8d8da4f4fa22d64eb19c0c01ef4bcedd77f175a7cf6e38f" dependencies = [ "arrow-array", "arrow-buffer", @@ -150,16 +150,16 @@ dependencies = [ [[package]] name = "arrow-array" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f" +checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da" dependencies = [ "ahash", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz 0.9.0", + "chrono-tz 0.10.0", "half", "hashbrown", "num", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d" +checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80" dependencies = [ "bytes", "half", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785" +checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740" +checksum = "d3a4e4d63830a341713e35d9a42452fbc6241d5f42fa5cf6a4681b8ad91370c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -218,9 +218,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65" +checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634" dependencies = [ "arrow-buffer", "arrow-schema", @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a" +checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6" +checksum = "b198b9c6fcf086501730efbbcb483317b39330a116125af7bb06467d04b352a3" dependencies = [ "arrow-array", "arrow-buffer", @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b" +checksum = "2427f37b4459a4b9e533045abe87a5183a5e0995a3fc2c2fd45027ae2cc4ef3f" dependencies = [ "arrow-array", "arrow-buffer", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb" +checksum = "15959657d92e2261a7a323517640af87f5afd9fd8a6492e424ebee2203c567f6" dependencies = [ "ahash", "arrow-array", @@ -294,18 +294,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858" +checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba" +checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a" dependencies = [ "ahash", "arrow-array", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230" +checksum = "7ab3db7c09dd826e74079661d84ed01ed06547cf75d52c2818ef776d0d852305" dependencies = [ "arrow-array", "arrow-buffer", @@ -527,12 +527,12 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" dependencies = [ "chrono", - "chrono-tz-build 0.3.0", + "chrono-tz-build 0.4.0", "phf", ] @@ -549,12 +549,11 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" dependencies = [ "parse-zoneinfo", - "phf", "phf_codegen", ] @@ -807,8 +806,7 @@ dependencies = [ [[package]] name = "datafusion" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee907b081e45e1d14e1f327e89ef134f91fcebad0bfc2dc229fa9f6044379682" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -856,8 +854,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2b914f6e33c429af7d8696c72a47ed9225d7e2b82c747ebdfa2408ed53579f" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow-schema", "async-trait", @@ -953,8 +950,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a84f8e76330c582a6b8ada0b2c599ca46cfe46b7585e458fc3f4092bc722a18" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -976,8 +972,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf08cc30d92720d557df13bd5a5696213bd5ea0f38a866d8d85055d866fba774" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "log", "tokio", @@ -986,8 +981,7 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bc4183d5c45b9f068a6f351678a0d1eb1225181424542bb75db18ec280b822" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow", "chrono", @@ -1007,8 +1001,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202119ce58e4d103e37ae64aab40d4e574c97bdd2bea994bf307b175fcbfa74d" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -1018,7 +1011,9 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", "datafusion-physical-expr-common", + "indexmap", "paste", "serde_json", "sqlparser", @@ -1029,8 +1024,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b181ce8569216abb01ef3294aa16c0a40d7d39350c2ff01ede00f167a535f2" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow", "datafusion-common", @@ -1040,8 +1034,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4124b8066444e05a24472f852e94cf56546c0f4d92d00f018f207216902712" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow", "arrow-buffer", @@ -1067,8 +1060,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94acdac235ea21810150a89751617ef2db7e32eba27f54be48a81bde2bfe119" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -1080,16 +1072,15 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "indexmap", "log", "paste", - "sqlparser", ] [[package]] name = "datafusion-functions-aggregate-common" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c9ea085bbf900bf16e2ca0f56fc56236b2e4f2e1a2cccb67bcd83c5ab4ad0ef" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -1102,8 +1093,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c882e61665ed60c5ce9b061c1e587aeb8ae5ae4bcb5e5f2465139ab25328e0f" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow", "arrow-array", @@ -1125,20 +1115,29 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a354ce96df3ca6d025093adac9fd55ca09931c9b6f2630140721a95873fde4" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "datafusion-common", "datafusion-expr", + "datafusion-functions-window-common", "datafusion-physical-expr-common", "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "42.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] name = "datafusion-optimizer" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf677c74fb7b5a1899ef52709e4a70fff3ed80bdfb4bbe495909810e83d5f39" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow", "async-trait", @@ -1157,8 +1156,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b077999f6eb6c43d6b25bc66332a3be2f693c382840f008dd763b8540f9530" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -1167,30 +1165,25 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-string", - "base64", "chrono", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", "hashbrown", - "hex", "indexmap", "itertools 0.13.0", "log", "paste", "petgraph", - "regex", ] [[package]] name = "datafusion-physical-expr-common" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce847f885c2b13bbe29f5c8b7948797131aa470af6e16d2a94f4428b4f4f1bd" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -1203,12 +1196,13 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d13238e3b9fdd62a4c18760bfef714bb990d1e1d3430e9f416aae4b3cfaa71af" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ + "arrow", "arrow-schema", "datafusion-common", "datafusion-execution", + "datafusion-expr-common", "datafusion-physical-expr", "datafusion-physical-plan", "itertools 0.13.0", @@ -1217,8 +1211,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faba6f55a7eaf0241d07d12c2640de52742646b10f754485d5192bdfe2c9ceae" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "ahash", "arrow", @@ -1232,8 +1225,8 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", @@ -1252,8 +1245,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "42.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad8d96a9b52e1aa24f9373696a815be828193efce7cb0bbd2140b6bb67d1819" +source = "git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558" dependencies = [ "arrow", "arrow-array", @@ -1767,9 +1759,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -1780,9 +1772,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -1791,9 +1783,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" dependencies = [ "lexical-util", "static_assertions", @@ -1801,18 +1793,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" dependencies = [ "lexical-util", "lexical-write-integer", @@ -1821,9 +1813,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" dependencies = [ "lexical-util", "static_assertions", @@ -2178,8 +2170,9 @@ dependencies = [ [[package]] name = "parquet" -version = "52.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=0a4d8a1#0a4d8a14b58e45ef92e31541f0b51a5b25de5f10" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e" dependencies = [ "ahash", "bytes", @@ -2734,9 +2727,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "sqlparser" -version = "0.50.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" dependencies = [ "log", "sqlparser_derive", diff --git a/native/Cargo.toml b/native/Cargo.toml index 87c988aaf..943d591d8 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -33,20 +33,20 @@ edition = "2021" rust-version = "1.75" [workspace.dependencies] -arrow = { version = "53.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } -arrow-array = { version = "53.0.0" } -arrow-buffer = { version = "53.0.0" } -arrow-data = { version = "53.0.0" } -arrow-schema = { version = "53.0.0" } -parquet = { version = "53.0.0", default-features = false, features = ["experimental"] } -datafusion-common = { version = "42.0.0" } -datafusion = { default-features = false, version = "42.0.0", features = ["unicode_expressions", "crypto_expressions"] } -datafusion-functions = { version = "42.0.0", features = ["crypto_expressions"] } -datafusion-functions-nested = { version = "42.0.0", default-features = false } -datafusion-expr = { version = "42.0.0", default-features = false } -datafusion-execution = { version = "42.0.0", default-features = false } -datafusion-physical-plan = { version = "42.0.0", default-features = false } -datafusion-physical-expr = { version = "42.0.0", default-features = false } +arrow = { version = "53.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } +arrow-array = { version = "53.1.0" } +arrow-buffer = { version = "53.1.0" } +arrow-data = { version = "53.1.0" } +arrow-schema = { version = "53.1.0" } +parquet = { version = "53.1.0", default-features = false, features = ["experimental"] } +datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2" } +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false, features = ["unicode_expressions", "crypto_expressions"] } +datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", features = ["crypto_expressions"] } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false } +datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false } +datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false } +datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false } +datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false } datafusion-comet-spark-expr = { path = "spark-expr", version = "0.4.0" } datafusion-comet-proto = { path = "proto", version = "0.4.0" } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 13f6b135f..30dce574b 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -40,7 +40,7 @@ arrow-array = { workspace = true } arrow-buffer = { workspace = true } arrow-data = { workspace = true } arrow-schema = { workspace = true } -parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "0a4d8a1", default-features = false, features = ["experimental"] } +parquet = { workspace = true, default-features = false, features = ["experimental"] } half = { version = "2.4.1", default-features = false } futures = "0.3.28" mimalloc = { version = "*", default-features = false, optional = true } diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index e45897a92..8347b7949 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -241,7 +241,7 @@ fn prepare_datafusion_session_context( session_config = session_config.set_str(key, value); } - let runtime = RuntimeEnv::new(rt_config).unwrap(); + let runtime = RuntimeEnv::try_new(rt_config).unwrap(); let mut session_ctx = SessionContext::new_with_config_rt(session_config, Arc::new(runtime)); diff --git a/native/core/src/execution/operators/filter.rs b/native/core/src/execution/operators/filter.rs index 6dfaf6bce..d9a54712d 100644 --- a/native/core/src/execution/operators/filter.rs +++ b/native/core/src/execution/operators/filter.rs @@ -203,10 +203,10 @@ impl FilterExec { ConstExpr::new(expr).with_across_partitions(true) }); // this is for statistics - eq_properties = eq_properties.add_constants(constants); + eq_properties = eq_properties.with_constants(constants); // this is for logical constant (for example: a = '1', then a could be marked as a constant) // to do: how to deal with multiple situation to represent = (for example c1 between 0 and 0) - eq_properties = eq_properties.add_constants(Self::extend_constants(input, predicate)); + eq_properties = eq_properties.with_constants(Self::extend_constants(input, predicate)); Ok(PlanProperties::new( eq_properties, input.output_partitioning().clone(), // Output Partitioning diff --git a/native/core/src/parquet/util/test_common/page_util.rs b/native/core/src/parquet/util/test_common/page_util.rs index 6303943c4..8fe5e65c2 100644 --- a/native/core/src/parquet/util/test_common/page_util.rs +++ b/native/core/src/parquet/util/test_common/page_util.rs @@ -122,7 +122,7 @@ impl DataPageBuilder for DataPageBuilderImpl { ); self.encoding = Some(encoding); let mut encoder: Box> = - get_encoder::(encoding).expect("get_encoder() should be OK"); + get_encoder::(encoding, &self.desc).expect("get_encoder() should be OK"); encoder.put(values).expect("put() should be OK"); let encoded_values = encoder .flush_buffer() diff --git a/native/spark-expr/src/scalar_funcs/hash_expressions.rs b/native/spark-expr/src/scalar_funcs/hash_expressions.rs index e76314f41..33c8cc5e9 100644 --- a/native/spark-expr/src/scalar_funcs/hash_expressions.rs +++ b/native/spark-expr/src/scalar_funcs/hash_expressions.rs @@ -22,7 +22,7 @@ use arrow_array::{ArrayRef, Int32Array, Int64Array, StringArray}; use datafusion::functions::crypto::{sha224, sha256, sha384, sha512}; use datafusion_common::cast::as_binary_array; use datafusion_common::{exec_err, internal_err, DataFusionError, ScalarValue}; -use datafusion_expr::{ColumnarValue, ScalarFunctionImplementation}; +use datafusion_expr::{ColumnarValue, ScalarUDF}; use std::sync::Arc; /// Spark compatible murmur3 hash (just `hash` in Spark) in vectorized execution fashion @@ -115,31 +115,31 @@ pub fn spark_xxhash64(args: &[ColumnarValue]) -> Result Result { - wrap_digest_result_as_hex_string(args, sha224().fun()) + wrap_digest_result_as_hex_string(args, sha224()) } /// `sha256` function that simulates Spark's `sha2` expression with bit width 0 or 256 pub fn spark_sha256(args: &[ColumnarValue]) -> Result { - wrap_digest_result_as_hex_string(args, sha256().fun()) + wrap_digest_result_as_hex_string(args, sha256()) } /// `sha384` function that simulates Spark's `sha2` expression with bit width 384 pub fn spark_sha384(args: &[ColumnarValue]) -> Result { - wrap_digest_result_as_hex_string(args, sha384().fun()) + wrap_digest_result_as_hex_string(args, sha384()) } /// `sha512` function that simulates Spark's `sha2` expression with bit width 512 pub fn spark_sha512(args: &[ColumnarValue]) -> Result { - wrap_digest_result_as_hex_string(args, sha512().fun()) + wrap_digest_result_as_hex_string(args, sha512()) } // Spark requires hex string as the result of sha2 functions, we have to wrap the // result of digest functions as hex string fn wrap_digest_result_as_hex_string( args: &[ColumnarValue], - digest: ScalarFunctionImplementation, + digest: Arc, ) -> Result { - let value = digest(args)?; + let value = digest.invoke(args)?; match value { ColumnarValue::Array(array) => { let binary_array = as_binary_array(&array)?;