From f75d1dc96f4793b90ed3311d647d8d03d7175993 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 18 Nov 2023 13:24:19 -0500 Subject: [PATCH 01/44] Update to DataFusion 33 --- Cargo.lock | 237 +++++++++--------- Cargo.toml | 21 +- vegafusion-common/src/data/scalar.rs | 79 +++++- vegafusion-common/src/data/table.rs | 29 +-- vegafusion-core/src/task_graph/memory.rs | 12 +- vegafusion-core/src/task_graph/task_value.rs | 2 +- vegafusion-dataframe/src/dataframe.rs | 4 +- vegafusion-datafusion-udfs/src/udafs/mod.rs | 11 +- .../src/udfs/array/indexof.rs | 7 +- .../src/udfs/array/length.rs | 5 +- .../src/udfs/array/span.rs | 21 +- .../src/udfs/datetime/date_part_tz.rs | 2 +- .../udfs/datetime/date_to_utc_timestamp.rs | 2 +- .../udfs/datetime/epoch_to_utc_timestamp.rs | 2 +- .../src/udfs/datetime/format_timestamp.rs | 2 +- .../src/udfs/datetime/from_utc_timestamp.rs | 2 +- .../src/udfs/datetime/make_utc_timestamp.rs | 4 +- .../src/udfs/datetime/str_to_utc_timestamp.rs | 2 +- .../src/udfs/datetime/timeunit.rs | 2 +- .../src/udfs/datetime/to_utc_timestamp.rs | 2 +- .../udfs/datetime/utc_timestamp_to_epoch.rs | 2 +- .../src/udfs/datetime/utc_timestamp_to_str.rs | 2 +- .../src/udfs/member/mod.rs | 13 +- vegafusion-python-embed/Cargo.toml | 2 +- .../data/vl_selection_resolve.rs | 37 ++- .../data/vl_selection_test.rs | 19 +- .../src/expression/compiler/member.rs | 2 +- .../src/expression/compiler/mod.rs | 46 +--- vegafusion-runtime/src/transform/bin.rs | 14 +- vegafusion-runtime/src/transform/extent.rs | 10 +- .../src/transform/identifier.rs | 2 +- .../tests/test_image_comparison.rs | 168 ++++++------- .../tests/test_vegajs_runtime.rs | 8 +- vegafusion-sql/src/compile/expr.rs | 21 +- vegafusion-sql/src/compile/function_arg.rs | 14 +- vegafusion-sql/src/compile/scalar.rs | 21 +- vegafusion-sql/src/compile/select.rs | 13 +- .../src/connection/datafusion_conn.rs | 6 +- vegafusion-sql/src/dataframe/mod.rs | 27 +- vegafusion-sql/src/dialect/mod.rs | 22 +- .../src/dialect/transforms/date_add_tz.rs | 18 ++ .../src/dialect/transforms/date_part_tz.rs | 16 ++ .../transforms/date_to_utc_timestamp.rs | 17 ++ .../src/dialect/transforms/date_trunc_tz.rs | 20 ++ .../transforms/epoch_ms_to_utc_timestamp.rs | 16 ++ .../dialect/transforms/make_utc_timestamp.rs | 24 ++ .../transforms/str_to_utc_timestamp.rs | 17 ++ .../dialect/transforms/to_utc_timestamp.rs | 6 + .../transforms/utc_timestamp_to_epoch_ms.rs | 12 + .../transforms/utc_timestamp_to_str.rs | 18 ++ 50 files changed, 645 insertions(+), 416 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fb3de97d..a03d9bf1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,9 +141,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" +checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378" dependencies = [ "ahash", "arrow-arith", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +checksum = "ef983914f477d4278b068f13b3224b7d19eb2b807ac9048544d3bfebdf2554c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b" dependencies = [ "ahash", "arrow-buffer", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +checksum = "55512d988c6fbd76e514fd3ff537ac50b0a675da5a245e4fdad77ecfd654205f" dependencies = [ "bytes", "half 2.2.1", @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +checksum = "655ee51a2156ba5375931ce21c1b2494b1d9260e6dcdc6d4db9060c37dc3325b" dependencies = [ "arrow-array", "arrow-buffer", @@ -225,9 +225,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" +checksum = "258bb689997ad5b6660b3ce3638bd6b383d668ec555ed41ad7c6559cbb2e4f91" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,9 +244,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +checksum = "6dc2b9fec74763427e2e5575b8cc31ce96ba4c9b4eb05ce40e0616d9fad12461" dependencies = [ "arrow-buffer", "arrow-schema", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +checksum = "6eaa6ab203cc6d89b7eaa1ac781c1dfeef325454c5d5a0419017f95e6bafc03c" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" +checksum = "fb64e30d9b73f66fdc5c52d5f4cf69bbf03d62f64ffeafa0715590a5320baed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +checksum = "f9a818951c0d11c428dda03e908175969c262629dd20bd0850bd6c7a8c3bfe48" dependencies = [ "arrow-array", "arrow-buffer", @@ -305,9 +305,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3" dependencies = [ "ahash", "arrow-array", @@ -320,18 +320,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +checksum = "aaf4d737bba93da59f16129bec21e087aed0be84ff840e74146d4703879436cb" dependencies = [ "bitflags 2.1.0", ] [[package]] name = "arrow-select" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a" dependencies = [ "ahash", "arrow-array", @@ -343,9 +343,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" +checksum = "b15aed5624bb23da09142f58502b59c23f5bea607393298bb81dab1ce60fc769" dependencies = [ "arrow-array", "arrow-buffer", @@ -354,7 +354,7 @@ dependencies = [ "arrow-select", "num", "regex", - "regex-syntax", + "regex-syntax 0.8.2", ] [[package]] @@ -407,8 +407,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd", - "zstd-safe", + "zstd 0.12.3+zstd.1.5.2", + "zstd-safe 6.0.4+zstd.1.5.4", ] [[package]] @@ -1238,9 +1238,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7014432223f4d721cb9786cd88bb89e7464e0ba984d4a7f49db7787f5f268674" +checksum = "676796427e638d85e9eadf13765705212be60b34f8fc5d3934d95184c63ca1b4" dependencies = [ "ahash", "arrow", @@ -1271,24 +1271,23 @@ dependencies = [ "object_store", "parking_lot 0.12.1", "parquet", - "percent-encoding", "pin-project-lite", "rand", - "sqlparser 0.38.0", + "sqlparser", "tempfile", "tokio", "tokio-util", "url", "uuid", "xz2", - "zstd", + "zstd 0.13.0", ] [[package]] name = "datafusion-common" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3903ed8f102892f17b48efa437f3542159241d41c564f0d1e78efdc5e663aa" +checksum = "31e23b3d21a6531259d291bd20ce59282ea794bda1018b0a1e278c13cd52e50c" dependencies = [ "ahash", "arrow", @@ -1301,14 +1300,14 @@ dependencies = [ "object_store", "parquet", "pyo3", - "sqlparser 0.38.0", + "sqlparser", ] [[package]] name = "datafusion-execution" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780b73b2407050e53f51a9781868593f694102c59e622de9a8aafc0343c4f237" +checksum = "4de1fd0d8db0f2b8e4f4121bfa1c7c09d3a5c08a0a65c2229cd849eb65cff855" dependencies = [ "arrow", "chrono", @@ -1327,24 +1326,24 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c382676338d8caba6c027ba0da47260f65ffedab38fda78f6d8043f607557c" +checksum = "18e227fe88bf6730cab378d0cd8fc4c6b2ea42bc7e414a8ea9feba7225932735" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", - "sqlparser 0.38.0", + "sqlparser", "strum 0.25.0", "strum_macros 0.25.1", ] [[package]] name = "datafusion-optimizer" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2904a432f795484fd45e29ded4537152adb60f636c05691db34fcd94c92c96" +checksum = "5c6648e62ea7605b9bfcd87fdc9d67e579c3b9ac563a87734ae5fe6d79ee4547" dependencies = [ "arrow", "async-trait", @@ -1355,19 +1354,20 @@ dependencies = [ "hashbrown 0.14.0", "itertools 0.11.0", "log", - "regex-syntax", + "regex-syntax 0.8.2", ] [[package]] name = "datafusion-physical-expr" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b4968e9a998dc0476c4db7a82f280e2026b25f464e4aa0c3bb9807ee63ddfd" +checksum = "f32b8574add16a32411a9b3fb3844ac1fc09ab4e7be289f86fd56d620e4f2508" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "base64", "blake2", @@ -1394,9 +1394,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd0d1fe54e37a47a2d58a1232c22786f2c28ad35805fdcd08f0253a8b0aaa90" +checksum = "796abd77d5bfecd9e5275a99daf0ec45f5b3a793ec431349ce8211a67826fd22" dependencies = [ "ahash", "arrow", @@ -1425,9 +1425,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca1db9605e1f545b852fa9dd05f03339b333c9f2814a4e33b9ac7e9e586a1014" +checksum = "26de2592417beb20f73f29b131a04d7de14e2a6336c631554d611584b4306236" dependencies = [ "arrow", "chrono", @@ -1440,16 +1440,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b568d44c87ead99604d704f942e257c8a236ee1bbf890ee3e034ad659dcb2c21" +checksum = "ced70b8a5648ba7b95c61fc512183c33287ffe2c9f22ffe22700619d7d48c84f" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.38.0", + "sqlparser", ] [[package]] @@ -2090,9 +2090,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "instant" @@ -2412,23 +2412,12 @@ dependencies = [ ] [[package]] -name = "lz4" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" -dependencies = [ - "libc", - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" +name = "lz4_flex" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" dependencies = [ - "cc", - "libc", + "twox-hash", ] [[package]] @@ -2637,9 +2626,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d359e231e5451f4f9fa889d56e3ce34f8724f1a61db2107739359717cf2bbf08" +checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" dependencies = [ "async-trait", "base64", @@ -2648,7 +2637,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.10.5", + "itertools 0.11.0", "parking_lot 0.12.1", "percent-encoding", "quick-xml", @@ -2756,9 +2745,9 @@ dependencies = [ [[package]] name = "parquet" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597" dependencies = [ "ahash", "arrow-array", @@ -2775,7 +2764,7 @@ dependencies = [ "flate2", "futures", "hashbrown 0.14.0", - "lz4", + "lz4_flex", "num", "num-bigint", "object_store", @@ -2785,7 +2774,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd", + "zstd 0.13.0", ] [[package]] @@ -3090,9 +3079,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb88ae05f306b4bfcde40ac4a51dc0b05936a9207a4b75b798c7729c4258a59" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", @@ -3107,9 +3096,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554db24f0b3c180a9c0b1268f91287ab3f17c162e15b54caaae5a6b3773396b0" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -3117,9 +3106,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "922ede8759e8600ad4da3195ae41259654b9c55da4f7eec84a0ccc7d067a70a4" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -3127,32 +3116,33 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5caec6a1dd355964a841fcbeeb1b89fe4146c87295573f94228911af3cc5a2" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b78ccbb160db1556cdb6fd96c50334c5d4ec44dc5e0a968d0a1208fa0efa8b" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "pythonize" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e35b716d430ace57e2d1b4afb51c9e5b7c46d2bce72926e07f9be6a98ced03e" +checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850" dependencies = [ "pyo3", "serde", @@ -3166,9 +3156,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", "serde", @@ -3262,7 +3252,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.3.8", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -3279,7 +3269,7 @@ checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -3288,6 +3278,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "relative-path" version = "1.9.0" @@ -3722,18 +3718,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ae05a8250b968a3f7db93155a84d68b2e6cea1583949af5ca5b5170c76c075" -dependencies = [ - "log", -] - -[[package]] -name = "sqlparser" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" +checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" dependencies = [ "log", "sqlparser_derive", @@ -4335,9 +4322,9 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "untrusted" @@ -4391,7 +4378,7 @@ dependencies = [ "object_store", "pyo3", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "thiserror", ] @@ -4418,7 +4405,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "thiserror", "tonic", "tonic-build", @@ -4526,7 +4513,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "tempfile", "test-case", "tokio", @@ -4585,7 +4572,7 @@ dependencies = [ "rstest_reuse", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "tempfile", "tokio", "toml", @@ -5014,7 +5001,16 @@ version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe", + "zstd-safe 6.0.4+zstd.1.5.4", +] + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe 7.0.0", ] [[package]] @@ -5027,6 +5023,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.7+zstd.1.5.4" diff --git a/Cargo.toml b/Cargo.toml index e69b749f8..1795da463 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,22 +14,23 @@ members = [ ] [workspace.dependencies] -arrow = { version = "47.0.0", default_features = false } -sqlparser = { version = "0.37.0" } +arrow = { version = "48.0.1", default_features = false } +sqlparser = { version = "0.39.0" } chrono = { version = "0.4.31", default_features = false } reqwest = { version = "0.11.22", default-features = false } tokio = { version = "1.32.0" } -pyo3 = { version = "0.19" } +pyo3 = { version = "0.20.0" } +pythonize = { version = "0.20.0" } prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } -object_store = { version="0.7" } +object_store = { version= "0.7.1" } -datafusion = { version = "32.0.0" } -datafusion-common = { version = "32.0.0", default_features = false} -datafusion-expr = { version = "32.0.0" } -datafusion-proto = { version = "32.0.0" } -datafusion-physical-expr = { version = "32.0.0" } -datafusion-optimizer = { version = "32.0.0" } +datafusion = { version = "33.0.0" } +datafusion-common = { version = "33.0.0", default_features = false} +datafusion-expr = { version = "33.0.0" } +datafusion-proto = { version = "33.0.0" } +datafusion-physical-expr = { version = "33.0.0" } +datafusion-optimizer = { version = "33.0.0" } [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 4518f58d2..df2d3ce5e 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -1,10 +1,13 @@ use crate::error::{Result, VegaFusionError}; +use arrow::array::{new_empty_array, Array, ArrayRef, ListArray}; +use datafusion_common::DataFusionError; +use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; #[cfg(feature = "json")] use { - arrow::datatypes::{DataType, Field}, + arrow::datatypes::DataType, serde_json::{Map, Value}, std::ops::Deref, std::sync::Arc, @@ -62,18 +65,18 @@ impl ScalarValueHelpers for ScalarValue { } } Value::Array(elements) => { - let (elements, dtype) = if elements.is_empty() { - (Vec::new(), DataType::Float64) + let array: ListArray = if elements.is_empty() { + array_into_list_array(Arc::new(new_empty_array(&DataType::Float64))) } else { let elements: Vec<_> = elements .iter() .map(ScalarValue::from_json) .collect::>>()?; - let dtype = elements[0].data_type(); - (elements, dtype) + + array_into_list_array(ScalarValue::iter_to_array(elements)?) }; - ScalarValue::List(Some(elements), Arc::new(Field::new("item", dtype, true))) + ScalarValue::List(Arc::new(array)) } }; Ok(scalar_value) @@ -132,13 +135,12 @@ impl ScalarValueHelpers for ScalarValue { ScalarValue::IntervalDayTime(Some(_v)) => { unimplemented!() } - ScalarValue::List(Some(v), _) => Value::Array( - v.clone() - .into_iter() - .map(|v| v.to_json()) + ScalarValue::List(a) => Value::Array( + a.list_el_to_scalar_vec()? + .iter() + .map(|s| s.to_json()) .collect::>>()?, ), - ScalarValue::List(None, _) => Value::Array(Vec::new()), ScalarValue::Struct(Some(v), fields) => { let mut pairs: Map = Default::default(); for (val, field) in v.iter().zip(fields.deref()) { @@ -173,7 +175,8 @@ impl ScalarValueHelpers for ScalarValue { } fn to_f64x2(&self) -> Result<[f64; 2]> { - if let ScalarValue::List(Some(elements), _) = self { + if let ScalarValue::List(array) = self { + let elements = array.list_el_to_scalar_vec()?; if let [v0, v1] = elements.as_slice() { return Ok([v0.to_f64()?, v1.to_f64()?]); } @@ -195,3 +198,55 @@ impl ScalarValueHelpers for ScalarValue { }) } } + +pub trait ArrayRefHelpers { + fn to_scalar_vec(&self) -> std::result::Result, DataFusionError>; + + fn list_el_to_scalar_vec(&self) -> std::result::Result, DataFusionError>; + + fn list_el_len(&self) -> std::result::Result; + + fn list_el_dtype(&self) -> std::result::Result; +} + +impl ArrayRefHelpers for ArrayRef { + /// Convert ArrayRef into vector of ScalarValues + fn to_scalar_vec(&self) -> std::result::Result, DataFusionError> { + (0..self.len()) + .map(|i| Ok(ScalarValue::try_from_array(self, i)?)) + .collect::, DataFusionError>>() + } + + /// Extract Vec for single element ListArray (as is stored inside ScalarValue::List(arr)) + fn list_el_to_scalar_vec(&self) -> std::result::Result, DataFusionError> { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_to_scalar_vec called on non-List type".to_string(), + ))?; + a.value(0).to_scalar_vec() + } + + /// Extract length of single element ListArray + fn list_el_len(&self) -> std::result::Result { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_len called on non-List type".to_string(), + ))?; + Ok(a.value(0).len()) + } + + /// Extract data type of single element ListArray + fn list_el_dtype(&self) -> std::result::Result { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_len called on non-List type".to_string(), + ))?; + Ok(a.value(0).data_type().clone()) + } +} diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index 37b896b02..a1cbe43fa 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -13,6 +13,7 @@ use crate::{ error::{Result, ResultWithContext, VegaFusionError}, }; +use arrow::array::new_empty_array; #[cfg(feature = "prettyprint")] use arrow::util::pretty::pretty_format_batches; use std::{ @@ -41,6 +42,7 @@ use { #[cfg(feature = "base64")] use base64::{engine::general_purpose, Engine as _}; +use datafusion_common::utils::array_into_list_array; #[derive(Clone, Debug)] pub struct VegaFusionTable { @@ -182,30 +184,11 @@ impl VegaFusionTable { pub fn to_scalar_value(&self) -> Result { if self.num_rows() == 0 { // Return empty list with (arbitrary) Float64 type - let dtype = DataType::Float64; - return Ok(ScalarValue::List( - Some(Vec::new()), - Arc::new(Field::new("item", dtype, true)), - )); + let array = Arc::new(new_empty_array(&DataType::Float64)); + return Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))); } - - let mut elements: Vec = Vec::new(); - for batch in &self.batches { - let array = Arc::new(StructArray::from(batch.clone())) as ArrayRef; - - for i in 0..array.len() { - let scalar = ScalarValue::try_from_array(&array, i).with_context(|| { - "Failed to convert record batch row to ScalarValue".to_string() - })?; - elements.push(scalar) - } - } - - let dtype = elements[0].data_type(); - Ok(ScalarValue::List( - Some(elements), - Arc::new(Field::new("item", dtype, true)), - )) + let array = Arc::new(StructArray::from(self.to_record_batch()?)) as ArrayRef; + Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))) } #[cfg(feature = "json")] diff --git a/vegafusion-core/src/task_graph/memory.rs b/vegafusion-core/src/task_graph/memory.rs index 68676b51b..b7d753bb0 100644 --- a/vegafusion-core/src/task_graph/memory.rs +++ b/vegafusion-core/src/task_graph/memory.rs @@ -47,17 +47,7 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { ScalarValue::LargeUtf8(Some(s)) => size_of_val(s.as_bytes()) + size_of::(), ScalarValue::Binary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), ScalarValue::LargeBinary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), - ScalarValue::List(Some(values), field) => { - let values_bytes: usize = size_of::>() - + values - .iter() - .map(|v| size_of::() + inner_size_of_scalar(v)) - .sum::(); - - let dtype_bytes = size_of::() + inner_size_of_dtype(field.data_type()); - - values_bytes + dtype_bytes - } + ScalarValue::List(array) => size_of::>() + size_of_array_ref(array), ScalarValue::Struct(Some(values), fields) => { let values_bytes: usize = size_of::>() + values diff --git a/vegafusion-core/src/task_graph/task_value.rs b/vegafusion-core/src/task_graph/task_value.rs index 734ca1eb4..f84d32328 100644 --- a/vegafusion-core/src/task_graph/task_value.rs +++ b/vegafusion-core/src/task_graph/task_value.rs @@ -70,7 +70,7 @@ impl TryFrom<&TaskValue> for ProtoTaskValue { fn try_from(value: &TaskValue) -> std::result::Result { match value { TaskValue::Scalar(scalar) => { - let scalar_array = scalar.to_array(); + let scalar_array = scalar.to_array()?; let scalar_rb = RecordBatch::try_from_iter(vec![("value", scalar_array)])?; let ipc_bytes = VegaFusionTable::from(scalar_rb).to_ipc_bytes()?; Ok(Self { diff --git a/vegafusion-dataframe/src/dataframe.rs b/vegafusion-dataframe/src/dataframe.rs index 7e2f39b45..93cfecfba 100644 --- a/vegafusion-dataframe/src/dataframe.rs +++ b/vegafusion-dataframe/src/dataframe.rs @@ -112,7 +112,7 @@ pub trait DataFrame: Send + Sync + 'static { async fn with_index(&self, index_name: &str) -> Result> { if self.schema().column_with_name(index_name).is_some() { // Column is already present, don't overwrite - self.select(vec![Expr::Wildcard]).await + self.select(vec![Expr::Wildcard { qualifier: None }]).await } else { let selections = vec![ Expr::WindowFunction(expr::WindowFunction { @@ -129,7 +129,7 @@ pub trait DataFrame: Send + Sync + 'static { }, }) .alias(index_name), - Expr::Wildcard, + Expr::Wildcard { qualifier: None }, ]; self.select(selections).await } diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index 7e6fbc27a..4f3a7d41b 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use vegafusion_common::arrow::array::{Array, ArrayRef, UInt32Array}; use vegafusion_common::arrow::compute::sort_to_indices; use vegafusion_common::arrow::datatypes::{DataType, Field, FieldRef}; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{create_udaf, Accumulator, AggregateUDF, Volatility}; @@ -19,8 +19,8 @@ pub(crate) struct PercentileContAccumulator { impl Accumulator for PercentileContAccumulator { fn state(&self) -> Result, DataFusionError> { - let state = ScalarValue::new_list(Some(self.all_values.clone()), self.data_type.clone()); - Ok(vec![state]) + let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type); + Ok(vec![ScalarValue::List(Arc::new(state))]) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<(), DataFusionError> { @@ -43,14 +43,13 @@ impl Accumulator for PercentileContAccumulator { assert!(matches!(array.data_type(), DataType::List(_))); for index in 0..array.len() { match ScalarValue::try_from_array(array, index)? { - ScalarValue::List(Some(values), _) => { - for scalar in values { + ScalarValue::List(array) => { + for scalar in array.list_el_to_scalar_vec()? { if !scalar_is_non_finite(&scalar) { self.all_values.push(scalar); } } } - ScalarValue::List(None, _) => {} // skip empty state v => { return Err(DataFusionError::Internal(format!( "unexpected state in percentile_cont. Expected DataType::List, got {v:?}" diff --git a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs index 9145ea710..e223518ec 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs @@ -6,7 +6,7 @@ use vegafusion_common::arrow::array::{ }; use vegafusion_common::arrow::compute::cast; use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, @@ -24,8 +24,9 @@ fn make_indexof_udf() -> ScalarUDF { let indexof_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { // Signature ensures there is a single argument let (array, array_dtype) = match &args[0] { - ColumnarValue::Scalar(ScalarValue::List(Some(scalar_array), field)) => { - (scalar_array.clone(), field.data_type().clone()) + ColumnarValue::Scalar(ScalarValue::List(array)) => { + let scalar_array = array.list_el_to_scalar_vec()?; + (scalar_array, array.list_el_dtype()?) } _ => { return Err(DataFusionError::Internal( diff --git a/vegafusion-datafusion-udfs/src/udfs/array/length.rs b/vegafusion-datafusion-udfs/src/udfs/array/length.rs index 3f166cab2..eb0da7603 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/length.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/length.rs @@ -21,8 +21,9 @@ fn make_length_udf() -> ScalarUDF { Ok(match arg { ColumnarValue::Scalar(value) => { match value { - ScalarValue::List(Some(arr), _) => { - ColumnarValue::Scalar(ScalarValue::from(arr.len() as f64)) + ScalarValue::List(arr) => { + let arr = arr.as_any().downcast_ref::().unwrap(); + ColumnarValue::Scalar(ScalarValue::from(arr.value(0).len() as f64)) } ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => { ColumnarValue::Scalar(ScalarValue::from(s.len() as f64)) diff --git a/vegafusion-datafusion-udfs/src/udfs/array/span.rs b/vegafusion-datafusion-udfs/src/udfs/array/span.rs index 925818e47..90319efdb 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/span.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/span.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use vegafusion_common::arrow::array::{Array, ListArray}; use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; @@ -25,21 +26,31 @@ fn make_span_udf() -> ScalarUDF { // Span of scalar (including null) is 0 ColumnarValue::Scalar(ScalarValue::from(0.0)) } - ScalarValue::List(Some(arr), element_type) => { - match element_type.data_type() { + ScalarValue::List(arr) => { + // Unwrap single element ListArray + let arr = arr.as_any().downcast_ref::().unwrap(); + let arr = arr.value(0); + match arr.data_type() { DataType::Float64 => { if arr.is_empty() { // Span of empty array is 0 ColumnarValue::Scalar(ScalarValue::from(0.0)) } else { - let first = arr.first().unwrap().to_f64().unwrap(); - let last = arr.last().unwrap().to_f64().unwrap(); + let first = ScalarValue::try_from_array(&arr, 0) + .unwrap() + .to_f64() + .unwrap(); + let last = ScalarValue::try_from_array(&arr, arr.len() - 1) + .unwrap() + .to_f64() + .unwrap(); ColumnarValue::Scalar(ScalarValue::from(last - first)) } } _ => { return Err(DataFusionError::Internal(format!( - "Unexpected element type for span function: {element_type}" + "Unexpected element type for span function: {}", + arr.data_type() ))) } } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs index d358ae35a..77eaffd4a 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs @@ -18,7 +18,7 @@ fn make_date_part_tz_udf() -> ScalarUDF { // [1] data array let timestamp_array = match &args[1] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = to_timestamp_ms(×tamp_array)?; diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs index f375023d8..921a7ddec 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs @@ -19,7 +19,7 @@ fn make_date_to_utc_timestamp() -> ScalarUDF { // [0] data array let date_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs index 7a8be8d31..7a3f041e4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs @@ -16,7 +16,7 @@ fn make_epoch_to_utc_timestamp() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = cast( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs index 35228436b..4c2048b88 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs @@ -19,7 +19,7 @@ fn make_format_timestamp_udf() -> ScalarUDF { // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] time format string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs index 190abe336..e26c7b937 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs @@ -23,7 +23,7 @@ fn make_from_utc_timestamp() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs index c53fbcf06..4b460bf1e 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs @@ -42,11 +42,11 @@ fn make_utc_timestamp_udf() -> ScalarUDF { let args = if let Some(len) = len { args.iter() .map(|arg| arg.clone().into_array(len)) - .collect::>() + .collect::, DataFusionError>>()? } else { args.iter() .map(|arg| arg.clone().into_array(1)) - .collect::>() + .collect::, DataFusionError>>()? }; // To int64 arrays diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs index 151cbd103..4bba48e40 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs @@ -334,7 +334,7 @@ fn make_str_to_utc_timestamp_udf() -> ScalarUDF { // [0] data array let str_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs index b2eccd41d..71066586c 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs @@ -39,7 +39,7 @@ fn unpack_timeunit_udf_args( DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) })?; - let timestamp = columns[0].clone().into_array(1); + let timestamp = columns[0].clone().into_array(1)?; let timestamp = process_input_datetime(×tamp, &tz)?; Ok(( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs index b35f93024..036cca048 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs @@ -22,7 +22,7 @@ fn make_to_utc_timestamp_udf() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs index 01ef27608..1fd4780e4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs @@ -18,7 +18,7 @@ fn make_utc_timestamp_to_epoch_ms_udf() -> ScalarUDF { // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let data_array = to_timestamp_ms(&data_array)?; diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs index 12c4f6ec3..88ae6749b 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs @@ -20,7 +20,7 @@ fn make_utc_timestamp_to_str_udf() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = to_timestamp_ms(×tamp_array)?; diff --git a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs index 2e7921a94..b7e912878 100644 --- a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs @@ -62,17 +62,16 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { Ok(match arg { ColumnarValue::Scalar(value) => { match value { - ScalarValue::List(Some(arr), element_dtype) => { - match arr.get(index as usize) { - Some(element) => { + ScalarValue::List(arr) => { + let arr = arr.as_any().downcast_ref::().unwrap(); + match ScalarValue::try_from_array(&arr.value(0), index as usize) { + Ok(element) => { // Scalar element of list ColumnarValue::Scalar(element.clone()) } - None => { + _ => { // out of bounds, null - ColumnarValue::Scalar( - ScalarValue::try_from(element_dtype.data_type()).unwrap(), - ) + ColumnarValue::Scalar(ScalarValue::try_from(arr.data_type())?) } } } diff --git a/vegafusion-python-embed/Cargo.toml b/vegafusion-python-embed/Cargo.toml index c482d91af..6179b7042 100644 --- a/vegafusion-python-embed/Cargo.toml +++ b/vegafusion-python-embed/Cargo.toml @@ -15,7 +15,7 @@ protobuf-src = [ "vegafusion-core/protobuf-src",] [dependencies] log = "0.4.17" env_logger = "0.10.0" -pythonize = "0.19.0" +pythonize = { workspace=true } async-trait = "0.1.73" uuid = "1.3.0" diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs index 9a63d94b3..99d21abb6 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs @@ -1,12 +1,11 @@ +use datafusion_common::utils::array_into_list_array; use datafusion_expr::{lit, Expr}; use itertools::Itertools; use std::collections::HashMap; use std::convert::TryFrom; use std::str::FromStr; use std::sync::Arc; - -use vegafusion_common::arrow::datatypes::{DataType, Field}; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; use vegafusion_common::error::{Result, VegaFusionError}; @@ -63,8 +62,8 @@ pub fn vl_selection_resolve_fn( let _op = parse_args(args)?; // Extract vector of rows for selection dataset - let rows = if let ScalarValue::List(Some(elements), _) = table.to_scalar_value()? { - elements + let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { + array.list_el_to_scalar_vec()? } else { unreachable!() }; @@ -79,16 +78,17 @@ pub fn vl_selection_resolve_fn( for (field, value) in row_spec.fields.iter().zip(&row_spec.values) { let value = match field.typ { SelectionType::Enum => { - if let ScalarValue::List(Some(elements), _) = value { - elements.clone() + if let ScalarValue::List(array) = value { + array.list_el_to_scalar_vec()? } else { vec![value.clone()] } } _ => { match &value { - ScalarValue::List(Some(elements), _) if elements.len() == 2 => { + ScalarValue::List(array) if array.list_el_len()? == 2 => { // Don't assume elements are in ascending order + let elements = array.list_el_to_scalar_vec()?; let first = elements[0].to_f64()?; let second = elements[1].to_f64()?; @@ -98,9 +98,6 @@ pub fn vl_selection_resolve_fn( (second, first) }; vec![ScalarValue::from(low), ScalarValue::from(high)] - // ScalarValue::List(Some(Box::new(vec![ - // ScalarValue::from(low), ScalarValue::from(high) - // ])), Box::new(DataType::Float64)) } v => { return Err(VegaFusionError::internal(format!( @@ -116,20 +113,20 @@ pub fn vl_selection_resolve_fn( } } - let props: Vec<_> = props + let props = props .into_iter() .map(|(name, values)| { // Turn values into a scalar list - let dtype = values - .get(0) - .map(|s| s.data_type()) - .unwrap_or(DataType::Float64); - let values = ScalarValue::List(Some(values), Arc::new(Field::new("item", dtype, true))); - (name, values) + let values = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(values)?, + ))); + Ok((name, values)) }) + .collect::>>()?; + let props: Vec<_> = props + .iter() .sorted_by_key(|(n, _)| n.clone()) - .collect(); - + .collect::>(); let props: Vec<_> = props .iter() .map(|(name, value)| (name.as_str(), value.clone())) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index 6e6956555..0bbf318f3 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -10,7 +10,7 @@ use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::column::flat_col; -use vegafusion_common::data::scalar::ScalarValue; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValue}; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_common::datatypes::{ @@ -142,8 +142,8 @@ impl FieldSpec { let expr = match self.typ { SelectionType::Enum => { let field_type = field_col.get_type(schema)?; - let list_scalars = if let ScalarValue::List(Some(elements), _) = &values { - elements.clone() + let list_scalars = if let ScalarValue::List(array) = &values { + array.list_el_to_scalar_vec()? } else { // convert values to single element list vec![values.clone()] @@ -178,7 +178,8 @@ impl FieldSpec { }; let (low, high) = match &values { - ScalarValue::List(Some(elements), _) if elements.len() == 2 => { + ScalarValue::List(array) if array.list_el_len()? == 2 => { + let elements = array.list_el_to_scalar_vec()?; let first = Self::cast_test_scalar( elements[0].clone(), &field_dtype, @@ -405,7 +406,7 @@ impl TryFrom for SelectionRow { .get("values") .with_context(|| "Missing required property 'values'".to_string())?; let values = match struct_values.get(*values_index) { - Some(ScalarValue::List(Some(elements), _)) => elements.clone(), + Some(ScalarValue::List(array)) => array.list_el_to_scalar_vec()?, _ => { return Err(VegaFusionError::internal( "Expected 'values' to be an array".to_string(), @@ -420,8 +421,8 @@ impl TryFrom for SelectionRow { let mut fields: Vec = Vec::new(); match struct_values.get(*fields_index) { - Some(ScalarValue::List(Some(elements), _)) => { - for el in elements.iter() { + Some(ScalarValue::List(array)) => { + for el in array.list_el_to_scalar_vec()?.iter() { fields.push(FieldSpec::try_from(el.clone())?) } } @@ -508,8 +509,8 @@ pub fn vl_selection_test_fn( let op = parse_args(args)?; // Extract vector of rows for selection dataset - let rows = if let ScalarValue::List(Some(elements), _) = table.to_scalar_value()? { - elements + let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { + array.list_el_to_scalar_vec()? } else { unreachable!() }; diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index b097de44a..d1804237b 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -33,7 +33,7 @@ pub fn compile_member( )?; let prop_str = evaluated_property.to_string(); if is_numeric_datatype(&evaluated_property.data_type()) { - let int_array = cast(&evaluated_property.to_array(), &DataType::Int64).unwrap(); + let int_array = cast(&evaluated_property.to_array()?, &DataType::Int64).unwrap(); let int_array = int_array.as_any().downcast_ref::().unwrap(); index = Some(int_array.value(0) as usize); } else { diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 951763421..ebbf40cbd 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -70,8 +70,10 @@ mod test_compile { use std::convert::TryFrom; use std::ops::Deref; use std::sync::Arc; + use datafusion_common::utils::array_into_list_array; use vegafusion_common::arrow::datatypes::{DataType, Field, Schema}; use vegafusion_common::column::flat_col; + use vegafusion_core::arrow::array::{Float64Array, new_empty_array}; use vegafusion_core::arrow::datatypes::Fields; use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; use vegafusion_datafusion_udfs::udfs::object::make_object_constructor_udf; @@ -399,14 +401,8 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![ - ScalarValue::from(1.0), - ScalarValue::from(2.0), - ScalarValue::from(3.0), - ]), - Arc::new(Field::new("item", DataType::Float64, true)), - ); + + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -427,10 +423,8 @@ mod test_compile { // Check evaluated value. Empty array is given Float64 data type let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![]), - Arc::new(Field::new("item", DataType::Float64, true)), - ); + + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array(&DataType::Float64)))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -463,27 +457,13 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![ - ScalarValue::List( - Some(vec![ScalarValue::from(1.0), ScalarValue::from(2.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ScalarValue::List( - Some(vec![ScalarValue::from(3.0), ScalarValue::from(4.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ScalarValue::List( - Some(vec![ScalarValue::from(5.0), ScalarValue::from(6.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ]), - Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), - true, - )), - ); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(ScalarValue::iter_to_array( + vec![ + ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![1.0, 2.0]))))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![3.0, 4.0]))))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![5.0, 6.0]))))), + ] + ).unwrap()))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); diff --git a/vegafusion-runtime/src/transform/bin.rs b/vegafusion-runtime/src/transform/bin.rs index 553309216..66b6d4fed 100644 --- a/vegafusion-runtime/src/transform/bin.rs +++ b/vegafusion-runtime/src/transform/bin.rs @@ -7,6 +7,7 @@ use async_trait::async_trait; use datafusion_expr::lit; use datafusion_common::scalar::ScalarValue; +use datafusion_common::utils::array_into_list_array; use datafusion_common::DFSchema; use datafusion_expr::{abs, floor, when, Expr}; use float_cmp::approx_eq; @@ -15,7 +16,6 @@ use std::sync::Arc; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datatypes::to_numeric; -use vegafusion_core::arrow::datatypes::{DataType, Field}; use vegafusion_core::error::{Result, VegaFusionError}; use vegafusion_core::proto::gen::transforms::Bin; use vegafusion_core::task_graph::task_value::TaskValue; @@ -52,7 +52,9 @@ impl TransformTrait for Bin { let bin_index = floor((numeric_field.clone().sub(lit(start)).div(lit(step))).add(lit(1.0e-14))) .alias(bin_index_name); - let sql_df = sql_df.select(vec![Expr::Wildcard, bin_index]).await?; + let sql_df = sql_df + .select(vec![Expr::Wildcard { qualifier: None }, bin_index]) + .await?; // Add column with bin start let bin_start = (flat_col(bin_index_name).mul(lit(step))).add(lit(start)); @@ -121,11 +123,9 @@ impl TransformTrait for Bin { fn compute_output_value(bin_tx: &Bin, start: f64, stop: f64, step: f64) -> Option { let mut fname = bin_tx.field.clone(); fname.insert_str(0, "bin_"); - - let fields = ScalarValue::List( - Some(vec![ScalarValue::from(bin_tx.field.as_str())]), - Arc::new(Field::new("item", DataType::Utf8, true)), - ); + let fields = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![ScalarValue::from(bin_tx.field.as_str())]).ok()?, + ))); if bin_tx.signal.is_some() { Some(TaskValue::Scalar(ScalarValue::from(vec![ diff --git a/vegafusion-runtime/src/transform/extent.rs b/vegafusion-runtime/src/transform/extent.rs index 338f46926..2458afdf6 100644 --- a/vegafusion-runtime/src/transform/extent.rs +++ b/vegafusion-runtime/src/transform/extent.rs @@ -2,10 +2,10 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; +use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::{max, min, Expr}; use std::sync::Arc; -use vegafusion_common::arrow::datatypes::Field; use vegafusion_common::column::unescaped_col; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datatypes::to_numeric; @@ -61,10 +61,8 @@ fn extract_extent_list(table: &VegaFusionTable) -> Result { let max_val_scalar = ScalarValue::try_from_array(max_val_array, 0).unwrap(); // Build two-element list of the extents - let element_datatype = min_val_scalar.data_type(); - let extent_list = TaskValue::Scalar(ScalarValue::List( - Some(vec![min_val_scalar, max_val_scalar]), - Arc::new(Field::new("item", element_datatype, true)), - )); + let extent_list = TaskValue::Scalar(ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![min_val_scalar, max_val_scalar])?, + )))); Ok(extent_list) } diff --git a/vegafusion-runtime/src/transform/identifier.rs b/vegafusion-runtime/src/transform/identifier.rs index 57640d67a..5facb150d 100644 --- a/vegafusion-runtime/src/transform/identifier.rs +++ b/vegafusion-runtime/src/transform/identifier.rs @@ -41,7 +41,7 @@ impl TransformTrait for Identifier { .alias(&self.r#as); let result = dataframe - .select(vec![Expr::Wildcard, row_number_expr]) + .select(vec![Expr::Wildcard { qualifier: None }, row_number_expr]) .await?; Ok((result, Default::default())) diff --git a/vegafusion-runtime/tests/test_image_comparison.rs b/vegafusion-runtime/tests/test_image_comparison.rs index 2964dec79..e9d9aaece 100644 --- a/vegafusion-runtime/tests/test_image_comparison.rs +++ b/vegafusion-runtime/tests/test_image_comparison.rs @@ -60,91 +60,91 @@ mod test_custom_specs { spec_name, tolerance, extract_inline_values, - case("custom/stacked_bar", 0.001, false), - case("custom/bar_colors", 0.001, false), + // case("custom/stacked_bar", 0.001, false), + // case("custom/bar_colors", 0.001, false), case("custom/imdb_histogram", 0.001, false), - case("custom/flights_crossfilter_a", 0.001, false), - case("custom/flights_crossfilter_csv", 0.001, false), - case("custom/log_scaled_histogram", 0.001, false), - case("custom/non_linear_histogram", 0.001, false), - case("custom/relative_frequency_histogram", 0.001, false), - case("custom/kde_iris", 0.001, false), - case("custom/2d_circles_histogram_imdb", 0.001, false), - case("custom/2d_histogram_imdb", 0.001, false), - case("custom/cumulative_window_imdb", 0.001, false), - case("custom/density_and_cumulative_histograms", 0.001, false), - case("custom/mean_strip_plot_movies", 0.001, false), - case("custom/table_heatmap_cars", 0.001, false), - case("custom/difference_from_mean", 0.001, false), - case("custom/nested_concat_align", 0.001, false), - case("custom/imdb_dashboard_cross_height", 0.001, false), - case("custom/stacked_bar_weather_year", 0.001, false), - case("custom/stacked_bar_weather_month", 0.001, false), - case("custom/stacked_bar_normalize", 0.001, false), - case("custom/layer_bar_labels_grey", 0.001, false), - case("custom/bar_month_temporal_initial", 0.001, false), - case("custom/selection_layer_bar_month", 0.001, false), - case("custom/interactive_layered_crossfilter", 0.001, false), - case("custom/interactive_seattle_weather", 0.001, false), - case("custom/concat_marginal_histograms", 0.001, false), - case("custom/joinaggregate_movie_rating", 0.001, false), - case("custom/joinaggregate_text_color_contrast", 0.001, false), - case("custom/cumulative_running_window", 0.001, false), - case("custom/point_bubble", 0.001, false), - case("custom/circle_bubble_health_income", 0.001, false), - case("custom/line_color_stocks", 0.001, false), - case("custom/line_slope_barley", 0.001, false), - case("custom/connected_scatterplot", 0.001, false), - case("custom/layer_line_co2_concentration", 0.001, false), - case("custom/window_rank_matches", 0.001, false), - case("custom/circle_github_punchcard", 0.001, false), - case("custom/rect_lasagna", 0.001, false), - case("custom/rect_heatmap_weather", 0.001, false), - case("custom/layer_line_rolling_mean_point_raw", 0.001, false), - case("custom/layer_histogram_global_mean", 0.001, false), - case("custom/layer_precipitation_mean", 0.001, false), - case("custom/wheat_wages", 0.001, false), - case("custom/trellis_stacked_bar", 0.001, false), - case("custom/trellis_bar_histogram", 0.001, false), - case("custom/interactive_average", 0.001, false), - case("custom/histogram_responsive", 0.001, false), - case("custom/grouped_bar_chart_with_error_bars", 0.001, false), - case("custom/one_dot_per_zipcode", 0.001, false), - case("custom/ridgeline", 0.001, false), - case("custom/binned_scatter", 0.001, false), - case("custom/seattle_temps_heatmap", 0.001, false), - case("custom/movies_agg_parameterize", 0.001, false), - case("custom/escaped_column_name1", 0.001, false), - case("custom/layered_movies", 0.001, false), - case("custom/shipping_mixed_scales", 0.001, false), - case("custom/datum_color", 0.001, false), - case("custom/bug_153", 0.001, false), - case("custom/period_in_field_name", 0.001, false), - case("custom/period_space_in_field_name", 0.001, false), - case("custom/pivot_tooltip1", 0.001, true), - case("custom/sorted_pivot_lines", 0.001, false), - case("custom/stacked_bar_initial_selection", 0.001, false), - case("custom/time_boolean_bug", 0.001, false), - case("custom/full_bin_histogram_selection", 0.001, false), - case("custom/datetime_scatter", 0.001, false), - case("custom/stack_divide_by_zero_error", 0.001, false), - case("custom/casestudy-us_population_pyramid_over_time", 0.001, true), - case("custom/sin_cos", 0.001, true), - case("custom/area_streamgraph", 0.001, true), - case("custom/pivot_join_on_bug", 0.001, true), - case("custom/special_chars_bar", 0.001, true), - case("custom/bar_sort_x_axis_categorical", 0.001, true), - case("custom/taxi_dashboard", 0.001, true), - case("custom/periods_in_formula_output", 0.001, true), - case("custom/bin_transform_rounding", 0.001, true), - case("custom/geojson_inline", 0.001, true), - case("custom/gh_361", 0.001, true), - case("custom/gh_379", 0.001, true), - case("custom/gh_383", 0.001, true), - case("custom/gh_391", 0.001, true), - case("custom/facet_grouped_bar_with_error_bars", 0.001, true), - case("custom/facet_grouped_bar_with_error_bars_with_sort", 0.001, true), - case("custom/binned_ordinal", 0.001, true) + // case("custom/flights_crossfilter_a", 0.001, false), + // case("custom/flights_crossfilter_csv", 0.001, false), + // case("custom/log_scaled_histogram", 0.001, false), + // case("custom/non_linear_histogram", 0.001, false), + // case("custom/relative_frequency_histogram", 0.001, false), + // case("custom/kde_iris", 0.001, false), + // case("custom/2d_circles_histogram_imdb", 0.001, false), + // case("custom/2d_histogram_imdb", 0.001, false), + // case("custom/cumulative_window_imdb", 0.001, false), + // case("custom/density_and_cumulative_histograms", 0.001, false), + // case("custom/mean_strip_plot_movies", 0.001, false), + // case("custom/table_heatmap_cars", 0.001, false), + // case("custom/difference_from_mean", 0.001, false), + // case("custom/nested_concat_align", 0.001, false), + // case("custom/imdb_dashboard_cross_height", 0.001, false), + // case("custom/stacked_bar_weather_year", 0.001, false), + // case("custom/stacked_bar_weather_month", 0.001, false), + // case("custom/stacked_bar_normalize", 0.001, false), + // case("custom/layer_bar_labels_grey", 0.001, false), + // case("custom/bar_month_temporal_initial", 0.001, false), + // case("custom/selection_layer_bar_month", 0.001, false), + // case("custom/interactive_layered_crossfilter", 0.001, false), + // case("custom/interactive_seattle_weather", 0.001, false), + // case("custom/concat_marginal_histograms", 0.001, false), + // case("custom/joinaggregate_movie_rating", 0.001, false), + // case("custom/joinaggregate_text_color_contrast", 0.001, false), + // case("custom/cumulative_running_window", 0.001, false), + // case("custom/point_bubble", 0.001, false), + // case("custom/circle_bubble_health_income", 0.001, false), + // case("custom/line_color_stocks", 0.001, false), + // case("custom/line_slope_barley", 0.001, false), + // case("custom/connected_scatterplot", 0.001, false), + // case("custom/layer_line_co2_concentration", 0.001, false), + // case("custom/window_rank_matches", 0.001, false), + // case("custom/circle_github_punchcard", 0.001, false), + // case("custom/rect_lasagna", 0.001, false), + // case("custom/rect_heatmap_weather", 0.001, false), + // case("custom/layer_line_rolling_mean_point_raw", 0.001, false), + // case("custom/layer_histogram_global_mean", 0.001, false), + // case("custom/layer_precipitation_mean", 0.001, false), + // case("custom/wheat_wages", 0.001, false), + // case("custom/trellis_stacked_bar", 0.001, false), + // case("custom/trellis_bar_histogram", 0.001, false), + // case("custom/interactive_average", 0.001, false), + // case("custom/histogram_responsive", 0.001, false), + // case("custom/grouped_bar_chart_with_error_bars", 0.001, false), + // case("custom/one_dot_per_zipcode", 0.001, false), + // case("custom/ridgeline", 0.001, false), + // case("custom/binned_scatter", 0.001, false), + // case("custom/seattle_temps_heatmap", 0.001, false), + // case("custom/movies_agg_parameterize", 0.001, false), + // case("custom/escaped_column_name1", 0.001, false), + // case("custom/layered_movies", 0.001, false), + // case("custom/shipping_mixed_scales", 0.001, false), + // case("custom/datum_color", 0.001, false), + // case("custom/bug_153", 0.001, false), + // case("custom/period_in_field_name", 0.001, false), + // case("custom/period_space_in_field_name", 0.001, false), + // case("custom/pivot_tooltip1", 0.001, true), + // case("custom/sorted_pivot_lines", 0.001, false), + // case("custom/stacked_bar_initial_selection", 0.001, false), + // case("custom/time_boolean_bug", 0.001, false), + // case("custom/full_bin_histogram_selection", 0.001, false), + // case("custom/datetime_scatter", 0.001, false), + // case("custom/stack_divide_by_zero_error", 0.001, false), + // case("custom/casestudy-us_population_pyramid_over_time", 0.001, true), + // case("custom/sin_cos", 0.001, true), + // case("custom/area_streamgraph", 0.001, true), + // case("custom/pivot_join_on_bug", 0.001, true), + // case("custom/special_chars_bar", 0.001, true), + // case("custom/bar_sort_x_axis_categorical", 0.001, true), + // case("custom/taxi_dashboard", 0.001, true), + // case("custom/periods_in_formula_output", 0.001, true), + // case("custom/bin_transform_rounding", 0.001, true), + // case("custom/geojson_inline", 0.001, true), + // case("custom/gh_361", 0.001, true), + // case("custom/gh_379", 0.001, true), + // case("custom/gh_383", 0.001, true), + // case("custom/gh_391", 0.001, true), + // case("custom/facet_grouped_bar_with_error_bars", 0.001, true), + // case("custom/facet_grouped_bar_with_error_bars_with_sort", 0.001, true), + // case("custom/binned_ordinal", 0.001, true) )] fn test_image_comparison(spec_name: &str, tolerance: f64, extract_inline_values: bool) { println!("spec_name: {spec_name}"); diff --git a/vegafusion-runtime/tests/test_vegajs_runtime.rs b/vegafusion-runtime/tests/test_vegajs_runtime.rs index 65fb14354..bdbcc85d7 100644 --- a/vegafusion-runtime/tests/test_vegajs_runtime.rs +++ b/vegafusion-runtime/tests/test_vegajs_runtime.rs @@ -9,9 +9,10 @@ use serde_json::json; use std::collections::HashMap; use std::fs; use std::sync::Arc; +use datafusion_common::utils::array_into_list_array; -use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::table::VegaFusionTable; +use vegafusion_core::arrow::array::Float64Array; use vegafusion_core::planning::watch::{ ExportUpdateBatch, ExportUpdateJSON, ExportUpdateNamespace, Watch, WatchNamespace, }; @@ -122,10 +123,7 @@ fn test_evaluate_filter_transform() { // Check extent signal assert_eq!( result_signals, - vec![ScalarValue::List( - Some(vec![ScalarValue::from(6.0), ScalarValue::from(10.0)]), - Arc::new(Field::new("item", DataType::Float64, true)) - )] + vec![ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![6.0, 10.0])))))] ); let expected_dataset = VegaFusionTable::from_json(&json!([ diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 4a24eef46..083441517 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -187,6 +187,7 @@ impl ToSqlExpr for Expr { SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } }; @@ -220,14 +221,17 @@ impl ToSqlExpr for Expr { TryCastMode::Supported => SqlExpr::TryCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::JustUseCast => SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::SafeCast => SqlExpr::SafeCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::SupportedOnStringsOtherwiseJustCast => { if let DataType::Utf8 | DataType::LargeUtf8 = from_dtype { @@ -235,12 +239,14 @@ impl ToSqlExpr for Expr { SqlExpr::TryCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } } else { // Fall back to regular CAST SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } } } @@ -379,6 +385,7 @@ impl ToSqlExpr for Expr { BuiltinScalarFunction::ArrayReplaceN => "array_replace_n", BuiltinScalarFunction::ArrayReplaceAll => "array_replace_all", BuiltinScalarFunction::ArraySlice => "array_slice", + BuiltinScalarFunction::ArrayIntersect => "array_intersect", BuiltinScalarFunction::Decode => "decode", BuiltinScalarFunction::Encode => "encode", BuiltinScalarFunction::Cot => "cot", @@ -387,6 +394,7 @@ impl ToSqlExpr for Expr { BuiltinScalarFunction::Nanvl => "nanvl", BuiltinScalarFunction::Flatten => "flatten", BuiltinScalarFunction::StringToArray => "string_to_array", + BuiltinScalarFunction::ToTimestampNanos => "to_timestamp_nanos", }; translate_scalar_function(fun_name, args, dialect, schema) } @@ -447,6 +455,8 @@ impl ToSqlExpr for Expr { return Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![Ident::new(alt_fun)]), args: vec![], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -533,6 +543,8 @@ impl ToSqlExpr for Expr { quote_style: None, }]), args, + filter: None, + null_treatment: None, over: Some(over), distinct: false, special: false, @@ -582,7 +594,7 @@ impl ToSqlExpr for Expr { negated: *negated, }) } - Expr::Wildcard => Err(VegaFusionError::internal( + Expr::Wildcard { .. } => Err(VegaFusionError::internal( "Wildcard cannot be converted to SQL", )), Expr::Exists { .. } => Err(VegaFusionError::internal( @@ -594,9 +606,6 @@ impl ToSqlExpr for Expr { Expr::ScalarSubquery(_) => Err(VegaFusionError::internal( "ScalarSubquery cannot be converted to SQL", )), - Expr::QualifiedWildcard { .. } => Err(VegaFusionError::internal( - "QualifiedWildcard cannot be converted to SQL", - )), Expr::GroupingSet(_) => Err(VegaFusionError::internal( "GroupingSet cannot be converted to SQL", )), @@ -631,6 +640,8 @@ fn translate_scalar_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -664,6 +675,8 @@ fn translate_aggregate_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args, + filter: None, + null_treatment: None, over: None, distinct, special: false, diff --git a/vegafusion-sql/src/compile/function_arg.rs b/vegafusion-sql/src/compile/function_arg.rs index 99b3a2f77..e257f8666 100644 --- a/vegafusion-sql/src/compile/function_arg.rs +++ b/vegafusion-sql/src/compile/function_arg.rs @@ -20,13 +20,13 @@ impl ToSqlFunctionArg for Expr { schema: &DFSchema, ) -> Result { Ok(match self { - Expr::Wildcard => SqlFunctionArgExpr::Wildcard, - Expr::QualifiedWildcard { qualifier } => { - SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { - value: qualifier.clone(), - quote_style: None, - }])) - } + Expr::Wildcard { qualifier: None } => SqlFunctionArgExpr::Wildcard, + Expr::Wildcard { + qualifier: Some(qualifier), + } => SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { + value: qualifier.clone(), + quote_style: None, + }])), expr => SqlFunctionArgExpr::Expr(expr.to_sql(dialect, schema)?), }) } diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index eefe47d1d..4e459a949 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -1,6 +1,7 @@ use crate::compile::data_type::ToSqlDataType; use crate::compile::expr::ToSqlExpr; use crate::dialect::Dialect; +use arrow::array::Array; use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; @@ -47,6 +48,7 @@ impl ToSqlScalar for ScalarValue { false, ))), data_type: cast_dtype, + format: None, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -79,6 +81,7 @@ impl ToSqlScalar for ScalarValue { false, ))), data_type: cast_dtype, + format: None, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -142,26 +145,24 @@ impl ToSqlScalar for ScalarValue { ScalarValue::FixedSizeBinary(_, _) => Err(VegaFusionError::internal( "FixedSizeBinary cannot be converted to SQL", )), - ScalarValue::List(args, _) => { + ScalarValue::List(array) => { let function_ident = Ident { value: "make_list".to_string(), quote_style: None, }; - let args = args - .clone() - .unwrap_or_default() - .iter() - .map(|expr| { - Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr( - expr.to_sql(dialect)?, - ))) + let args = (0..array.len()) + .map(|i| { + let sql_expr = ScalarValue::try_from_array(array, i)?.to_sql(dialect)?; + Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr(sql_expr))) }) .collect::>>()?; Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![function_ident]), args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -280,6 +281,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { None => Ok(SqlExpr::Cast { expr: Box::new(ScalarValue::Utf8(None).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, + format: None, }), Some(days) => { let date = epoch.add(chrono::Duration::days(*days as i64)); @@ -287,6 +289,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { Ok(SqlExpr::Cast { expr: Box::new(ScalarValue::from(date_str.as_str()).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, + format: None, }) } } diff --git a/vegafusion-sql/src/compile/select.rs b/vegafusion-sql/src/compile/select.rs index b55ff858e..099e42858 100644 --- a/vegafusion-sql/src/compile/select.rs +++ b/vegafusion-sql/src/compile/select.rs @@ -2,7 +2,7 @@ use crate::compile::expr::ToSqlExpr; use crate::dialect::Dialect; use datafusion_common::DFSchema; use datafusion_expr::{expr, Expr}; -use sqlparser::ast::{Ident, SelectItem as SqlSelectItem}; +use sqlparser::ast::{Ident, ObjectName, SelectItem as SqlSelectItem}; use vegafusion_common::error::Result; pub trait ToSqlSelectItem { @@ -19,7 +19,16 @@ impl ToSqlSelectItem for Expr { quote_style: Some(dialect.quote_style), }, }, - Expr::Wildcard => SqlSelectItem::Wildcard(Default::default()), + Expr::Wildcard { qualifier: None } => SqlSelectItem::Wildcard(Default::default()), + Expr::Wildcard { + qualifier: Some(qualifier), + } => SqlSelectItem::QualifiedWildcard( + ObjectName(vec![Ident { + value: qualifier.to_string(), + quote_style: Some(dialect.quote_style), + }]), + Default::default(), + ), expr => SqlSelectItem::UnnamedExpr(expr.to_sql(dialect, schema)?), }) } diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index b1cd94b73..12e66ed8c 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -197,14 +197,12 @@ impl Connection for DataFusionConnection { writeln!(file, "{body}").unwrap(); } - let path = tempdir.path().to_str().unwrap(); - // Build final csv schema that combines the requested and inferred schemas - let final_schema = build_csv_schema(&df_csv_opts, path, &self.ctx).await?; + let final_schema = build_csv_schema(&df_csv_opts, &filepath, &self.ctx).await?; df_csv_opts = df_csv_opts.schema(&final_schema); // Load through VegaFusionTable so that temp file can be deleted - let df = self.ctx.read_csv(path, df_csv_opts).await?; + let df = self.ctx.read_csv(&filepath, df_csv_opts).await?; let schema: SchemaRef = Arc::new(df.schema().into()) as SchemaRef; let batches = df.collect().await?; diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index 5ee92268d..fa7386241 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -13,8 +13,8 @@ use datafusion_expr::{ WindowFrameBound, WindowFrameUnits, WindowFunction, }; use sqlparser::ast::{ - Cte, Expr as SqlExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, TableAlias, - TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, + Cte, Expr as SqlExpr, GroupByExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, + TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, }; use sqlparser::parser::Parser; use std::any::Any; @@ -278,7 +278,7 @@ impl SqlDataFrame { selection: None, from: Default::default(), lateral_views: Default::default(), - group_by: Default::default(), + group_by: GroupByExpr::Expressions(Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), @@ -320,6 +320,7 @@ impl SqlDataFrame { body: Box::new(values_body), order_by: Default::default(), limit: None, + limit_by: vec![], offset: None, fetch: None, locks: Default::default(), @@ -385,7 +386,7 @@ impl SqlDataFrame { }], lateral_views: Default::default(), selection: None, - group_by: Default::default(), + group_by: GroupByExpr::Expressions(Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), @@ -398,6 +399,7 @@ impl SqlDataFrame { body: Box::new(select_body), order_by: Default::default(), limit: None, + limit_by: vec![], offset: None, fetch: None, locks: Default::default(), @@ -923,7 +925,7 @@ impl SqlDataFrame { // then union the results. This is required to make sure stacks do not overlap. Negative // values stack in the negative direction and positive values stack in the positive // direction. - let schema_exprs = vec![Expr::Wildcard, window_expr]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, window_expr]; let new_schema = make_new_schema_from_exprs( self.schema.as_ref(), schema_exprs.as_slice(), @@ -967,7 +969,10 @@ impl SqlDataFrame { // Create __stack column with numeric field let stack_col_name = "__stack"; let dataframe = self - .select(vec![Expr::Wildcard, numeric_field.alias(stack_col_name)]) + .select(vec![ + Expr::Wildcard { qualifier: None }, + numeric_field.alias(stack_col_name), + ]) .await?; let dataframe = dataframe @@ -990,7 +995,7 @@ impl SqlDataFrame { .to_string(); // Add __total column with total or total per partition - let schema_exprs = vec![Expr::Wildcard, total_agg]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, total_agg]; let new_schema = make_new_schema_from_exprs( &dataframe.schema(), schema_exprs.as_slice(), @@ -1043,7 +1048,9 @@ impl SqlDataFrame { .alias(cumulative_field); // Perform selection to add new field value - let dataframe = dataframe.select(vec![Expr::Wildcard, window_expr]).await?; + let dataframe = dataframe + .select(vec![Expr::Wildcard { qualifier: None }, window_expr]) + .await?; // Build final_selection let mut final_selection: Vec<_> = input_fields @@ -1066,7 +1073,7 @@ impl SqlDataFrame { .to_string(); // Compute new schema - let schema_exprs = vec![Expr::Wildcard, max_total]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, max_total]; let new_schema = make_new_schema_from_exprs( &dataframe.schema(), schema_exprs.as_slice(), @@ -1395,7 +1402,7 @@ fn make_new_schema_from_exprs( ) -> Result { let mut fields: Vec = Vec::new(); for expr in exprs { - if let Expr::Wildcard = expr { + if let Expr::Wildcard { .. } = expr { // Add field for each input schema field fields.extend(schema.fields().iter().map(|f| f.as_ref().clone())); } else { diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index c42ee0514..06026fd4d 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -516,7 +516,7 @@ impl Dialect { (DataType::Float16, float64dtype.clone()), (DataType::Float32, float64dtype.clone()), (DataType::Float64, float64dtype), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -796,7 +796,7 @@ impl Dialect { (DataType::Float16, SqlDataType::Float(None)), (DataType::Float32, SqlDataType::Float(None)), (DataType::Float64, SqlDataType::Double), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -962,7 +962,7 @@ impl Dialect { (DataType::Float16, SqlDataType::Float(None)), (DataType::Float32, SqlDataType::Float(None)), (DataType::Float64, SqlDataType::Double), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -1750,6 +1750,8 @@ impl BinaryOperatorTransformer for ModulusOpToFunction { quote_style: None, }]), args: vec![arg0, arg1], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1793,6 +1795,8 @@ impl FunctionTransformer for RenameFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1825,6 +1829,8 @@ impl FunctionTransformer for ExpWithPowFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1854,6 +1860,7 @@ impl FunctionTransformer for CastArgsFunctionTransformer { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Cast { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: self.cast_dtype.clone(), + format: None, }))) }) .collect::>>()?; @@ -1864,6 +1871,8 @@ impl FunctionTransformer for CastArgsFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1902,6 +1911,8 @@ impl FunctionTransformer for LogBaseTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1929,6 +1940,7 @@ impl FunctionTransformer for LogBaseWithLnTransformer { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Cast { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: cast_dtype.clone(), + format: None, }))) }) .collect::>>()? @@ -1942,6 +1954,8 @@ impl FunctionTransformer for LogBaseWithLnTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1957,6 +1971,8 @@ impl FunctionTransformer for LogBaseWithLnTransformer { quote_style: None, }]), args: vec![base_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index 15f085719..245a29eb5 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -67,6 +67,8 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { SqlValue::SingleQuotedString(time_zone.to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -90,6 +92,8 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { SqlValue::SingleQuotedString(time_zone.to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -126,6 +130,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -150,6 +156,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { }, ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -167,6 +175,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -249,6 +259,8 @@ impl FunctionTransformer for DateAddTzDatabricksTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -339,6 +351,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -361,6 +375,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -384,6 +400,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index 810a57729..f6444205c 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -111,6 +111,8 @@ impl FunctionTransformer for DatePartTzWithDatePartAndAtTimezoneTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -172,6 +174,8 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -202,6 +206,8 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( to_timezone_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -241,6 +247,8 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -259,6 +267,8 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -297,6 +307,8 @@ impl FunctionTransformer for DatePartTzMySqlTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -342,6 +354,8 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -360,6 +374,8 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs index 098d4f330..c9df3acb1 100644 --- a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs @@ -52,6 +52,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastAndAtTimeZoneTransformer let timestamp_arg = SqlExpr::Cast { expr: Box::new(date_arg), data_type: SqlDataType::Timestamp(None, SqlTimezoneInfo::None), + format: None, }; let utc_timestamp = if time_zone == "UTC" { @@ -97,6 +98,7 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { }]), Vec::new(), ), + format: None, }; if time_zone == "UTC" { @@ -117,6 +119,8 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -158,6 +162,8 @@ impl FunctionTransformer for DateToUtcTimestampWithFunctionTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -198,6 +204,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(date_arg), data_type: self.timestamp_type.clone(), + format: None, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -210,6 +217,8 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -252,6 +261,8 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -269,6 +280,8 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -302,6 +315,8 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -326,6 +341,8 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs index bfeae3625..d03f0c948 100644 --- a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs @@ -74,6 +74,8 @@ impl FunctionTransformer for DateTruncTzWithDateTruncAndAtTimezoneTransformer { quote_style: None, }]), args: vec![part_func_arg, ts_func_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -130,6 +132,8 @@ impl FunctionTransformer for DateTruncTzWithTimestampTruncTransformer { quote_style: None, }]), args: vec![ts_func_arg, part_func_arg, tz_func_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -179,6 +183,8 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -199,6 +205,8 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -241,6 +249,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -259,6 +269,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -279,6 +291,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -320,6 +334,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -338,6 +354,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -361,6 +379,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs index a32b04905..322b8843b 100644 --- a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs @@ -44,6 +44,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampBigQueryTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -88,6 +90,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( div_1000_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -101,6 +105,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( floor_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -119,6 +125,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(mod_1000_expr)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(from_unix_time_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -150,6 +158,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDuckDbTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -194,6 +204,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( div_1000_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -207,6 +219,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( floor_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -270,6 +284,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampSnowflakeTransformer { SqlValue::Number("3".to_string(), false), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs index 2b7203e9f..042463fc4 100644 --- a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs @@ -104,6 +104,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -120,6 +122,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -143,6 +147,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { }, ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -158,6 +164,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_expr)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_with_ms_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -175,6 +183,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -215,6 +225,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -234,6 +246,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.millisecond)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -254,6 +268,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -295,6 +311,8 @@ impl FunctionTransformer for MakeUtcTimestampDuckDbTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -348,6 +366,8 @@ impl FunctionTransformer for MakeUtcTimestampPostgresTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -402,6 +422,8 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { ))), })), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -425,6 +447,8 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs index 6f888a0fe..a14e2e087 100644 --- a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs @@ -54,6 +54,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastAndAtTimeZoneTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), + format: None, }; let utc_expr = if time_zone == "UTC" { cast_expr @@ -103,6 +104,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), + format: None, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -115,6 +117,8 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -161,6 +165,8 @@ impl FunctionTransformer for StrToUtcTimestampWithFunctionTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -195,6 +201,8 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -212,6 +220,8 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -245,6 +255,8 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -269,6 +281,8 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -306,6 +320,7 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { }]), Vec::new(), ), + format: None, }; if time_zone == "UTC" { @@ -326,6 +341,8 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs index dc573974f..05b5e23ab 100644 --- a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs @@ -117,6 +117,8 @@ impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -155,6 +157,8 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -172,6 +176,8 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs index 59c5ad598..f125d11ee 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs @@ -46,6 +46,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( ts_expr.clone(), ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -68,6 +70,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -116,6 +120,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( ts_expr.clone(), ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -138,6 +144,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -189,6 +197,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsPostgresTransform { right: Box::new(SqlExpr::Value(SqlValue::Number("1000".to_string(), false))), }, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -226,6 +236,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsSnowflakeTransform { }))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs index 3705124fe..79b2c29cb 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs @@ -58,6 +58,8 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -75,6 +77,8 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -114,6 +118,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -132,6 +138,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString("y-MM-dd HH:mm:ss.SSS".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -154,6 +162,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString("T".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -204,6 +214,8 @@ impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%S.%g".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -254,6 +266,8 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.MS".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -298,6 +312,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -316,6 +332,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.FF3".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, From c6d294fb555684359170e307a15b95c3bde68285 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 10 Feb 2024 17:23:58 -0500 Subject: [PATCH 02/44] Fix result --- vegafusion-common/src/data/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index a1cbe43fa..1f6b3fd8f 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -234,7 +234,7 @@ impl VegaFusionTable { "__dummy", ScalarValue::try_from(&DataType::Float64).unwrap(), )]); - let array = empty_scalar.to_array_of_size(values.len()); + let array = empty_scalar.to_array_of_size(values.len())?; let struct_array = array.as_any().downcast_ref::().unwrap(); let record_batch = RecordBatch::from(struct_array); Self::try_new(record_batch.schema(), vec![record_batch]) From d49de644dce54bf487f488e6a0380f9f5753a277 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 10 Feb 2024 18:22:45 -0500 Subject: [PATCH 03/44] Update to datafusion 34 --- Cargo.lock | 198 ++++++++++------- Cargo.toml | 16 +- vegafusion-common/src/datatypes.rs | 4 +- vegafusion-runtime/src/data/tasks.rs | 26 +-- .../src/expression/compiler/array.rs | 6 +- .../data/vl_selection_test.rs | 14 +- .../date_time/date_format.rs | 30 +-- .../builtin_functions/date_time/date_parts.rs | 18 +- .../builtin_functions/date_time/datetime.rs | 22 +- .../builtin_functions/date_time/time.rs | 14 +- .../date_time/time_offset.rs | 6 +- .../builtin_functions/math/isfinite.rs | 6 +- .../src/expression/compiler/call.rs | 8 +- .../src/expression/compiler/member.rs | 16 +- .../src/expression/compiler/object.rs | 6 +- vegafusion-runtime/src/transform/aggregate.rs | 21 +- vegafusion-runtime/src/transform/timeunit.rs | 58 ++--- vegafusion-sql/src/compile/expr.rs | 200 ++---------------- vegafusion-sql/src/compile/scalar.rs | 18 +- vegafusion-sql/src/compile/select.rs | 2 +- vegafusion-sql/src/dataframe/mod.rs | 17 +- 21 files changed, 290 insertions(+), 416 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d939bff4e..cfa9f8830 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,9 +141,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378" +checksum = "5bc25126d18a012146a888a0298f2c22e1150327bd2765fc76d710a556b2d614" dependencies = [ "ahash", "arrow-arith", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef983914f477d4278b068f13b3224b7d19eb2b807ac9048544d3bfebdf2554c4" +checksum = "34ccd45e217ffa6e53bbb0080990e77113bdd4e91ddb84e97b77649810bcf1a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b" +checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d" dependencies = [ "ahash", "arrow-buffer", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55512d988c6fbd76e514fd3ff537ac50b0a675da5a245e4fdad77ecfd654205f" +checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c" dependencies = [ "bytes", "half 2.2.1", @@ -207,15 +207,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655ee51a2156ba5375931ce21c1b2494b1d9260e6dcdc6d4db9060c37dc3325b" +checksum = "5dc0368ed618d509636c1e3cc20db1281148190a78f43519487b2daf07b63b4a" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "base64", "chrono", "comfy-table", "half 2.2.1", @@ -225,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bb689997ad5b6660b3ce3638bd6b383d668ec555ed41ad7c6559cbb2e4f91" +checksum = "2e09aa6246a1d6459b3f14baeaa49606cfdbca34435c46320e14054d244987ca" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc2b9fec74763427e2e5575b8cc31ce96ba4c9b4eb05ce40e0616d9fad12461" +checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634" dependencies = [ "arrow-buffer", "arrow-schema", @@ -256,9 +257,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eaa6ab203cc6d89b7eaa1ac781c1dfeef325454c5d5a0419017f95e6bafc03c" +checksum = "79a43d6808411886b8c7d4f6f7dd477029c1e77ffffffb7923555cc6579639cd" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,9 +271,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb64e30d9b73f66fdc5c52d5f4cf69bbf03d62f64ffeafa0715590a5320baed7" +checksum = "d82565c91fd627922ebfe2810ee4e8346841b6f9361b87505a9acea38b614fee" dependencies = [ "arrow-array", "arrow-buffer", @@ -290,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a818951c0d11c428dda03e908175969c262629dd20bd0850bd6c7a8c3bfe48" +checksum = "9b23b0e53c0db57c6749997fd343d4c0354c994be7eca67152dd2bdb9a3e1bb4" dependencies = [ "arrow-array", "arrow-buffer", @@ -305,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3" +checksum = "361249898d2d6d4a6eeb7484be6ac74977e48da12a4dd81a708d620cc558117a" dependencies = [ "ahash", "arrow-array", @@ -320,18 +321,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf4d737bba93da59f16129bec21e087aed0be84ff840e74146d4703879436cb" +checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167" dependencies = [ "bitflags 2.1.0", ] [[package]] name = "arrow-select" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a" +checksum = "4f6208466590960efc1d2a7172bc4ff18a67d6e25c529381d7f96ddaf0dc4036" dependencies = [ "ahash", "arrow-array", @@ -343,9 +344,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15aed5624bb23da09142f58502b59c23f5bea607393298bb81dab1ce60fc769" +checksum = "a4a48149c63c11c9ff571e50ab8f017d2a7cb71037a882b42f6354ed2da9acc7" dependencies = [ "arrow-array", "arrow-buffer", @@ -1238,9 +1239,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "676796427e638d85e9eadf13765705212be60b34f8fc5d3934d95184c63ca1b4" +checksum = "193fd1e7628278d0641c5122860f9a7fd6a1d77d055838d12f55d15bbe28d4d0" dependencies = [ "ahash", "arrow", @@ -1265,7 +1266,7 @@ dependencies = [ "half 2.2.1", "hashbrown 0.14.0", "indexmap 2.0.0", - "itertools 0.11.0", + "itertools 0.12.1", "log", "num_cpus", "object_store", @@ -1273,7 +1274,7 @@ dependencies = [ "parquet", "pin-project-lite", "rand", - "sqlparser", + "sqlparser 0.40.0", "tempfile", "tokio", "tokio-util", @@ -1285,9 +1286,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e23b3d21a6531259d291bd20ce59282ea794bda1018b0a1e278c13cd52e50c" +checksum = "548bc49c4a489e3de474813831ea556dc9d368f9ed8d867b1493da42e8e9f613" dependencies = [ "ahash", "arrow", @@ -1296,18 +1297,19 @@ dependencies = [ "arrow-schema", "chrono", "half 2.2.1", + "libc", "num_cpus", "object_store", "parquet", "pyo3", - "sqlparser", + "sqlparser 0.40.0", ] [[package]] name = "datafusion-execution" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de1fd0d8db0f2b8e4f4121bfa1c7c09d3a5c08a0a65c2229cd849eb65cff855" +checksum = "ecc865657ffcf4da5ff08bdc6436a9a833bc0aa96c3254c8d18ab8a0ad4e437d" dependencies = [ "arrow", "chrono", @@ -1326,24 +1328,25 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e227fe88bf6730cab378d0cd8fc4c6b2ea42bc7e414a8ea9feba7225932735" +checksum = "33c473f72d8d81a532e63f6e562ed66dd9209dfd8e433d9712abd42444ee161e" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", - "sqlparser", + "paste", + "sqlparser 0.40.0", "strum 0.25.0", "strum_macros 0.25.1", ] [[package]] name = "datafusion-optimizer" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6648e62ea7605b9bfcd87fdc9d67e579c3b9ac563a87734ae5fe6d79ee4547" +checksum = "cb6218318001d2f6783b7fffa17592318f65f26609d7aab605a3dd0c7c2e2618" dependencies = [ "arrow", "async-trait", @@ -1352,16 +1355,16 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.0", - "itertools 0.11.0", + "itertools 0.12.1", "log", "regex-syntax 0.8.2", ] [[package]] name = "datafusion-physical-expr" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f32b8574add16a32411a9b3fb3844ac1fc09ab4e7be289f86fd56d620e4f2508" +checksum = "9e1ca7e35ca22f9dc506c2375b92054b03ccf91afe25c0a90b395a1473a09735" dependencies = [ "ahash", "arrow", @@ -1379,8 +1382,7 @@ dependencies = [ "hashbrown 0.14.0", "hex", "indexmap 2.0.0", - "itertools 0.11.0", - "libc", + "itertools 0.12.1", "log", "md-5", "paste", @@ -1394,9 +1396,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796abd77d5bfecd9e5275a99daf0ec45f5b3a793ec431349ce8211a67826fd22" +checksum = "ddde97adefcca3a55257c646ffee2a95b6cac66f74d1146a6e3a6dbb37830631" dependencies = [ "ahash", "arrow", @@ -1413,7 +1415,7 @@ dependencies = [ "half 2.2.1", "hashbrown 0.14.0", "indexmap 2.0.0", - "itertools 0.11.0", + "itertools 0.12.1", "log", "once_cell", "parking_lot 0.12.1", @@ -1425,9 +1427,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26de2592417beb20f73f29b131a04d7de14e2a6336c631554d611584b4306236" +checksum = "344700ad4505ffcab6ba0715188ce317ad44e024fe1c5dc7267762113a9dcdf8" dependencies = [ "arrow", "chrono", @@ -1440,16 +1442,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced70b8a5648ba7b95c61fc512183c33287ffe2c9f22ffe22700619d7d48c84f" +checksum = "a60d9d6460a64fddb8663db41da97e6b8b0bf79da42f997ebe81722731eaf0e5" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser", + "sqlparser 0.40.0", ] [[package]] @@ -2159,6 +2161,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -2626,9 +2637,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" +checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" dependencies = [ "async-trait", "base64", @@ -2643,7 +2654,7 @@ dependencies = [ "quick-xml", "rand", "reqwest", - "ring", + "ring 0.17.3", "serde", "serde_json", "snafu", @@ -2745,9 +2756,9 @@ dependencies = [ [[package]] name = "parquet" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597" +checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4" dependencies = [ "ahash", "arrow-array", @@ -3156,9 +3167,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", "serde", @@ -3409,12 +3420,26 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babe80d5c16becf6594aa32ad2be8fe08498e7ae60b77de8df700e67f191d7e" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rstest" version = "0.18.2" @@ -3492,7 +3517,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", - "ring", + "ring 0.16.20", "rustls-webpki", "sct", ] @@ -3512,8 +3537,8 @@ version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -3561,8 +3586,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -3716,11 +3741,26 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sqlparser" version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" +dependencies = [ + "log", +] + +[[package]] +name = "sqlparser" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c80afe31cdb649e56c0d9bb5503be9166600d68a852c38dd445636d126858e5" dependencies = [ "log", "sqlparser_derive", @@ -3728,13 +3768,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.1.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] @@ -4332,6 +4372,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.3.1" @@ -4378,7 +4424,7 @@ dependencies = [ "object_store", "pyo3", "serde_json", - "sqlparser", + "sqlparser 0.39.0", "thiserror", ] @@ -4405,7 +4451,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sqlparser", + "sqlparser 0.39.0", "thiserror", "tonic", "tonic-build", @@ -4514,7 +4560,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "sqlparser", + "sqlparser 0.39.0", "tempfile", "test-case", "tokio", @@ -4574,7 +4620,7 @@ dependencies = [ "rstest_reuse", "serde", "serde_json", - "sqlparser", + "sqlparser 0.39.0", "tempfile", "tokio", "toml", diff --git a/Cargo.toml b/Cargo.toml index 1795da463..f483b84ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ ] [workspace.dependencies] -arrow = { version = "48.0.1", default_features = false } +arrow = { version = "49.0.0", default_features = false } sqlparser = { version = "0.39.0" } chrono = { version = "0.4.31", default_features = false } reqwest = { version = "0.11.22", default-features = false } @@ -23,14 +23,14 @@ pyo3 = { version = "0.20.0" } pythonize = { version = "0.20.0" } prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } -object_store = { version= "0.7.1" } +object_store = { version= "0.8.0" } -datafusion = { version = "33.0.0" } -datafusion-common = { version = "33.0.0", default_features = false} -datafusion-expr = { version = "33.0.0" } -datafusion-proto = { version = "33.0.0" } -datafusion-physical-expr = { version = "33.0.0" } -datafusion-optimizer = { version = "33.0.0" } +datafusion = { version = "34.0.0" } +datafusion-common = { version = "34.0.0", default_features = false} +datafusion-expr = { version = "34.0.0" } +datafusion-proto = { version = "34.0.0" } +datafusion-physical-expr = { version = "34.0.0" } +datafusion-optimizer = { version = "34.0.0" } [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/datatypes.rs b/vegafusion-common/src/datatypes.rs index debeb7846..70f880a0d 100644 --- a/vegafusion-common/src/datatypes.rs +++ b/vegafusion-common/src/datatypes.rs @@ -1,7 +1,7 @@ use crate::error::{Result, ResultWithContext}; use arrow::datatypes::DataType; use datafusion_common::DFSchema; -use datafusion_expr::{coalesce, expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, TryCast}; +use datafusion_expr::{coalesce, expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, TryCast, ScalarFunctionDefinition}; pub fn is_numeric_datatype(dtype: &DataType) -> bool { matches!( @@ -87,7 +87,7 @@ pub fn to_numeric(value: Expr, schema: &DFSchema) -> Result { // Convert to milliseconds Expr::TryCast(TryCast { expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::ToTimestampMillis, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ToTimestampMillis), args: vec![value], })), data_type: DataType::Int64, diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index 36b7ce263..fbc63bc67 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -5,7 +5,7 @@ use crate::task_graph::task::TaskCall; use async_trait::async_trait; -use datafusion_expr::{expr, lit, Expr}; +use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::collections::{HashMap, HashSet}; use std::path::Path; @@ -336,16 +336,16 @@ async fn process_datetimes( .map(|tz_config| tz_config.default_input_tz.to_string()) .unwrap_or_else(|| "UTC".to_string()); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![flat_col(&spec.name), lit(default_input_tz_str)], }) } else if is_integer_datatype(dtype) { // Assume Year was parsed numerically, use local time let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: vec![ flat_col(&spec.name), // year lit(0), // month @@ -395,8 +395,8 @@ async fn process_datetimes( DataType::Timestamp(_, tz) => match tz { Some(tz) => { // Timestamp has explicit timezone - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*TO_UTC_TIMESTAMP_UDF).clone())), args: vec![flat_col(field.name()), lit(tz.as_ref())], }) } @@ -405,8 +405,8 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*TO_UTC_TIMESTAMP_UDF).clone())), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -418,8 +418,8 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*TO_UTC_TIMESTAMP_UDF).clone())), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -430,8 +430,8 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*DATE_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![flat_col(field.name()), lit(tz_config.local_tz.to_string())], }) } diff --git a/vegafusion-runtime/src/expression/compiler/array.rs b/vegafusion-runtime/src/expression/compiler/array.rs index 9b95869f2..6aa527325 100644 --- a/vegafusion-runtime/src/expression/compiler/array.rs +++ b/vegafusion-runtime/src/expression/compiler/array.rs @@ -1,5 +1,5 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{expr, Expr}; +use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::ops::Deref; use std::sync::Arc; use vegafusion_common::datafusion_common::DFSchema; @@ -17,8 +17,8 @@ pub fn compile_array( let phys_expr = compile(el, config, Some(schema))?; elements.push(phys_expr); } - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: elements, })) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index 0bbf318f3..8011ad66d 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -5,7 +5,7 @@ use std::convert::TryFrom; use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_expr::expr::Case; -use datafusion_expr::{ceil, expr, lit, Between, Expr, ExprSchemable}; +use datafusion_expr::{ceil, expr, lit, Between, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; @@ -131,8 +131,8 @@ impl FieldSpec { field_col.get_type(schema)?, DataType::Timestamp(TimeUnit::Millisecond, _) ) { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), args: vec![field_col], }) } else { @@ -270,12 +270,12 @@ impl FieldSpec { if parse_datetime(&s, &Some(chrono_tz::UTC)).is_some() && is_numeric_datatype(field_type) => { - let timestamp_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + let timestamp_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![lit(s), lit(default_input_tz)], }); - let ms_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), + let ms_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), args: vec![timestamp_expr], }); cast_to(ms_expr, field_type, schema) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs index 93fd3c521..ab9ab2d22 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; @@ -41,23 +41,23 @@ pub fn time_format_fn( // Special case for ISO-8601 format with milliseconds. The UTC_TIMESTAMP_TO_STR_UDF // is compatible with more SQL dialects, so we want to use it if possible let udf_args = vec![timestamptz_expr, lit(&format_tz_str)]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone())), args: udf_args, })) } else { // General case if format_tz_str.to_ascii_lowercase() != "utc" { - timestamptz_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*FROM_UTC_TIMESTAMP_UDF).clone()), + timestamptz_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*FROM_UTC_TIMESTAMP_UDF).clone())), args: vec![timestamptz_expr, lit(format_tz_str)], }) } let udf_args = vec![timestamptz_expr, lit(format_str)]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*FORMAT_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*FORMAT_TIMESTAMP_UDF).clone())), args: udf_args, })) } @@ -76,15 +76,15 @@ pub fn utc_format_fn( // Special case for ISO-8601 format with milliseconds. The UTC_TIMESTAMP_TO_STR_UDF // is compatible with more SQL dialects, so we want to use it if possible let udf_args = vec![timestamptz_expr, lit("UTC")]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone())), args: udf_args, })) } else { // General case let udf_args = vec![timestamptz_expr, lit(format_str)]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*FORMAT_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*FORMAT_TIMESTAMP_UDF).clone())), args: udf_args, })) } @@ -101,14 +101,14 @@ fn to_timestamptz_expr(arg: &Expr, schema: &DFSchema, default_input_tz: &str) -> data_type: DataType::Timestamp(TimeUnit::Millisecond, None), }), DataType::Timestamp(_, _) => arg.clone(), - DataType::Utf8 => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg.clone(), lit(default_input_tz)], }), DataType::Null => arg.clone(), dtype if is_numeric_datatype(&dtype) || matches!(dtype, DataType::Boolean) => { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs index c5de56276..eef1cbc4f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::call::TzTransformFn; use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::datafusion_common::DFSchema; @@ -19,8 +19,8 @@ pub fn make_local_datepart_transform(part: &str, tx: Option Expr>) - let arg = extract_timestamp_arg(&part, args, schema, &tz_config.default_input_tz.to_string())?; let udf_args = vec![lit(part.clone()), arg, lit(tz_config.local_tz.to_string())]; - let mut expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let mut expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: udf_args, }); @@ -42,8 +42,8 @@ pub fn make_utc_datepart_transform(part: &str, tx: Option Expr>) -> let arg = extract_timestamp_arg(&part, args, schema, &tz_config.default_input_tz.to_string())?; let udf_args = vec![lit(part.clone()), arg, lit("UTC")]; - let mut expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let mut expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: udf_args, }); @@ -73,12 +73,12 @@ fn extract_timestamp_arg( data_type: DataType::Timestamp(TimeUnit::Millisecond, None), }), DataType::Timestamp(_, _) => arg.clone(), - DataType::Utf8 => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg.clone(), lit(default_input_tz)], }), - dtype if is_numeric_datatype(&dtype) => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }), dtype => { diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs index 364f9e3fc..a20972b54 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::ops::Deref; use std::str::FromStr; use std::sync::Arc; @@ -43,13 +43,13 @@ pub fn to_date_transform( tz_config.default_input_tz }; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg, lit(default_input_tz.to_string())], })) } else if is_numeric_datatype(&dtype) { - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![cast_to(arg, &DataType::Int64, schema)?], })) } else { @@ -71,8 +71,8 @@ pub fn datetime_transform_fn( if is_string_datatype(&dtype) { let default_input_tz_str = tz_config.default_input_tz.to_string(); - arg = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + arg = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg, lit(default_input_tz_str)], }) } @@ -81,8 +81,8 @@ pub fn datetime_transform_fn( } else { let udf_args = extract_datetime_component_args(args, &tz_config.default_input_tz.to_string(), schema)?; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: udf_args, })) } @@ -95,8 +95,8 @@ pub fn make_datetime_components_fn( ) -> Result { let udf_args = extract_datetime_component_args(args, &tz_config.default_input_tz.to_string(), schema)?; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.deref().clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.deref().clone())), args: udf_args, })) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs index 42f4442e2..febb81d90 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -23,18 +23,18 @@ pub fn time_fn(tz_config: &RuntimeTzConfig, args: &[Expr], schema: &DFSchema) -> // Dispatch handling on data type let expr = match arg.get_type(schema)? { DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64 => { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), args: vec![arg.clone()], }) } DataType::Utf8 => { let mut udf_args = vec![lit(tz_config.default_input_tz.to_string())]; udf_args.extend(Vec::from(args)); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), - args: vec![Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), + args: vec![Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg.clone(), lit(tz_config.default_input_tz.to_string())], })], }) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs index 15535feb0..4d8d3fc8c 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs @@ -1,6 +1,6 @@ use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::{expr, lit, Expr}; +use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::error::VegaFusionError; @@ -79,8 +79,8 @@ pub fn time_offset_fn( let mut udf_args = vec![lit(tz_config.local_tz.to_string())]; udf_args.extend(Vec::from(args)); - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*DATE_ADD_TZ_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_ADD_TZ_UDF).clone())), args: vec![ lit(unit), step, diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs index 3ded9fa88..5c9f763ad 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::ops::Deref; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; @@ -21,8 +21,8 @@ pub fn is_finite_fn(args: &[Expr], schema: &DFSchema) -> Result { Ok(match dtype { DataType::Float16 | DataType::Float32 | DataType::Float64 => { let is_finite_udf = ISFINITE_UDF.deref().clone(); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(is_finite_udf), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(is_finite_udf)), args: vec![arg], }) } diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 190128977..ee57a317c 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -6,7 +6,7 @@ use crate::expression::compiler::builtin_functions::date_time::datetime::{ use crate::expression::compiler::builtin_functions::type_checking::isvalid::is_valid_fn; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; -use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarUDF}; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarUDF, ScalarFunctionDefinition}; use std::collections::HashMap; use std::ops::Deref; use std::str::FromStr; @@ -133,15 +133,15 @@ pub fn compile_call( } VegaFusionCallable::ScalarUDF { udf, cast } => { let args = compile_scalar_arguments(node, config, schema, cast)?; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(udf.clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(udf.clone())), args, })) } VegaFusionCallable::BuiltinScalarFunction { function, cast } => { let args = compile_scalar_arguments(node, config, schema, cast)?; Ok(Expr::ScalarFunction(expr::ScalarFunction { - fun: *function, + func_def: ScalarFunctionDefinition::BuiltIn(*function), args, })) } diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index d1804237b..ffe3492b6 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -1,7 +1,7 @@ use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; -use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr}; +use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use std::convert::TryFrom; use std::ops::Deref; use std::sync::Arc; @@ -73,8 +73,8 @@ pub fn compile_member( let expr = match dtype { DataType::Struct(ref fields) => { if fields.iter().any(|f| f.name() == &property_string) { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_get_object_member_udf(&dtype, &property_string)?), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_object_member_udf(&dtype, &property_string)?)), args: vec![compiled_object], }) } else { @@ -86,15 +86,15 @@ pub fn compile_member( if property_string == "length" { // Special case to treat foo.length as length(foo) when foo is not an object // make_length() - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(LENGTH_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(LENGTH_UDF.deref().clone())), args: vec![compiled_object], }) } else if matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { if let Some(index) = index { // SQL substr function is 1-indexed so add one Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Substr, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Substr), args: vec![compiled_object, lit((index + 1) as i64), lit(1i64)], }) } else { @@ -104,8 +104,8 @@ pub fn compile_member( } } else if matches!(dtype, DataType::List(_) | DataType::FixedSizeList(_, _)) { if let Some(index) = index { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_get_element_udf(index as i32)), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_element_udf(index as i32))), args: vec![compiled_object], }) } else { diff --git a/vegafusion-runtime/src/expression/compiler/object.rs b/vegafusion-runtime/src/expression/compiler/object.rs index 3a1689061..364718e7d 100644 --- a/vegafusion-runtime/src/expression/compiler/object.rs +++ b/vegafusion-runtime/src/expression/compiler/object.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{expr, Expr, ExprSchemable}; +use datafusion_expr::{expr, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -26,8 +26,8 @@ pub fn compile_object( let udf = make_object_constructor_udf(keys.as_slice(), value_types.as_slice()); - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(udf), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(udf)), args: values, })) } diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index ce4ca1c56..91ff4ef3c 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use async_trait::async_trait; use datafusion_expr::{aggregate_function, expr}; use std::sync::Arc; +use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; @@ -162,35 +163,35 @@ pub fn make_agg_expr_for_col_expr( AggregateOp::Max => max(column), AggregateOp::Sum => sum(numeric_column()?), AggregateOp::Median => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::Median, + func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::Median), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Variance => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::Variance, + func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::Variance), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Variancep => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::VariancePop, + func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::VariancePop), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Stdev => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::Stddev, + func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::Stddev), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Stdevp => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::StddevPop, + func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::StddevPop), distinct: false, args: vec![numeric_column()?], filter: None, @@ -218,15 +219,17 @@ pub fn make_agg_expr_for_col_expr( }); count_distinct(column) + max(missing) } - AggregateOp::Q1 => Expr::AggregateUDF(expr::AggregateUDF { - fun: Arc::new((*Q1_UDF).clone()), + AggregateOp::Q1 => Expr::AggregateFunction(expr::AggregateFunction { + func_def: AggregateFunctionDefinition::UDF(Arc::new((*Q1_UDF).clone())), args: vec![numeric_column()?], + distinct: false, filter: None, order_by: None, }), - AggregateOp::Q3 => Expr::AggregateUDF(expr::AggregateUDF { - fun: Arc::new((*Q3_UDF).clone()), + AggregateOp::Q3 => Expr::AggregateFunction(expr::AggregateFunction { + func_def: AggregateFunctionDefinition::UDF(Arc::new((*Q3_UDF).clone())), args: vec![numeric_column()?], + distinct: false, filter: None, order_by: None, }), diff --git a/vegafusion-runtime/src/transform/timeunit.rs b/vegafusion-runtime/src/transform/timeunit.rs index c00c515a1..524a076ee 100644 --- a/vegafusion-runtime/src/transform/timeunit.rs +++ b/vegafusion-runtime/src/transform/timeunit.rs @@ -11,7 +11,7 @@ use vegafusion_core::proto::gen::transforms::{TimeUnit, TimeUnitTimeZone, TimeUn use vegafusion_core::task_graph::task_value::TaskValue; use datafusion_expr::expr::Cast; -use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use itertools::Itertools; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::datatypes::{cast_to, is_numeric_datatype}; @@ -53,8 +53,8 @@ fn timeunit_date_trunc( // Compute input timestamp expression based on timezone let tz_str = local_tz.clone().unwrap_or_else(|| "UTC".to_string()); - let start_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + let start_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit(part_str), field_col, lit(tz_str)], }); @@ -92,8 +92,8 @@ fn timeunit_date_part_tz( // Year if units_set.contains(&TimeUnitUnit::Year) { - make_timestamptz_args[0] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[0] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("year"), field_col.clone(), lit(&tz_str)], }); @@ -102,8 +102,8 @@ fn timeunit_date_part_tz( // Quarter if units_set.contains(&TimeUnitUnit::Quarter) { - let month = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let month = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("month"), field_col.clone(), lit(&tz_str)], }) .sub(lit(1.0)); @@ -118,8 +118,8 @@ fn timeunit_date_part_tz( // Month if units_set.contains(&TimeUnitUnit::Month) { - make_timestamptz_args[1] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[1] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("month"), field_col.clone(), lit(&tz_str)], }) .sub(lit(1.0)); @@ -129,8 +129,8 @@ fn timeunit_date_part_tz( // Date if units_set.contains(&TimeUnitUnit::Date) { - make_timestamptz_args[2] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[2] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("day"), field_col.clone(), lit(&tz_str)], }); @@ -139,8 +139,8 @@ fn timeunit_date_part_tz( // Hour if units_set.contains(&TimeUnitUnit::Hours) { - make_timestamptz_args[3] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[3] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), field_col.clone(), lit(&tz_str)], }); @@ -149,8 +149,8 @@ fn timeunit_date_part_tz( // Minute if units_set.contains(&TimeUnitUnit::Minutes) { - make_timestamptz_args[4] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[4] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("minute"), field_col.clone(), lit(&tz_str)], }); @@ -159,8 +159,8 @@ fn timeunit_date_part_tz( // Second if units_set.contains(&TimeUnitUnit::Seconds) { - make_timestamptz_args[5] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[5] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("second"), field_col, lit(&tz_str)], }); @@ -168,8 +168,8 @@ fn timeunit_date_part_tz( } // Construct expression to make timestamp from components - let start_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + let start_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: make_timestamptz_args, }); @@ -185,12 +185,12 @@ fn to_timestamp_col(field: &str, schema: &DFSchema, default_input_tz: &String) - &DataType::Timestamp(ArrowTimeUnit::Millisecond, None), schema, )?, - DataType::Utf8 => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![field_col, lit(default_input_tz)], }), - dtype if is_numeric_datatype(&dtype) => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![cast_to(field_col, &DataType::Int64, schema)?], }), dtype => { @@ -215,8 +215,8 @@ fn timeunit_weekday( // Use DATE_PART_TZ to extract the weekday // where Sunday is 0 and Saturday is 6 - let weekday0 = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let weekday0 = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("dow"), field_col, lit(tz_str)], }); @@ -237,8 +237,8 @@ fn timeunit_weekday( ]; // Construct expression to make timestamp from components - let start_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + let start_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: make_timestamptz_args, }); @@ -505,8 +505,8 @@ impl TransformTrait for TimeUnit { }; let tz_str = local_tz.unwrap_or_else(|| "UTC".to_string()); - let timeunit_end_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*DATE_ADD_TZ_UDF).clone()), + let timeunit_end_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF( Arc::new((*DATE_ADD_TZ_UDF).clone())), args: vec![ lit(&interval.1), lit(interval.0), diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 083441517..2b12cfe97 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -11,10 +11,7 @@ use sqlparser::ast::{ }; use datafusion_expr::expr::{BinaryExpr, Case, Cast, Sort}; -use datafusion_expr::{ - expr, lit, AggregateFunction, Between, BuiltInWindowFunction, BuiltinScalarFunction, Expr, - ExprSchemable, Operator, WindowFrameBound, WindowFrameUnits, WindowFunction, -}; +use datafusion_expr::{expr, lit, Between, BuiltInWindowFunction, Expr, ExprSchemable, Operator, WindowFrameBound, WindowFrameUnits, WindowFunction}; use crate::compile::function_arg::ToSqlFunctionArg; use crate::compile::order::ToSqlOrderByExpr; @@ -273,145 +270,16 @@ impl ToSqlExpr for Expr { // Sort expressions need to be handled at a higher level Err(VegaFusionError::internal("Sort cannot be converted to SQL")) } - Expr::ScalarFunction(expr::ScalarFunction { fun, args }) => { - let fun_name = match fun { - BuiltinScalarFunction::Abs => "abs", - BuiltinScalarFunction::Acos => "acos", - BuiltinScalarFunction::Asin => "asin", - BuiltinScalarFunction::Atan => "atan", - BuiltinScalarFunction::Atan2 => "atan2", - BuiltinScalarFunction::Ceil => "ceil", - BuiltinScalarFunction::Coalesce => "coalesce", - BuiltinScalarFunction::Cos => "cos", - BuiltinScalarFunction::Cbrt => "cbrt", - BuiltinScalarFunction::Digest => "digest", - BuiltinScalarFunction::Exp => "exp", - BuiltinScalarFunction::Floor => "floor", - BuiltinScalarFunction::Ln => "ln", - BuiltinScalarFunction::Log => "log", - BuiltinScalarFunction::Log10 => "log10", - BuiltinScalarFunction::Log2 => "log2", - BuiltinScalarFunction::Power => "pow", - BuiltinScalarFunction::Round => "round", - BuiltinScalarFunction::Signum => "signum", - BuiltinScalarFunction::Sin => "sin", - BuiltinScalarFunction::Sqrt => "sqrt", - BuiltinScalarFunction::Tan => "tan", - BuiltinScalarFunction::Trunc => "trunc", - BuiltinScalarFunction::MakeArray => "make_array", - BuiltinScalarFunction::Ascii => "ascii", - BuiltinScalarFunction::BitLength => "bit_length", - BuiltinScalarFunction::Btrim => "btrim", - BuiltinScalarFunction::CharacterLength => "length", - BuiltinScalarFunction::Chr => "chr", - BuiltinScalarFunction::Concat => "concat", - BuiltinScalarFunction::ConcatWithSeparator => "concat_ws", - BuiltinScalarFunction::DatePart => "date_part", - BuiltinScalarFunction::DateTrunc => "date_trunc", - BuiltinScalarFunction::DateBin => "date_bin", - BuiltinScalarFunction::InitCap => "initcap", - BuiltinScalarFunction::Left => "left", - BuiltinScalarFunction::Lpad => "lpad", - BuiltinScalarFunction::Lower => "lower", - BuiltinScalarFunction::Ltrim => "ltrim", - BuiltinScalarFunction::MD5 => "md5", - BuiltinScalarFunction::NullIf => "nullif", - BuiltinScalarFunction::OctetLength => "octet_length", - BuiltinScalarFunction::Random => "random", - BuiltinScalarFunction::RegexpReplace => "regexp_replace", - BuiltinScalarFunction::Repeat => "repeat", - BuiltinScalarFunction::Replace => "replace", - BuiltinScalarFunction::Reverse => "reverse", - BuiltinScalarFunction::Right => "right", - BuiltinScalarFunction::Rpad => "rpad", - BuiltinScalarFunction::Rtrim => "rtrim", - BuiltinScalarFunction::SHA224 => "sha224", - BuiltinScalarFunction::SHA256 => "sha256", - BuiltinScalarFunction::SHA384 => "sha384", - BuiltinScalarFunction::SHA512 => "sha512", - BuiltinScalarFunction::SplitPart => "split_part", - BuiltinScalarFunction::StartsWith => "starts_with", - BuiltinScalarFunction::Strpos => "strpos", - BuiltinScalarFunction::Substr => "substr", - BuiltinScalarFunction::ToHex => "to_hex", - BuiltinScalarFunction::ToTimestamp => "to_timestamp", - BuiltinScalarFunction::ToTimestampMillis => "to_timestamp_millis", - BuiltinScalarFunction::ToTimestampMicros => "to_timestamp_micros", - BuiltinScalarFunction::ToTimestampSeconds => "to_timestamp_seconds", - BuiltinScalarFunction::FromUnixtime => "from_unixtime", - BuiltinScalarFunction::Now => "now", - BuiltinScalarFunction::Translate => "translate", - BuiltinScalarFunction::Trim => "trim", - BuiltinScalarFunction::Upper => "upper", - BuiltinScalarFunction::RegexpMatch => "regexp_match", - BuiltinScalarFunction::Struct => "struct", - BuiltinScalarFunction::ArrowTypeof => "arrow_typeof", - BuiltinScalarFunction::CurrentDate => "current_date", - BuiltinScalarFunction::CurrentTime => "current_time", - BuiltinScalarFunction::Uuid => "uuid", - BuiltinScalarFunction::Acosh => "acosh", - BuiltinScalarFunction::Asinh => "asinh", - BuiltinScalarFunction::Atanh => "atanh", - BuiltinScalarFunction::Cosh => "cosh", - BuiltinScalarFunction::Degrees => "degrees", - BuiltinScalarFunction::Pi => "pi", - BuiltinScalarFunction::Radians => "radians", - BuiltinScalarFunction::Sinh => "sinh", - BuiltinScalarFunction::Tanh => "tanh", - BuiltinScalarFunction::Factorial => "factorial", - BuiltinScalarFunction::Gcd => "gcd", - BuiltinScalarFunction::Lcm => "lcm", - BuiltinScalarFunction::ArrayAppend => "array_append", - BuiltinScalarFunction::ArrayConcat => "array_concat", - BuiltinScalarFunction::ArrayDims => "array_dims", - BuiltinScalarFunction::ArrayLength => "array_length", - BuiltinScalarFunction::ArrayNdims => "array_ndims", - BuiltinScalarFunction::ArrayPosition => "array_position", - BuiltinScalarFunction::ArrayPositions => "array_positions", - BuiltinScalarFunction::ArrayPrepend => "array_prepend", - BuiltinScalarFunction::ArrayRemove => "array_remove", - BuiltinScalarFunction::ArrayReplace => "array_replace", - BuiltinScalarFunction::ArrayToString => "array_to_string", - BuiltinScalarFunction::Cardinality => "array_cardinality", - BuiltinScalarFunction::ArrayHas => "array_has", - BuiltinScalarFunction::ArrayHasAll => "array_has_all", - BuiltinScalarFunction::ArrayHasAny => "array_has_any", - BuiltinScalarFunction::ArrayPopBack => "array_pop_back", - BuiltinScalarFunction::ArrayElement => "array_element", - BuiltinScalarFunction::ArrayEmpty => "array_empty", - BuiltinScalarFunction::ArrayRemoveN => "array_remove_n", - BuiltinScalarFunction::ArrayRemoveAll => "array_remove_all", - BuiltinScalarFunction::ArrayRepeat => "array_repeat", - BuiltinScalarFunction::ArrayReplaceN => "array_replace_n", - BuiltinScalarFunction::ArrayReplaceAll => "array_replace_all", - BuiltinScalarFunction::ArraySlice => "array_slice", - BuiltinScalarFunction::ArrayIntersect => "array_intersect", - BuiltinScalarFunction::Decode => "decode", - BuiltinScalarFunction::Encode => "encode", - BuiltinScalarFunction::Cot => "cot", - BuiltinScalarFunction::Isnan => "isnan", - BuiltinScalarFunction::Iszero => "iszero", - BuiltinScalarFunction::Nanvl => "nanvl", - BuiltinScalarFunction::Flatten => "flatten", - BuiltinScalarFunction::StringToArray => "string_to_array", - BuiltinScalarFunction::ToTimestampNanos => "to_timestamp_nanos", - }; - translate_scalar_function(fun_name, args, dialect, schema) - } - Expr::ScalarUDF(expr::ScalarUDF { fun, args }) => { - translate_scalar_function(&fun.name, args, dialect, schema) + Expr::ScalarFunction(fun) => { + translate_scalar_function(&fun.name(), &fun.args, dialect, schema) } Expr::AggregateFunction(expr::AggregateFunction { - fun, - args, - distinct, - .. + func_def, + args, + distinct, + .. }) => { - let fun_name = aggr_fn_to_name(fun); - translate_aggregate_function(fun_name, args.as_slice(), *distinct, dialect, schema) - } - Expr::AggregateUDF(expr::AggregateUDF { fun, args, .. }) => { - translate_aggregate_function(&fun.name, args.as_slice(), false, dialect, schema) + translate_aggregate_function(func_def.name(), args.as_slice(), *distinct, dialect, schema) } Expr::WindowFunction(expr::WindowFunction { fun, @@ -423,7 +291,7 @@ impl ToSqlExpr for Expr { // Extract function name let (fun_name, supports_frame) = match fun { WindowFunction::AggregateFunction(agg) => { - (aggr_fn_to_name(agg).to_string().to_ascii_lowercase(), true) + (agg.name().to_string().to_ascii_lowercase(), true) } WindowFunction::BuiltInWindowFunction(win_fn) => { let is_navigation_function = matches!( @@ -444,8 +312,8 @@ impl ToSqlExpr for Expr { (win_fn.to_string().to_ascii_lowercase(), supports_frame) } - WindowFunction::AggregateUDF(udf) => (udf.name.to_ascii_lowercase(), true), - WindowFunction::WindowUDF(udf) => (udf.name.to_ascii_lowercase(), true), + WindowFunction::AggregateUDF(udf) => (udf.name().to_ascii_lowercase(), true), + WindowFunction::WindowUDF(udf) => (udf.name().to_ascii_lowercase(), true), }; // Handle unordered row_number @@ -707,46 +575,6 @@ fn translate_function_args( .collect::>>() } -fn aggr_fn_to_name(fun: &AggregateFunction) -> &str { - match fun { - AggregateFunction::Min => "min", - AggregateFunction::Max => "max", - AggregateFunction::Count => "count", - AggregateFunction::Avg => "avg", - AggregateFunction::Sum => "sum", - AggregateFunction::Median => "median", - AggregateFunction::ApproxDistinct => "approx_distinct", - AggregateFunction::ArrayAgg => "array_agg", - AggregateFunction::Variance => "var", - AggregateFunction::VariancePop => "var_pop", - AggregateFunction::Stddev => "stddev", - AggregateFunction::StddevPop => "stddev_pop", - AggregateFunction::Covariance => "covar", - AggregateFunction::CovariancePop => "covar_pop", - AggregateFunction::Correlation => "corr", - AggregateFunction::ApproxPercentileCont => "approx_percentile_cont", - AggregateFunction::ApproxPercentileContWithWeight => "approx_percentile_cont_with_weight", - AggregateFunction::ApproxMedian => "approx_median", - AggregateFunction::Grouping => "grouping", - AggregateFunction::BitAnd => "bit_and", - AggregateFunction::BitOr => "bit_or", - AggregateFunction::BitXor => "bit_xor", - AggregateFunction::BoolAnd => "bool_and", - AggregateFunction::BoolOr => "bool_or", - AggregateFunction::FirstValue => "first_value", - AggregateFunction::LastValue => "last_value", - AggregateFunction::RegrSlope => "regr_slope", - AggregateFunction::RegrIntercept => "regr_intercept", - AggregateFunction::RegrCount => "regr_count", - AggregateFunction::RegrR2 => "regr_r2", - AggregateFunction::RegrAvgx => "regr_avgx", - AggregateFunction::RegrAvgy => "regr_avgy", - AggregateFunction::RegrSXX => "regr_sxx", - AggregateFunction::RegrSYY => "regr_syy", - AggregateFunction::RegrSXY => "regr_sxy", - } -} - fn compile_window_frame_bound( bound: &WindowFrameBound, dialect: &Dialect, @@ -776,7 +604,7 @@ mod tests { use arrow::datatypes::DataType; use datafusion_common::DFSchema; use datafusion_expr::expr::Cast; - use datafusion_expr::{expr, lit, Between, BuiltinScalarFunction, Expr}; + use datafusion_expr::{expr, lit, Between, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use vegafusion_common::column::flat_col; fn schema() -> DFSchema { @@ -795,7 +623,7 @@ mod tests { #[test] pub fn test2() { let df_expr = Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Sin, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Sin), args: vec![lit(1.2)], }) + flat_col("B"); @@ -809,7 +637,7 @@ mod tests { #[test] pub fn test3() { let df_expr = Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Upper, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Upper), args: vec![lit("foo")], }); diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 4e459a949..932a4301a 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -5,10 +5,7 @@ use arrow::array::Array; use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; -use datafusion_expr::{ - expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, - Signature, Volatility, -}; +use datafusion_expr::{expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, Volatility, ScalarFunctionDefinition}; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr, Ident, ObjectName as SqlObjectName, Value as SqlValue, @@ -234,9 +231,6 @@ impl ToSqlScalar for ScalarValue { ScalarValue::Time64Nanosecond(_) => Err(VegaFusionError::internal( "Time64Nanosecond cannot be converted to SQL", )), - ScalarValue::Fixedsizelist(_, _, _) => Err(VegaFusionError::internal( - "Fixedsizelist cannot be converted to SQL", - )), ScalarValue::DurationSecond(_) => Err(VegaFusionError::internal( "DurationSecond cannot be converted to SQL", )), @@ -249,6 +243,12 @@ impl ToSqlScalar for ScalarValue { ScalarValue::DurationNanosecond(_) => Err(VegaFusionError::internal( "DurationNanosecond cannot be converted to SQL", )), + ScalarValue::FixedSizeList(_) => Err(VegaFusionError::internal( + "FixedSizeList cannot be converted to SQL", + )), + ScalarValue::LargeList(_) => Err(VegaFusionError::internal( + "LargeList cannot be converted to SQL", + )), } } } @@ -268,8 +268,8 @@ fn ms_to_timestamp(v: i64, dialect: &Dialect) -> Result { &return_type, &scalar_fn, ); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(udf), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(udf)), args: vec![lit(v)], }) .to_sql(dialect, &DFSchema::empty()) diff --git a/vegafusion-sql/src/compile/select.rs b/vegafusion-sql/src/compile/select.rs index 099e42858..11a60d8d8 100644 --- a/vegafusion-sql/src/compile/select.rs +++ b/vegafusion-sql/src/compile/select.rs @@ -12,7 +12,7 @@ pub trait ToSqlSelectItem { impl ToSqlSelectItem for Expr { fn to_sql_select(&self, dialect: &Dialect, schema: &DFSchema) -> Result { Ok(match self { - Expr::Alias(expr::Alias { expr, name: alias }) => SqlSelectItem::ExprWithAlias { + Expr::Alias(expr::Alias { expr, name: alias, .. }) => SqlSelectItem::ExprWithAlias { expr: expr.to_sql(dialect, schema)?, alias: Ident { value: alias.clone(), diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index fa7386241..cdebc21ce 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -7,11 +7,7 @@ use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{Column, DFSchema, OwnedTableReference, ScalarValue}; -use datafusion_expr::{ - abs, expr, is_null, lit, max, min, when, window_function, AggregateFunction, - BuiltInWindowFunction, BuiltinScalarFunction, Expr, ExprSchemable, WindowFrame, - WindowFrameBound, WindowFrameUnits, WindowFunction, -}; +use datafusion_expr::{abs, expr, is_null, lit, max, min, when, window_function, AggregateFunction, BuiltInWindowFunction, BuiltinScalarFunction, Expr, ExprSchemable, WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunction, ScalarFunctionDefinition}; use sqlparser::ast::{ Cte, Expr as SqlExpr, GroupByExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, @@ -23,6 +19,7 @@ use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::ops::{Add, Div, Sub}; use std::sync::Arc; +use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datatypes::to_numeric; @@ -895,7 +892,7 @@ impl SqlDataFrame { let partition_by: Vec<_> = groupby.iter().map(|group| flat_col(group)).collect(); let numeric_field = Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![to_numeric(flat_col(field), &self.schema_df()?)?, lit(0.0)], }); @@ -983,7 +980,7 @@ impl SqlDataFrame { // Create aggregate for total of stack value let total_agg = Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Sum, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Sum), args: vec![flat_col(stack_col_name)], distinct: false, filter: None, @@ -1154,7 +1151,7 @@ impl SqlDataFrame { if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1199,7 +1196,7 @@ impl SqlDataFrame { .map(|col_name| { if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1217,7 +1214,7 @@ impl SqlDataFrame { .map(|col_name| { let expr = if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) From baf1f51b79941f80a18b9c5826413246f74bd06b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 11 Feb 2024 07:09:21 -0500 Subject: [PATCH 04/44] Update to datafusion 35, arrow 50, sqlparser 0.41 --- Cargo.lock | 253 +++++++++++++----- Cargo.toml | 18 +- vegafusion-common/src/data/scalar.rs | 19 +- vegafusion-common/src/datatypes.rs | 9 +- vegafusion-core/src/task_graph/memory.rs | 10 +- vegafusion-dataframe/src/dataframe.rs | 12 +- vegafusion-datafusion-udfs/src/udafs/mod.rs | 4 +- .../src/udfs/array/indexof.rs | 4 +- vegafusion-runtime/src/data/tasks.rs | 24 +- .../data/vl_selection_resolve.rs | 8 +- .../data/vl_selection_test.rs | 24 +- .../date_time/date_format.rs | 8 +- .../builtin_functions/date_time/date_parts.rs | 8 +- .../builtin_functions/date_time/datetime.rs | 8 +- .../builtin_functions/date_time/time.rs | 12 +- .../src/expression/compiler/call.rs | 2 +- .../src/expression/compiler/conditional.rs | 4 +- .../src/expression/compiler/member.rs | 9 +- .../src/expression/compiler/mod.rs | 35 ++- vegafusion-runtime/src/transform/aggregate.rs | 22 +- vegafusion-runtime/src/transform/collect.rs | 16 +- .../src/transform/identifier.rs | 10 +- vegafusion-runtime/src/transform/timeunit.rs | 22 +- vegafusion-runtime/src/transform/window.rs | 29 +- .../tests/test_vegajs_runtime.rs | 6 +- vegafusion-sql/src/compile/expr.rs | 39 ++- vegafusion-sql/src/compile/scalar.rs | 9 +- vegafusion-sql/src/compile/select.rs | 4 +- vegafusion-sql/src/dataframe/mod.rs | 55 ++-- 29 files changed, 436 insertions(+), 247 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cfa9f8830..ba5ae69a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,11 +141,10 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc25126d18a012146a888a0298f2c22e1150327bd2765fc76d710a556b2d614" +checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" dependencies = [ - "ahash", "arrow-arith", "arrow-array", "arrow-buffer", @@ -164,9 +163,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ccd45e217ffa6e53bbb0080990e77113bdd4e91ddb84e97b77649810bcf1a7" +checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" dependencies = [ "arrow-array", "arrow-buffer", @@ -179,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d" +checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ "ahash", "arrow-buffer", @@ -196,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c" +checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" dependencies = [ "bytes", "half 2.2.1", @@ -207,9 +206,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc0368ed618d509636c1e3cc20db1281148190a78f43519487b2daf07b63b4a" +checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" dependencies = [ "arrow-array", "arrow-buffer", @@ -226,9 +225,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e09aa6246a1d6459b3f14baeaa49606cfdbca34435c46320e14054d244987ca" +checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,9 +244,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634" +checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" dependencies = [ "arrow-buffer", "arrow-schema", @@ -257,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a43d6808411886b8c7d4f6f7dd477029c1e77ffffffb7923555cc6579639cd" +checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,13 +266,14 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] name = "arrow-json" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82565c91fd627922ebfe2810ee4e8346841b6f9361b87505a9acea38b614fee" +checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b23b0e53c0db57c6749997fd343d4c0354c994be7eca67152dd2bdb9a3e1bb4" +checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "361249898d2d6d4a6eeb7484be6ac74977e48da12a4dd81a708d620cc558117a" +checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ "ahash", "arrow-array", @@ -321,18 +321,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167" +checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" dependencies = [ "bitflags 2.1.0", ] [[package]] name = "arrow-select" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f6208466590960efc1d2a7172bc4ff18a67d6e25c529381d7f96ddaf0dc4036" +checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ "ahash", "arrow-array", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a48149c63c11c9ff571e50ab8f017d2a7cb71037a882b42f6354ed2da9acc7" +checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -1239,13 +1239,14 @@ dependencies = [ [[package]] name = "datafusion" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193fd1e7628278d0641c5122860f9a7fd6a1d77d055838d12f55d15bbe28d4d0" +checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-ipc", "arrow-schema", "async-compression", "async-trait", @@ -1274,7 +1275,7 @@ dependencies = [ "parquet", "pin-project-lite", "rand", - "sqlparser 0.40.0", + "sqlparser", "tempfile", "tokio", "tokio-util", @@ -1286,9 +1287,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "548bc49c4a489e3de474813831ea556dc9d368f9ed8d867b1493da42e8e9f613" +checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" dependencies = [ "ahash", "arrow", @@ -1302,14 +1303,14 @@ dependencies = [ "object_store", "parquet", "pyo3", - "sqlparser 0.40.0", + "sqlparser", ] [[package]] name = "datafusion-execution" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc865657ffcf4da5ff08bdc6436a9a833bc0aa96c3254c8d18ab8a0ad4e437d" +checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" dependencies = [ "arrow", "chrono", @@ -1328,25 +1329,25 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c473f72d8d81a532e63f6e562ed66dd9209dfd8e433d9712abd42444ee161e" +checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", "paste", - "sqlparser 0.40.0", + "sqlparser", "strum 0.25.0", "strum_macros 0.25.1", ] [[package]] name = "datafusion-optimizer" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6218318001d2f6783b7fffa17592318f65f26609d7aab605a3dd0c7c2e2618" +checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" dependencies = [ "arrow", "async-trait", @@ -1362,9 +1363,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1ca7e35ca22f9dc506c2375b92054b03ccf91afe25c0a90b395a1473a09735" +checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" dependencies = [ "ahash", "arrow", @@ -1396,9 +1397,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddde97adefcca3a55257c646ffee2a95b6cac66f74d1146a6e3a6dbb37830631" +checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" dependencies = [ "ahash", "arrow", @@ -1427,9 +1428,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "344700ad4505ffcab6ba0715188ce317ad44e024fe1c5dc7267762113a9dcdf8" +checksum = "5742f993d1812d6bb3cdc4ce2a0aa99e10f6dc0daa11dd69b0ff57f2d8e7518c" dependencies = [ "arrow", "chrono", @@ -1442,16 +1443,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60d9d6460a64fddb8663db41da97e6b8b0bf79da42f997ebe81722731eaf0e5" +checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.40.0", + "sqlparser", ] [[package]] @@ -2637,9 +2638,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" dependencies = [ "async-trait", "base64", @@ -2648,7 +2649,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.11.0", + "itertools 0.12.1", "parking_lot 0.12.1", "percent-encoding", "quick-xml", @@ -2676,6 +2677,12 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "ordered-float" version = "2.10.0" @@ -2756,9 +2763,9 @@ dependencies = [ [[package]] name = "parquet" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4" +checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ "ahash", "arrow-array", @@ -2774,6 +2781,7 @@ dependencies = [ "chrono", "flate2", "futures", + "half 2.2.1", "hashbrown 0.14.0", "lz4_flex", "num", @@ -3326,6 +3334,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", + "rustls-native-certs", "rustls-pemfile", "serde", "serde_json", @@ -3522,6 +3531,18 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -3562,6 +3583,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3590,6 +3620,29 @@ dependencies = [ "untrusted 0.7.1", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -3749,18 +3802,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.39.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" -dependencies = [ - "log", -] - -[[package]] -name = "sqlparser" -version = "0.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c80afe31cdb649e56c0d9bb5503be9166600d68a852c38dd445636d126858e5" +checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" dependencies = [ "log", "sqlparser_derive", @@ -4045,7 +4089,6 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", "pin-project-lite", "socket2 0.5.3", "tokio-macros", @@ -4424,7 +4467,7 @@ dependencies = [ "object_store", "pyo3", "serde_json", - "sqlparser 0.39.0", + "sqlparser", "thiserror", ] @@ -4451,7 +4494,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sqlparser 0.39.0", + "sqlparser", "thiserror", "tonic", "tonic-build", @@ -4560,7 +4603,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "sqlparser 0.39.0", + "sqlparser", "tempfile", "test-case", "tokio", @@ -4620,7 +4663,7 @@ dependencies = [ "rstest_reuse", "serde", "serde_json", - "sqlparser 0.39.0", + "sqlparser", "tempfile", "tokio", "toml", @@ -4902,6 +4945,15 @@ dependencies = [ "windows-targets 0.48.0", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -4932,6 +4984,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.0", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -4944,6 +5011,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -4956,6 +5029,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -4968,6 +5047,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -4980,6 +5065,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -4992,6 +5083,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -5004,6 +5101,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -5016,6 +5119,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index f483b84ac..897cba9a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ members = [ ] [workspace.dependencies] -arrow = { version = "49.0.0", default_features = false } -sqlparser = { version = "0.39.0" } +arrow = { version = "50.0.0", default_features = false } +sqlparser = { version = "0.41.0" } chrono = { version = "0.4.31", default_features = false } reqwest = { version = "0.11.22", default-features = false } tokio = { version = "1.32.0" } @@ -23,14 +23,14 @@ pyo3 = { version = "0.20.0" } pythonize = { version = "0.20.0" } prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } -object_store = { version= "0.8.0" } +object_store = { version= "0.9.0" } -datafusion = { version = "34.0.0" } -datafusion-common = { version = "34.0.0", default_features = false} -datafusion-expr = { version = "34.0.0" } -datafusion-proto = { version = "34.0.0" } -datafusion-physical-expr = { version = "34.0.0" } -datafusion-optimizer = { version = "34.0.0" } +datafusion = { version = "35.0.0" } +datafusion-common = { version = "35.0.0", default_features = false} +datafusion-expr = { version = "35.0.0" } +datafusion-proto = { version = "35.0.0" } +datafusion-physical-expr = { version = "35.0.0" } +datafusion-optimizer = { version = "35.0.0" } [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index c66c2b01d..4f33103df 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -136,12 +136,16 @@ impl ScalarValueHelpers for ScalarValue { ScalarValue::IntervalDayTime(Some(_v)) => { unimplemented!() } - ScalarValue::List(a) => Value::Array( - a.list_el_to_scalar_vec()? - .iter() - .map(|s| s.to_json()) - .collect::>>()?, - ), + ScalarValue::List(a) => { + let values = a + .value(0) + .to_scalar_vec()? + .into_iter() + .map(|v| v.to_json()) + .collect::>>()?; + + Value::Array(values) + } ScalarValue::Struct(Some(v), fields) => { let mut pairs: Map = Default::default(); for (val, field) in v.iter().zip(fields.deref()) { @@ -151,7 +155,6 @@ impl ScalarValueHelpers for ScalarValue { } _ => Value::Null, }; - Ok(res) } @@ -177,7 +180,7 @@ impl ScalarValueHelpers for ScalarValue { fn to_f64x2(&self) -> Result<[f64; 2]> { if let ScalarValue::List(array) = self { - let elements = array.list_el_to_scalar_vec()?; + let elements = array.value(0).to_scalar_vec()?; if let [v0, v1] = elements.as_slice() { return Ok([v0.to_f64()?, v1.to_f64()?]); } diff --git a/vegafusion-common/src/datatypes.rs b/vegafusion-common/src/datatypes.rs index 70f880a0d..784e343f2 100644 --- a/vegafusion-common/src/datatypes.rs +++ b/vegafusion-common/src/datatypes.rs @@ -1,7 +1,10 @@ use crate::error::{Result, ResultWithContext}; use arrow::datatypes::DataType; use datafusion_common::DFSchema; -use datafusion_expr::{coalesce, expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, TryCast, ScalarFunctionDefinition}; +use datafusion_expr::{ + coalesce, expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, + TryCast, +}; pub fn is_numeric_datatype(dtype: &DataType) -> bool { matches!( @@ -87,7 +90,9 @@ pub fn to_numeric(value: Expr, schema: &DFSchema) -> Result { // Convert to milliseconds Expr::TryCast(TryCast { expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ToTimestampMillis), + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::ToTimestampMillis, + ), args: vec![value], })), data_type: DataType::Int64, diff --git a/vegafusion-core/src/task_graph/memory.rs b/vegafusion-core/src/task_graph/memory.rs index b7d753bb0..279201e01 100644 --- a/vegafusion-core/src/task_graph/memory.rs +++ b/vegafusion-core/src/task_graph/memory.rs @@ -1,3 +1,4 @@ +use crate::arrow::array::ListArray; use datafusion_common::ScalarValue; use std::mem::{size_of, size_of_val}; use vegafusion_common::arrow::array::ArrayRef; @@ -47,7 +48,7 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { ScalarValue::LargeUtf8(Some(s)) => size_of_val(s.as_bytes()) + size_of::(), ScalarValue::Binary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), ScalarValue::LargeBinary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), - ScalarValue::List(array) => size_of::>() + size_of_array_ref(array), + ScalarValue::List(array) => size_of::>() + size_of_list_array(array), ScalarValue::Struct(Some(values), fields) => { let values_bytes: usize = size_of::>() + values @@ -67,6 +68,13 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { } } +pub fn size_of_list_array(array: &ListArray) -> usize { + array + .iter() + .map(|el| el.map(|el| size_of_array_ref(&el)).unwrap_or(0)) + .sum() +} + pub fn size_of_array_ref(array: &ArrayRef) -> usize { array.get_array_memory_size() + inner_size_of_dtype(array.data_type()) + size_of::() } diff --git a/vegafusion-dataframe/src/dataframe.rs b/vegafusion-dataframe/src/dataframe.rs index 93cfecfba..6d0519664 100644 --- a/vegafusion-dataframe/src/dataframe.rs +++ b/vegafusion-dataframe/src/dataframe.rs @@ -5,8 +5,8 @@ use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::{ - expr, window_function, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, - WindowFrameUnits, + expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, WindowFrameUnits, + WindowFunctionDefinition, }; use std::any::Any; use std::fmt::{Display, Formatter}; @@ -116,17 +116,13 @@ pub trait DataFrame: Send + Sync + 'static { } else { let selections = vec![ Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], partition_by: vec![], order_by: vec![], - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(index_name), Expr::Wildcard { qualifier: None }, diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index 4f3a7d41b..60131ecf2 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -20,7 +20,7 @@ pub(crate) struct PercentileContAccumulator { impl Accumulator for PercentileContAccumulator { fn state(&self) -> Result, DataFusionError> { let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type); - Ok(vec![ScalarValue::List(Arc::new(state))]) + Ok(vec![ScalarValue::List(state)]) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<(), DataFusionError> { @@ -44,7 +44,7 @@ impl Accumulator for PercentileContAccumulator { for index in 0..array.len() { match ScalarValue::try_from_array(array, index)? { ScalarValue::List(array) => { - for scalar in array.list_el_to_scalar_vec()? { + for scalar in array.value(0).to_scalar_vec()? { if !scalar_is_non_finite(&scalar) { self.all_values.push(scalar); } diff --git a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs index e223518ec..f5c47aa80 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs @@ -25,8 +25,8 @@ fn make_indexof_udf() -> ScalarUDF { // Signature ensures there is a single argument let (array, array_dtype) = match &args[0] { ColumnarValue::Scalar(ScalarValue::List(array)) => { - let scalar_array = array.list_el_to_scalar_vec()?; - (scalar_array, array.list_el_dtype()?) + let scalar_array = array.value(0).to_scalar_vec()?; + (scalar_array, array.value(0).data_type().clone()) } _ => { return Err(DataFusionError::Internal( diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index fbc63bc67..a1b35aec0 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -337,7 +337,9 @@ async fn process_datetimes( .unwrap_or_else(|| "UTC".to_string()); Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![flat_col(&spec.name), lit(default_input_tz_str)], }) } else if is_integer_datatype(dtype) { @@ -345,7 +347,9 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*MAKE_UTC_TIMESTAMP).clone(), + )), args: vec![ flat_col(&spec.name), // year lit(0), // month @@ -396,7 +400,9 @@ async fn process_datetimes( Some(tz) => { // Timestamp has explicit timezone Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![flat_col(field.name()), lit(tz.as_ref())], }) } @@ -406,7 +412,9 @@ async fn process_datetimes( tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -419,7 +427,9 @@ async fn process_datetimes( tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -431,7 +441,9 @@ async fn process_datetimes( tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*DATE_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![flat_col(field.name()), lit(tz_config.local_tz.to_string())], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs index 99d21abb6..024c3079f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs @@ -63,7 +63,7 @@ pub fn vl_selection_resolve_fn( // Extract vector of rows for selection dataset let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { - array.list_el_to_scalar_vec()? + array.value(0).to_scalar_vec()? } else { unreachable!() }; @@ -79,16 +79,16 @@ pub fn vl_selection_resolve_fn( let value = match field.typ { SelectionType::Enum => { if let ScalarValue::List(array) = value { - array.list_el_to_scalar_vec()? + array.value(0).to_scalar_vec()? } else { vec![value.clone()] } } _ => { match &value { - ScalarValue::List(array) if array.list_el_len()? == 2 => { + ScalarValue::List(array) if array.value(0).len() == 2 => { // Don't assume elements are in ascending order - let elements = array.list_el_to_scalar_vec()?; + let elements = array.value(0).to_scalar_vec()?; let first = elements[0].to_f64()?; let second = elements[1].to_f64()?; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index 8011ad66d..da9d1cd45 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -132,7 +132,9 @@ impl FieldSpec { DataType::Timestamp(TimeUnit::Millisecond, _) ) { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![field_col], }) } else { @@ -143,7 +145,7 @@ impl FieldSpec { SelectionType::Enum => { let field_type = field_col.get_type(schema)?; let list_scalars = if let ScalarValue::List(array) = &values { - array.list_el_to_scalar_vec()? + array.value(0).to_scalar_vec()? } else { // convert values to single element list vec![values.clone()] @@ -178,8 +180,8 @@ impl FieldSpec { }; let (low, high) = match &values { - ScalarValue::List(array) if array.list_el_len()? == 2 => { - let elements = array.list_el_to_scalar_vec()?; + ScalarValue::List(array) if array.value(0).len() == 2 => { + let elements = array.value(0).to_scalar_vec()?; let first = Self::cast_test_scalar( elements[0].clone(), &field_dtype, @@ -271,11 +273,15 @@ impl FieldSpec { && is_numeric_datatype(field_type) => { let timestamp_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![lit(s), lit(default_input_tz)], }); let ms_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![timestamp_expr], }); cast_to(ms_expr, field_type, schema) @@ -406,7 +412,7 @@ impl TryFrom for SelectionRow { .get("values") .with_context(|| "Missing required property 'values'".to_string())?; let values = match struct_values.get(*values_index) { - Some(ScalarValue::List(array)) => array.list_el_to_scalar_vec()?, + Some(ScalarValue::List(array)) => array.value(0).to_scalar_vec()?, _ => { return Err(VegaFusionError::internal( "Expected 'values' to be an array".to_string(), @@ -422,7 +428,7 @@ impl TryFrom for SelectionRow { let mut fields: Vec = Vec::new(); match struct_values.get(*fields_index) { Some(ScalarValue::List(array)) => { - for el in array.list_el_to_scalar_vec()?.iter() { + for el in array.value(0).to_scalar_vec()?.iter() { fields.push(FieldSpec::try_from(el.clone())?) } } @@ -510,7 +516,7 @@ pub fn vl_selection_test_fn( // Extract vector of rows for selection dataset let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { - array.list_el_to_scalar_vec()? + array.value(0).to_scalar_vec()? } else { unreachable!() }; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs index ab9ab2d22..9835c1ba4 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs @@ -49,7 +49,9 @@ pub fn time_format_fn( // General case if format_tz_str.to_ascii_lowercase() != "utc" { timestamptz_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*FROM_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*FROM_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![timestamptz_expr, lit(format_tz_str)], }) } @@ -108,7 +110,9 @@ fn to_timestamptz_expr(arg: &Expr, schema: &DFSchema, default_input_tz: &str) -> DataType::Null => arg.clone(), dtype if is_numeric_datatype(&dtype) || matches!(dtype, DataType::Boolean) => { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs index eef1cbc4f..7701bc7f9 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -74,11 +74,15 @@ fn extract_timestamp_arg( }), DataType::Timestamp(_, _) => arg.clone(), DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![arg.clone(), lit(default_input_tz)], }), dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }), dtype => { diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs index a20972b54..b79440491 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs @@ -49,7 +49,9 @@ pub fn to_date_transform( })) } else if is_numeric_datatype(&dtype) { Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(arg, &DataType::Int64, schema)?], })) } else { @@ -72,7 +74,9 @@ pub fn datetime_transform_fn( if is_string_datatype(&dtype) { let default_input_tz_str = tz_config.default_input_tz.to_string(); arg = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![arg, lit(default_input_tz_str)], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs index febb81d90..730d8b73f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs @@ -24,7 +24,9 @@ pub fn time_fn(tz_config: &RuntimeTzConfig, args: &[Expr], schema: &DFSchema) -> let expr = match arg.get_type(schema)? { DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64 => { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![arg.clone()], }) } @@ -32,9 +34,13 @@ pub fn time_fn(tz_config: &RuntimeTzConfig, args: &[Expr], schema: &DFSchema) -> let mut udf_args = vec![lit(tz_config.default_input_tz.to_string())]; udf_args.extend(Vec::from(args)); Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![arg.clone(), lit(tz_config.default_input_tz.to_string())], })], }) diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index ee57a317c..facdd24e9 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -6,7 +6,7 @@ use crate::expression::compiler::builtin_functions::date_time::datetime::{ use crate::expression::compiler::builtin_functions::type_checking::isvalid::is_valid_fn; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; -use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarUDF, ScalarFunctionDefinition}; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, ScalarUDF}; use std::collections::HashMap; use std::ops::Deref; use std::str::FromStr; diff --git a/vegafusion-runtime/src/expression/compiler/conditional.rs b/vegafusion-runtime/src/expression/compiler/conditional.rs index 7828856f7..42050ebf2 100644 --- a/vegafusion-runtime/src/expression/compiler/conditional.rs +++ b/vegafusion-runtime/src/expression/compiler/conditional.rs @@ -41,8 +41,8 @@ pub fn compile_conditional( if let Expr::Case(case) = alternate_expr { // Flatten nested case statements - let mut when_then_expr = case.when_then_expr; - when_then_expr.extend(vec![(Box::new(test), Box::new(consequent_expr))]); + let mut when_then_expr = vec![(Box::new(test), Box::new(consequent_expr))]; + when_then_expr.extend(case.when_then_expr); Ok(Expr::Case(Case { expr: None, when_then_expr, diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index ffe3492b6..f6fc08e4e 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -74,7 +74,10 @@ pub fn compile_member( DataType::Struct(ref fields) => { if fields.iter().any(|f| f.name() == &property_string) { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_object_member_udf(&dtype, &property_string)?)), + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_object_member_udf( + &dtype, + &property_string, + )?)), args: vec![compiled_object], }) } else { @@ -105,7 +108,9 @@ pub fn compile_member( } else if matches!(dtype, DataType::List(_) | DataType::FixedSizeList(_, _)) { if let Some(index) = index { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_element_udf(index as i32))), + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_element_udf( + index as i32, + ))), args: vec![compiled_object], }) } else { diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index ebbf40cbd..b39712910 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -63,6 +63,7 @@ mod test_compile { use vegafusion_core::expression::parser::parse; use crate::task_graph::timezone::RuntimeTzConfig; + use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Case, TryCast}; use datafusion_expr::{concat, expr, lit, BuiltinScalarFunction, Expr, Operator}; @@ -70,10 +71,9 @@ mod test_compile { use std::convert::TryFrom; use std::ops::Deref; use std::sync::Arc; - use datafusion_common::utils::array_into_list_array; use vegafusion_common::arrow::datatypes::{DataType, Field, Schema}; use vegafusion_common::column::flat_col; - use vegafusion_core::arrow::array::{Float64Array, new_empty_array}; + use vegafusion_core::arrow::array::{new_empty_array, Float64Array}; use vegafusion_core::arrow::datatypes::Fields; use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; use vegafusion_datafusion_udfs::udfs::object::make_object_constructor_udf; @@ -401,8 +401,9 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - - let expected_value = ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))))); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![1.0, 2.0, 3.0]), + )))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -423,8 +424,9 @@ mod test_compile { // Check evaluated value. Empty array is given Float64 data type let result_value = result_expr.eval_to_scalar().unwrap(); - - let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array(&DataType::Float64)))); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array( + &DataType::Float64, + )))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -457,13 +459,20 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List(Arc::new(array_into_list_array(ScalarValue::iter_to_array( - vec![ - ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![1.0, 2.0]))))), - ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![3.0, 4.0]))))), - ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![5.0, 6.0]))))), - ] - ).unwrap()))); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![ + ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![1.0, 2.0]), + )))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![3.0, 4.0]), + )))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![5.0, 6.0]), + )))), + ]) + .unwrap(), + ))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index 91ff4ef3c..11b7e776a 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -5,9 +5,9 @@ use datafusion_expr::{avg, count, count_distinct, lit, max, min, sum, Expr}; use std::collections::HashMap; use async_trait::async_trait; +use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{aggregate_function, expr}; use std::sync::Arc; -use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; @@ -163,35 +163,45 @@ pub fn make_agg_expr_for_col_expr( AggregateOp::Max => max(column), AggregateOp::Sum => sum(numeric_column()?), AggregateOp::Median => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::Median), + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Median, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Variance => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::Variance), + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Variance, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Variancep => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::VariancePop), + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::VariancePop, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Stdev => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::Stddev), + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Stddev, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Stdevp => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(aggregate_function::AggregateFunction::StddevPop), + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::StddevPop, + ), distinct: false, args: vec![numeric_column()?], filter: None, diff --git a/vegafusion-runtime/src/transform/collect.rs b/vegafusion-runtime/src/transform/collect.rs index 2c213db80..ed43f06ee 100644 --- a/vegafusion-runtime/src/transform/collect.rs +++ b/vegafusion-runtime/src/transform/collect.rs @@ -1,7 +1,7 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; -use datafusion_expr::{expr, Expr}; +use datafusion_expr::{expr, Expr, WindowFunctionDefinition}; use std::sync::Arc; use vegafusion_core::error::{Result, ResultWithContext}; @@ -9,9 +9,7 @@ use vegafusion_core::proto::gen::transforms::{Collect, SortOrder}; use async_trait::async_trait; use datafusion_common::scalar::ScalarValue; -use datafusion_expr::{ - window_function, BuiltInWindowFunction, WindowFrame, WindowFrameBound, WindowFrameUnits, -}; +use datafusion_expr::{BuiltInWindowFunction, WindowFrame, WindowFrameBound, WindowFrameUnits}; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_core::task_graph::task_value::TaskValue; @@ -47,17 +45,11 @@ impl TransformTrait for Collect { // criteria. This column becomes the new ORDER_COL, which will be sorted at the end of // the pipeline. let order_col = Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( - BuiltInWindowFunction::RowNumber, - ), + fun: WindowFunctionDefinition::BuiltInWindowFunction(BuiltInWindowFunction::RowNumber), args: vec![], partition_by: vec![], order_by: sort_exprs, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(ORDER_COL); diff --git a/vegafusion-runtime/src/transform/identifier.rs b/vegafusion-runtime/src/transform/identifier.rs index 5facb150d..30b356a5d 100644 --- a/vegafusion-runtime/src/transform/identifier.rs +++ b/vegafusion-runtime/src/transform/identifier.rs @@ -4,7 +4,7 @@ use crate::transform::TransformTrait; use async_trait::async_trait; use datafusion_expr::{ expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, WindowFrameUnits, - WindowFunction, + WindowFunctionDefinition, }; use std::sync::Arc; use vegafusion_common::column::flat_col; @@ -24,7 +24,7 @@ impl TransformTrait for Identifier { ) -> Result<(Arc, Vec)> { // Add row number column with the desired name, sorted by the input order column let row_number_expr = Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunction::BuiltInWindowFunction(BuiltInWindowFunction::RowNumber), + fun: WindowFunctionDefinition::BuiltInWindowFunction(BuiltInWindowFunction::RowNumber), args: Vec::new(), partition_by: Vec::new(), order_by: vec![Expr::Sort(expr::Sort { @@ -32,11 +32,7 @@ impl TransformTrait for Identifier { asc: true, nulls_first: false, })], - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(&self.r#as); diff --git a/vegafusion-runtime/src/transform/timeunit.rs b/vegafusion-runtime/src/transform/timeunit.rs index 524a076ee..477b77fb1 100644 --- a/vegafusion-runtime/src/transform/timeunit.rs +++ b/vegafusion-runtime/src/transform/timeunit.rs @@ -130,7 +130,7 @@ fn timeunit_date_part_tz( // Date if units_set.contains(&TimeUnitUnit::Date) { make_timestamptz_args[2] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("day"), field_col.clone(), lit(&tz_str)], }); @@ -140,7 +140,7 @@ fn timeunit_date_part_tz( // Hour if units_set.contains(&TimeUnitUnit::Hours) { make_timestamptz_args[3] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), field_col.clone(), lit(&tz_str)], }); @@ -150,7 +150,7 @@ fn timeunit_date_part_tz( // Minute if units_set.contains(&TimeUnitUnit::Minutes) { make_timestamptz_args[4] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("minute"), field_col.clone(), lit(&tz_str)], }); @@ -160,7 +160,7 @@ fn timeunit_date_part_tz( // Second if units_set.contains(&TimeUnitUnit::Seconds) { make_timestamptz_args[5] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("second"), field_col, lit(&tz_str)], }); @@ -169,7 +169,7 @@ fn timeunit_date_part_tz( // Construct expression to make timestamp from components let start_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new((*MAKE_UTC_TIMESTAMP).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: make_timestamptz_args, }); @@ -186,11 +186,13 @@ fn to_timestamp_col(field: &str, schema: &DFSchema, default_input_tz: &String) - schema, )?, DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![field_col, lit(default_input_tz)], }), dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(field_col, &DataType::Int64, schema)?], }), dtype => { @@ -216,7 +218,7 @@ fn timeunit_weekday( // Use DATE_PART_TZ to extract the weekday // where Sunday is 0 and Saturday is 6 let weekday0 = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new(DATE_PART_TZ_UDF.clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("dow"), field_col, lit(tz_str)], }); @@ -238,7 +240,7 @@ fn timeunit_weekday( // Construct expression to make timestamp from components let start_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new((*MAKE_UTC_TIMESTAMP).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: make_timestamptz_args, }); @@ -506,7 +508,7 @@ impl TransformTrait for TimeUnit { let tz_str = local_tz.unwrap_or_else(|| "UTC".to_string()); let timeunit_end_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF( Arc::new((*DATE_ADD_TZ_UDF).clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_ADD_TZ_UDF).clone())), args: vec![ lit(&interval.1), lit(interval.0), diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index 5e36db142..f44bff5b0 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -3,7 +3,7 @@ use crate::transform::TransformTrait; use async_trait::async_trait; use datafusion_common::ScalarValue; -use datafusion_expr::{aggregate_function, expr, lit, Expr}; +use datafusion_expr::{aggregate_function, expr, lit, Expr, WindowFrame, WindowFunctionDefinition}; use std::sync::Arc; use vegafusion_core::error::Result; use vegafusion_core::proto::gen::transforms::{ @@ -11,9 +11,7 @@ use vegafusion_core::proto::gen::transforms::{ }; use vegafusion_core::task_graph::task_value::TaskValue; -use datafusion_expr::{ - window_frame, BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits, WindowFunction, -}; +use datafusion_expr::{window_frame, BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits}; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_common::datatypes::to_numeric; @@ -86,15 +84,12 @@ impl TransformTrait for Window { let ignore_peers = self.ignore_peers.unwrap_or(false); - let window_frame = window_frame::WindowFrame { - units: if ignore_peers { - WindowFrameUnits::Rows - } else { - WindowFrameUnits::Groups - }, - start_bound, - end_bound, + let units = if ignore_peers { + WindowFrameUnits::Rows + } else { + WindowFrameUnits::Groups }; + let window_frame = WindowFrame::new_bounds(units, start_bound, end_bound); let schema_df = dataframe.schema_df()?; let window_exprs = self @@ -146,7 +141,10 @@ impl TransformTrait for Window { ))) } }; - (WindowFunction::AggregateFunction(agg_fn), vec![arg]) + ( + WindowFunctionDefinition::AggregateFunction(agg_fn), + vec![arg], + ) } window_transform_op::Op::WindowOp(op) => { let op = WindowOp::try_from(*op).unwrap(); @@ -173,7 +171,10 @@ impl TransformTrait for Window { ))) } }; - (WindowFunction::BuiltInWindowFunction(window_fn), args) + ( + WindowFunctionDefinition::BuiltInWindowFunction(window_fn), + args, + ) } }; diff --git a/vegafusion-runtime/tests/test_vegajs_runtime.rs b/vegafusion-runtime/tests/test_vegajs_runtime.rs index bdbcc85d7..85b792845 100644 --- a/vegafusion-runtime/tests/test_vegajs_runtime.rs +++ b/vegafusion-runtime/tests/test_vegajs_runtime.rs @@ -4,12 +4,12 @@ extern crate lazy_static; mod util; use crate::util::vegajs_runtime::{vegajs_runtime, ExportImage, ExportImageFormat}; +use datafusion_common::utils::array_into_list_array; use datafusion_common::ScalarValue; use serde_json::json; use std::collections::HashMap; use std::fs; use std::sync::Arc; -use datafusion_common::utils::array_into_list_array; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_core::arrow::array::Float64Array; @@ -123,7 +123,9 @@ fn test_evaluate_filter_transform() { // Check extent signal assert_eq!( result_signals, - vec![ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![6.0, 10.0])))))] + vec![ScalarValue::List(Arc::new(array_into_list_array( + Arc::new(Float64Array::from(vec![6.0, 10.0])) + )))] ); let expected_dataset = VegaFusionTable::from_json(&json!([ diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 2b12cfe97..23c67a612 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -11,7 +11,10 @@ use sqlparser::ast::{ }; use datafusion_expr::expr::{BinaryExpr, Case, Cast, Sort}; -use datafusion_expr::{expr, lit, Between, BuiltInWindowFunction, Expr, ExprSchemable, Operator, WindowFrameBound, WindowFrameUnits, WindowFunction}; +use datafusion_expr::{ + expr, lit, Between, BuiltInWindowFunction, Expr, ExprSchemable, Operator, WindowFrameBound, + WindowFrameUnits, WindowFunctionDefinition, +}; use crate::compile::function_arg::ToSqlFunctionArg; use crate::compile::order::ToSqlOrderByExpr; @@ -274,13 +277,17 @@ impl ToSqlExpr for Expr { translate_scalar_function(&fun.name(), &fun.args, dialect, schema) } Expr::AggregateFunction(expr::AggregateFunction { - func_def, - args, - distinct, - .. - }) => { - translate_aggregate_function(func_def.name(), args.as_slice(), *distinct, dialect, schema) - } + func_def, + args, + distinct, + .. + }) => translate_aggregate_function( + func_def.name(), + args.as_slice(), + *distinct, + dialect, + schema, + ), Expr::WindowFunction(expr::WindowFunction { fun, args, @@ -290,10 +297,10 @@ impl ToSqlExpr for Expr { }) => { // Extract function name let (fun_name, supports_frame) = match fun { - WindowFunction::AggregateFunction(agg) => { + WindowFunctionDefinition::AggregateFunction(agg) => { (agg.name().to_string().to_ascii_lowercase(), true) } - WindowFunction::BuiltInWindowFunction(win_fn) => { + WindowFunctionDefinition::BuiltInWindowFunction(win_fn) => { let is_navigation_function = matches!( win_fn, BuiltInWindowFunction::FirstValue @@ -312,8 +319,12 @@ impl ToSqlExpr for Expr { (win_fn.to_string().to_ascii_lowercase(), supports_frame) } - WindowFunction::AggregateUDF(udf) => (udf.name().to_ascii_lowercase(), true), - WindowFunction::WindowUDF(udf) => (udf.name().to_ascii_lowercase(), true), + WindowFunctionDefinition::AggregateUDF(udf) => { + (udf.name().to_ascii_lowercase(), true) + } + WindowFunctionDefinition::WindowUDF(udf) => { + (udf.name().to_ascii_lowercase(), true) + } }; // Handle unordered row_number @@ -604,7 +615,9 @@ mod tests { use arrow::datatypes::DataType; use datafusion_common::DFSchema; use datafusion_expr::expr::Cast; - use datafusion_expr::{expr, lit, Between, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{ + expr, lit, Between, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, + }; use vegafusion_common::column::flat_col; fn schema() -> DFSchema { diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 932a4301a..692529bee 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -5,13 +5,17 @@ use arrow::array::Array; use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; -use datafusion_expr::{expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, Volatility, ScalarFunctionDefinition}; +use datafusion_expr::{ + expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionDefinition, + ScalarFunctionImplementation, ScalarUDF, Signature, Volatility, +}; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr, Ident, ObjectName as SqlObjectName, Value as SqlValue, }; use std::ops::Add; use std::sync::Arc; +use vegafusion_common::data::scalar::ArrayRefHelpers; use vegafusion_common::error::{Result, VegaFusionError}; pub trait ToSqlScalar { @@ -150,7 +154,8 @@ impl ToSqlScalar for ScalarValue { let args = (0..array.len()) .map(|i| { - let sql_expr = ScalarValue::try_from_array(array, i)?.to_sql(dialect)?; + let v = array.value(0).to_scalar_vec()?; + let sql_expr = v[i].to_sql(dialect)?; Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr(sql_expr))) }) .collect::>>()?; diff --git a/vegafusion-sql/src/compile/select.rs b/vegafusion-sql/src/compile/select.rs index 11a60d8d8..2294dc833 100644 --- a/vegafusion-sql/src/compile/select.rs +++ b/vegafusion-sql/src/compile/select.rs @@ -12,7 +12,9 @@ pub trait ToSqlSelectItem { impl ToSqlSelectItem for Expr { fn to_sql_select(&self, dialect: &Dialect, schema: &DFSchema) -> Result { Ok(match self { - Expr::Alias(expr::Alias { expr, name: alias, .. }) => SqlSelectItem::ExprWithAlias { + Expr::Alias(expr::Alias { + expr, name: alias, .. + }) => SqlSelectItem::ExprWithAlias { expr: expr.to_sql(dialect, schema)?, alias: Ident { value: alias.clone(), diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index cdebc21ce..e52cf69c3 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -7,7 +7,12 @@ use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{Column, DFSchema, OwnedTableReference, ScalarValue}; -use datafusion_expr::{abs, expr, is_null, lit, max, min, when, window_function, AggregateFunction, BuiltInWindowFunction, BuiltinScalarFunction, Expr, ExprSchemable, WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunction, ScalarFunctionDefinition}; +use datafusion_expr::expr::AggregateFunctionDefinition; +use datafusion_expr::{ + abs, expr, is_null, lit, max, min, when, AggregateFunction, BuiltInWindowFunction, + BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, WindowFrame, + WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, +}; use sqlparser::ast::{ Cte, Expr as SqlExpr, GroupByExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, @@ -19,7 +24,6 @@ use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::ops::{Add, Div, Sub}; use std::sync::Arc; -use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datatypes::to_numeric; @@ -321,6 +325,7 @@ impl SqlDataFrame { offset: None, fetch: None, locks: Default::default(), + for_clause: None, }; let (projection, table_alias) = if let ValuesMode::ValuesWithSelectColumnAliases { @@ -400,6 +405,7 @@ impl SqlDataFrame { offset: None, fetch: None, locks: Default::default(), + for_clause: None, } } }; @@ -821,7 +827,7 @@ impl SqlDataFrame { // 2. field index let field_order_col = format!("{order_field}_field"); let order_col = Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], @@ -838,11 +844,7 @@ impl SqlDataFrame { nulls_first: true, }), ], - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(order_field); @@ -898,19 +900,14 @@ impl SqlDataFrame { if let StackMode::Zero = mode { // Build window expression - let fun = WindowFunction::AggregateFunction(AggregateFunction::Sum); // Build window function to compute stacked value let window_expr = Expr::WindowFunction(expr::WindowFunction { - fun, + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![numeric_field.clone()], partition_by, order_by: orderby, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(stop_field); @@ -1030,17 +1027,13 @@ impl SqlDataFrame { // Build window function to compute cumulative sum of stack column let cumulative_field = "_cumulative"; - let fun = WindowFunction::AggregateFunction(AggregateFunction::Sum); + let fun = WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum); let window_expr = Expr::WindowFunction(expr::WindowFunction { fun, args: vec![flat_col(stack_col_name)], partition_by, order_by: orderby, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(cumulative_field); @@ -1151,7 +1144,9 @@ impl SqlDataFrame { if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::Coalesce, + ), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1196,7 +1191,9 @@ impl SqlDataFrame { .map(|col_name| { if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::Coalesce, + ), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1214,7 +1211,9 @@ impl SqlDataFrame { .map(|col_name| { let expr = if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::Coalesce, + ), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1337,17 +1336,13 @@ impl SqlDataFrame { }; let order_col = Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], partition_by: vec![], order_by, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(order_field); From 7eddac6481355747e6ff19d9a3c45bb60d1f7923 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Feb 2024 08:15:35 -0500 Subject: [PATCH 05/44] Update to datafusion 35+ on datafusion main --- Cargo.lock | 108 ++++++++++-------- Cargo.toml | 33 ++++-- vegafusion-common/src/data/scalar.rs | 8 +- vegafusion-core/src/task_graph/memory.rs | 11 +- vegafusion-datafusion-udfs/src/udafs/mod.rs | 4 +- .../src/udfs/math/isnan.rs | 50 -------- .../src/udfs/math/mod.rs | 1 - .../data/vl_selection_test.rs | 27 +++-- .../src/expression/compiler/call.rs | 7 +- .../src/expression/compiler/mod.rs | 46 ++++---- vegafusion-sql/src/compile/expr.rs | 25 +++- vegafusion-sql/src/compile/scalar.rs | 2 +- .../src/connection/datafusion_conn.rs | 4 - 13 files changed, 164 insertions(+), 162 deletions(-) delete mode 100644 vegafusion-datafusion-udfs/src/udfs/math/isnan.rs diff --git a/Cargo.lock b/Cargo.lock index ba5ae69a5..1e69d18d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1240,8 +1240,7 @@ dependencies = [ [[package]] name = "datafusion" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "ahash", "arrow", @@ -1257,6 +1256,7 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -1275,7 +1275,7 @@ dependencies = [ "parquet", "pin-project-lite", "rand", - "sqlparser", + "sqlparser 0.43.1", "tempfile", "tokio", "tokio-util", @@ -1288,8 +1288,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "ahash", "arrow", @@ -1303,14 +1302,13 @@ dependencies = [ "object_store", "parquet", "pyo3", - "sqlparser", + "sqlparser 0.43.1", ] [[package]] name = "datafusion-execution" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "arrow", "chrono", @@ -1330,24 +1328,36 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", "paste", - "sqlparser", - "strum 0.25.0", - "strum_macros 0.25.1", + "sqlparser 0.43.1", + "strum 0.26.1", + "strum_macros 0.26.1", +] + +[[package]] +name = "datafusion-functions" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +dependencies = [ + "arrow", + "base64", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hex", + "log", ] [[package]] name = "datafusion-optimizer" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "arrow", "async-trait", @@ -1364,8 +1374,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "ahash", "arrow", @@ -1373,11 +1382,13 @@ dependencies = [ "arrow-buffer", "arrow-ord", "arrow-schema", + "arrow-string", "base64", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-execution", "datafusion-expr", "half 2.2.1", "hashbrown 0.14.0", @@ -1398,8 +1409,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "ahash", "arrow", @@ -1429,8 +1439,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5742f993d1812d6bb3cdc4ce2a0aa99e10f6dc0daa11dd69b0ff57f2d8e7518c" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "arrow", "chrono", @@ -1444,15 +1453,14 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" +source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser", + "sqlparser 0.43.1", ] [[package]] @@ -2341,9 +2349,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libm" @@ -2518,14 +2526,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", - "log", "wasi", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -3780,9 +3787,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -3805,6 +3812,15 @@ name = "sqlparser" version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" +dependencies = [ + "log", +] + +[[package]] +name = "sqlparser" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" dependencies = [ "log", "sqlparser_derive", @@ -3841,11 +3857,11 @@ checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "723b93e8addf9aa965ebe2d11da6d7540fa2283fcea14b3371ff055f7ba13f5f" dependencies = [ - "strum_macros 0.25.1", + "strum_macros 0.26.1", ] [[package]] @@ -3863,9 +3879,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.1" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" +checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" dependencies = [ "heck", "proc-macro2", @@ -4080,9 +4096,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -4090,7 +4106,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -4107,9 +4123,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -4467,7 +4483,7 @@ dependencies = [ "object_store", "pyo3", "serde_json", - "sqlparser", + "sqlparser 0.41.0", "thiserror", ] @@ -4494,7 +4510,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sqlparser", + "sqlparser 0.41.0", "thiserror", "tonic", "tonic-build", @@ -4603,7 +4619,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "sqlparser", + "sqlparser 0.41.0", "tempfile", "test-case", "tokio", @@ -4663,7 +4679,7 @@ dependencies = [ "rstest_reuse", "serde", "serde_json", - "sqlparser", + "sqlparser 0.41.0", "tempfile", "tokio", "toml", diff --git a/Cargo.toml b/Cargo.toml index 897cba9a5..8fffc2706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,19 +18,38 @@ arrow = { version = "50.0.0", default_features = false } sqlparser = { version = "0.41.0" } chrono = { version = "0.4.31", default_features = false } reqwest = { version = "0.11.22", default-features = false } -tokio = { version = "1.32.0" } +tokio = { version = "1.36.0" } pyo3 = { version = "0.20.0" } pythonize = { version = "0.20.0" } prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } object_store = { version= "0.9.0" } -datafusion = { version = "35.0.0" } -datafusion-common = { version = "35.0.0", default_features = false} -datafusion-expr = { version = "35.0.0" } -datafusion-proto = { version = "35.0.0" } -datafusion-physical-expr = { version = "35.0.0" } -datafusion-optimizer = { version = "35.0.0" } +# Post 35.0.0 +[workspace.dependencies.datafusion] +git = "https://github.com/apache/arrow-datafusion.git" +ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" + +[workspace.dependencies.datafusion-common] +git = "https://github.com/apache/arrow-datafusion.git" +default_features = false +ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" + +[workspace.dependencies.datafusion-expr] +git = "https://github.com/apache/arrow-datafusion.git" +ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" + +[workspace.dependencies.datafusion-proto] +git = "https://github.com/apache/arrow-datafusion.git" +ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" + +[workspace.dependencies.datafusion-physical-expr] +git = "https://github.com/apache/arrow-datafusion.git" +ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" + +[workspace.dependencies.datafusion-optimizer] +git = "https://github.com/apache/arrow-datafusion.git" +ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 4f33103df..7ccb047e3 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -4,6 +4,7 @@ use datafusion_common::DataFusionError; use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; +use datafusion_expr::col; #[cfg(feature = "json")] use { @@ -146,10 +147,11 @@ impl ScalarValueHelpers for ScalarValue { Value::Array(values) } - ScalarValue::Struct(Some(v), fields) => { + ScalarValue::Struct(sa) => { let mut pairs: Map = Default::default(); - for (val, field) in v.iter().zip(fields.deref()) { - pairs.insert(field.name().clone(), val.to_json()?); + for (col_ind, field) in sa.fields().deref().iter().enumerate() { + let column = sa.column(col_ind); + pairs.insert(field.name().clone(), ScalarValue::try_from_array(column, 0)?.to_json()?); } Value::Object(pairs) } diff --git a/vegafusion-core/src/task_graph/memory.rs b/vegafusion-core/src/task_graph/memory.rs index 279201e01..afb4dd5dc 100644 --- a/vegafusion-core/src/task_graph/memory.rs +++ b/vegafusion-core/src/task_graph/memory.rs @@ -49,16 +49,11 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { ScalarValue::Binary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), ScalarValue::LargeBinary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), ScalarValue::List(array) => size_of::>() + size_of_list_array(array), - ScalarValue::Struct(Some(values), fields) => { - let values_bytes: usize = size_of::>() - + values - .iter() - .map(|v| size_of::() + inner_size_of_scalar(v)) - .sum::(); - + ScalarValue::Struct(sa) => { + let fields = sa.fields(); let fields_bytes: usize = size_of::>() + fields.iter().map(size_of_field).sum::(); - + let values_bytes: usize = sa.columns().iter().map(|col| col.get_array_memory_size()).sum(); values_bytes + fields_bytes } _ => { diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index 60131ecf2..8afefd60a 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -18,7 +18,7 @@ pub(crate) struct PercentileContAccumulator { } impl Accumulator for PercentileContAccumulator { - fn state(&self) -> Result, DataFusionError> { + fn state(&mut self) -> Result, DataFusionError> { let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type); Ok(vec![ScalarValue::List(state)]) } @@ -60,7 +60,7 @@ impl Accumulator for PercentileContAccumulator { Ok(()) } - fn evaluate(&self) -> Result { + fn evaluate(&mut self) -> Result { if !self.all_values.iter().any(|v| !v.is_null()) { return ScalarValue::try_from(&self.data_type); } diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isnan.rs b/vegafusion-datafusion-udfs/src/udfs/math/isnan.rs deleted file mode 100644 index ed782977c..000000000 --- a/vegafusion-datafusion-udfs/src/udfs/math/isnan.rs +++ /dev/null @@ -1,50 +0,0 @@ -use datafusion_physical_expr::functions::make_scalar_function; -use std::sync::Arc; -use vegafusion_common::{ - arrow::{ - array::{ArrayRef, BooleanArray, Float32Array, Float64Array}, - datatypes::DataType, - }, - datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}, -}; - -/// isNaN(value) -/// -/// Returns true if value is not a number. Same as JavaScript’s Number.isNaN. -/// -/// See: https://vega.github.io/vega/docs/expressions/#isNaN -fn make_is_nan_udf() -> ScalarUDF { - let is_nan = |args: &[ArrayRef]| { - // Signature ensures there is a single argument - let arg = &args[0]; - - let is_nan_array = match arg.data_type() { - DataType::Float32 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_nan()) - } - DataType::Float64 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_nan()) - } - _ => { - // No other type can be NaN - BooleanArray::from(vec![false; arg.len()]) - } - }; - Ok(Arc::new(is_nan_array) as ArrayRef) - }; - let is_nan = make_scalar_function(is_nan); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Boolean))); - ScalarUDF::new( - "isnan", - &Signature::any(1, Volatility::Immutable), - &return_type, - &is_nan, - ) -} - -lazy_static! { - pub static ref ISNAN_UDF: ScalarUDF = make_is_nan_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/math/mod.rs b/vegafusion-datafusion-udfs/src/udfs/math/mod.rs index 4dfcdc32f..cbe4d8772 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/mod.rs @@ -1,2 +1 @@ pub mod isfinite; -pub mod isnan; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index da9d1cd45..ec7d6c047 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -310,7 +310,8 @@ impl TryFrom for FieldSpec { fn try_from(value: ScalarValue) -> Result { match value { - ScalarValue::Struct(Some(values), fields) => { + ScalarValue::Struct(sa) => { + let fields = sa.fields(); let field_names: HashMap<_, _> = fields .iter() .enumerate() @@ -322,8 +323,10 @@ impl TryFrom for FieldSpec { .get("field") .with_context(|| "Missing required property 'field'".to_string())?; - let field = match values.get(*field_index) { - Some(ScalarValue::Utf8(Some(field))) => field.clone(), + let field_value = ScalarValue::try_from_array(sa.column(*field_index), 0)?; + + let field = match field_value { + ScalarValue::Utf8(Some(field)) => field.clone(), _ => { return Err(VegaFusionError::internal( "Expected field to be a string".to_string(), @@ -335,7 +338,8 @@ impl TryFrom for FieldSpec { let typ_index = field_names .get("type") .with_context(|| "Missing required property 'type'".to_string())?; - let typ = SelectionType::try_from(values.get(*typ_index).unwrap().clone())?; + let type_value = ScalarValue::try_from_array(sa.column(*typ_index), 0)?; + let typ = SelectionType::try_from(type_value)?; Ok(Self { field, typ }) } @@ -400,8 +404,9 @@ impl TryFrom for SelectionRow { fn try_from(value: ScalarValue) -> Result { match value { - ScalarValue::Struct(Some(struct_values), struct_fields) => { - let field_names: HashMap<_, _> = struct_fields + ScalarValue::Struct(sa) => { + let fields = sa.fields(); + let field_names: HashMap<_, _> = fields .iter() .enumerate() .map(|(ind, f)| (f.name().clone(), ind)) @@ -411,8 +416,9 @@ impl TryFrom for SelectionRow { let values_index = field_names .get("values") .with_context(|| "Missing required property 'values'".to_string())?; - let values = match struct_values.get(*values_index) { - Some(ScalarValue::List(array)) => array.value(0).to_scalar_vec()?, + let struct_value = ScalarValue::try_from_array(sa.column(*values_index), 0)?; + let values = match struct_value { + ScalarValue::List(array) => array.value(0).to_scalar_vec()?, _ => { return Err(VegaFusionError::internal( "Expected 'values' to be an array".to_string(), @@ -426,8 +432,9 @@ impl TryFrom for SelectionRow { .with_context(|| "Missing required property 'fields'".to_string())?; let mut fields: Vec = Vec::new(); - match struct_values.get(*fields_index) { - Some(ScalarValue::List(array)) => { + let struct_field = ScalarValue::try_from_array(sa.column(*fields_index), 0)?; + match struct_field { + ScalarValue::List(array) => { for el in array.value(0).to_scalar_vec()?.iter() { fields.push(FieldSpec::try_from(el.clone())?) } diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index facdd24e9..6a31429d8 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -22,7 +22,6 @@ use vegafusion_core::proto::gen::expression::{ use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; use vegafusion_datafusion_udfs::udfs::array::span::SPAN_UDF; -use vegafusion_datafusion_udfs::udfs::math::isnan::ISNAN_UDF; use crate::expression::compiler::builtin_functions::data::data_fn::data_fn; use crate::expression::compiler::builtin_functions::data::vl_selection_resolve::vl_selection_resolve_fn; @@ -236,9 +235,9 @@ pub fn default_callables() -> HashMap { callables.insert( "isNaN".to_string(), - VegaFusionCallable::ScalarUDF { - udf: ISNAN_UDF.deref().clone(), - cast: None, + VegaFusionCallable::BuiltinScalarFunction { + function: BuiltinScalarFunction::Isnan, + cast: Some(DataType::Float64), }, ); diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index b39712910..9f8ade104 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -66,7 +66,7 @@ mod test_compile { use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Case, TryCast}; - use datafusion_expr::{concat, expr, lit, BuiltinScalarFunction, Expr, Operator}; + use datafusion_expr::{concat, expr, lit, BuiltinScalarFunction, Expr, Operator, ScalarFunctionDefinition}; use std::collections::HashMap; use std::convert::TryFrom; use std::ops::Deref; @@ -174,7 +174,7 @@ mod test_compile { // unary not should cast numeric value to boolean let expected_expr = !Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(32.0)), @@ -204,7 +204,7 @@ mod test_compile { expr: None, when_then_expr: vec![( Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(32.0)), @@ -259,7 +259,7 @@ mod test_compile { expr: None, when_then_expr: vec![( Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(5.0)), @@ -391,8 +391,8 @@ mod test_compile { let expr = parse("[1, 2, 3]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: vec![lit(1.0), lit(2.0), lit(3.0)], }); println!("expr: {result_expr:?}"); @@ -414,8 +414,8 @@ mod test_compile { let expr = parse("[]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: vec![], }); println!("expr: {result_expr:?}"); @@ -437,19 +437,19 @@ mod test_compile { let expr = parse("[[1, 2], [3, 4], [5, 6]]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: vec![ - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: vec![lit(1.0), lit(2.0)], }), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: vec![lit(3.0), lit(4.0)], }), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), args: vec![lit(5.0), lit(6.0)], }), ], @@ -483,8 +483,8 @@ mod test_compile { let expr = parse("{a: 1, 'two': {three: 3}}").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_object_constructor_udf( + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_object_constructor_udf( &["a".to_string(), "two".to_string()], &[ DataType::Float64, @@ -494,14 +494,14 @@ mod test_compile { true, )])), ], - )), + ))), args: vec![ lit(1.0), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_object_constructor_udf( + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_object_constructor_udf( &["three".to_string()], &[DataType::Float64], - )), + ))), args: vec![lit(3.0)], }), ], @@ -604,7 +604,7 @@ mod test_compile { expr: None, when_then_expr: vec![( Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(32.0)), diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 23c67a612..0e587325a 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -95,6 +95,18 @@ impl ToSqlExpr for Expr { "ArrowAt cannot be converted to SQL".to_string(), )) } + Operator::LikeMatch => return Err(VegaFusionError::internal( + "LikeMatch cannot be converted to SQL".to_string(), + )), + Operator::ILikeMatch => return Err(VegaFusionError::internal( + "ILikeMatch cannot be converted to SQL".to_string(), + )), + Operator::NotLikeMatch => return Err(VegaFusionError::internal( + "NotLikeMatch cannot be converted to SQL".to_string(), + )), + Operator::NotILikeMatch => return Err(VegaFusionError::internal( + "NotILikeMatch cannot be converted to SQL".to_string(), + )) }; Ok(SqlExpr::Nested(Box::new(SqlExpr::BinaryOp { left: Box::new(left.to_sql(dialect, schema)?), @@ -274,7 +286,11 @@ impl ToSqlExpr for Expr { Err(VegaFusionError::internal("Sort cannot be converted to SQL")) } Expr::ScalarFunction(fun) => { - translate_scalar_function(&fun.name(), &fun.args, dialect, schema) + let fun_name = match fun.name().to_ascii_lowercase().as_str() { + "power" => "pow".to_string(), + fun_name => fun_name.to_string() + }; + translate_scalar_function(&fun_name, &fun.args, dialect, schema) } Expr::AggregateFunction(expr::AggregateFunction { func_def, @@ -282,7 +298,7 @@ impl ToSqlExpr for Expr { distinct, .. }) => translate_aggregate_function( - func_def.name(), + &func_def.name().to_ascii_lowercase(), args.as_slice(), *distinct, dialect, @@ -298,7 +314,7 @@ impl ToSqlExpr for Expr { // Extract function name let (fun_name, supports_frame) = match fun { WindowFunctionDefinition::AggregateFunction(agg) => { - (agg.name().to_string().to_ascii_lowercase(), true) + (agg.name().to_ascii_lowercase(), true) } WindowFunctionDefinition::BuiltInWindowFunction(win_fn) => { let is_navigation_function = matches!( @@ -498,6 +514,9 @@ impl ToSqlExpr for Expr { Expr::OuterReferenceColumn(_, _) => Err(VegaFusionError::internal( "OuterReferenceColumn cannot be converted to SQL", )), + Expr::Unnest(_) => Err(VegaFusionError::internal( + "Unnest cannot be converted to SQL", + )), } } } diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 692529bee..6547ce130 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -212,7 +212,7 @@ impl ToSqlScalar for ScalarValue { ScalarValue::IntervalMonthDayNano(_) => Err(VegaFusionError::internal( "IntervalMonthDayNano cannot be converted to SQL", )), - ScalarValue::Struct(_, _) => Err(VegaFusionError::internal( + ScalarValue::Struct(_) => Err(VegaFusionError::internal( "Struct cannot be converted to SQL", )), ScalarValue::Dictionary(_, _) => Err(VegaFusionError::internal( diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index c88562ea4..db0c4bfea 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -48,7 +48,6 @@ use vegafusion_datafusion_udfs::udfs::datetime::to_utc_timestamp::TO_UTC_TIMESTA use vegafusion_datafusion_udfs::udfs::datetime::utc_timestamp_to_epoch::UTC_TIMESTAMP_TO_EPOCH_MS; use vegafusion_datafusion_udfs::udfs::datetime::utc_timestamp_to_str::UTC_TIMESTAMP_TO_STR_UDF; use vegafusion_datafusion_udfs::udfs::math::isfinite::ISFINITE_UDF; -use vegafusion_datafusion_udfs::udfs::math::isnan::ISNAN_UDF; #[cfg(feature = "pyarrow")] use {crate::connection::datafusion_py_datasource::PyDatasource, pyo3::PyObject}; @@ -495,9 +494,6 @@ pub fn make_datafusion_context() -> SessionContext { let ctx = SessionContext::new_with_state(session_state); - // isNan - ctx.register_udf((*ISNAN_UDF).clone()); - // isFinite ctx.register_udf((*ISFINITE_UDF).clone()); From eac6b20cca2f30d78bc1ffd4111a2f1c097ffad1 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Feb 2024 08:35:56 -0500 Subject: [PATCH 06/44] fmt --- vegafusion-common/src/data/scalar.rs | 5 ++- vegafusion-core/src/task_graph/memory.rs | 6 +++- .../src/expression/compiler/mod.rs | 28 +++++++++++---- vegafusion-runtime/tests/util/equality.rs | 29 ++++++++-------- vegafusion-sql/src/compile/expr.rs | 34 ++++++++++++------- 5 files changed, 66 insertions(+), 36 deletions(-) diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 7ccb047e3..03daa018d 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -151,7 +151,10 @@ impl ScalarValueHelpers for ScalarValue { let mut pairs: Map = Default::default(); for (col_ind, field) in sa.fields().deref().iter().enumerate() { let column = sa.column(col_ind); - pairs.insert(field.name().clone(), ScalarValue::try_from_array(column, 0)?.to_json()?); + pairs.insert( + field.name().clone(), + ScalarValue::try_from_array(column, 0)?.to_json()?, + ); } Value::Object(pairs) } diff --git a/vegafusion-core/src/task_graph/memory.rs b/vegafusion-core/src/task_graph/memory.rs index afb4dd5dc..7d1702ab7 100644 --- a/vegafusion-core/src/task_graph/memory.rs +++ b/vegafusion-core/src/task_graph/memory.rs @@ -53,7 +53,11 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { let fields = sa.fields(); let fields_bytes: usize = size_of::>() + fields.iter().map(size_of_field).sum::(); - let values_bytes: usize = sa.columns().iter().map(|col| col.get_array_memory_size()).sum(); + let values_bytes: usize = sa + .columns() + .iter() + .map(|col| col.get_array_memory_size()) + .sum(); values_bytes + fields_bytes } _ => { diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 9f8ade104..c60ba3248 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -66,7 +66,9 @@ mod test_compile { use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Case, TryCast}; - use datafusion_expr::{concat, expr, lit, BuiltinScalarFunction, Expr, Operator, ScalarFunctionDefinition}; + use datafusion_expr::{ + concat, expr, lit, BuiltinScalarFunction, Expr, Operator, ScalarFunctionDefinition, + }; use std::collections::HashMap; use std::convert::TryFrom; use std::ops::Deref; @@ -392,7 +394,9 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + ARRAY_CONSTRUCTOR_UDF.deref().clone(), + )), args: vec![lit(1.0), lit(2.0), lit(3.0)], }); println!("expr: {result_expr:?}"); @@ -415,7 +419,9 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + ARRAY_CONSTRUCTOR_UDF.deref().clone(), + )), args: vec![], }); println!("expr: {result_expr:?}"); @@ -438,18 +444,26 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + ARRAY_CONSTRUCTOR_UDF.deref().clone(), + )), args: vec![ Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + ARRAY_CONSTRUCTOR_UDF.deref().clone(), + )), args: vec![lit(1.0), lit(2.0)], }), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + ARRAY_CONSTRUCTOR_UDF.deref().clone(), + )), args: vec![lit(3.0), lit(4.0)], }), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::UDF(Arc::new( + ARRAY_CONSTRUCTOR_UDF.deref().clone(), + )), args: vec![lit(5.0), lit(6.0)], }), ], diff --git a/vegafusion-runtime/tests/util/equality.rs b/vegafusion-runtime/tests/util/equality.rs index 719d5ac6f..cafa777c0 100644 --- a/vegafusion-runtime/tests/util/equality.rs +++ b/vegafusion-runtime/tests/util/equality.rs @@ -179,30 +179,31 @@ pub fn assert_scalars_almost_equals( index: usize, ) { match (lhs, rhs) { - ( - ScalarValue::Struct(Some(lhs_vals), lhs_fields), - ScalarValue::Struct(Some(rhs_vals), rhs_fields), - ) => { - let lhs_map: HashMap<_, _> = lhs_fields + (ScalarValue::Struct(lhs_sa), ScalarValue::Struct(rhs_sa)) => { + let lhs_map: HashMap<_, _> = lhs_sa + .fields() .iter() - .zip(lhs_vals.iter()) - .filter_map(|(f, val)| { - if DROP_COLS.contains(&f.name().as_str()) { + .enumerate() + .filter_map(|(field_ind, field)| { + if DROP_COLS.contains(&field.name().as_str()) { None } else { - Some((f.name().clone(), val.clone())) + let val = ScalarValue::try_from_array(lhs_sa.column(field_ind), 0).unwrap(); + Some((field.name().clone(), val)) } }) .collect(); - let rhs_map: HashMap<_, _> = rhs_fields + let rhs_map: HashMap<_, _> = rhs_sa + .fields() .iter() - .zip(rhs_vals.iter()) - .filter_map(|(f, val)| { - if DROP_COLS.contains(&f.name().as_str()) { + .enumerate() + .filter_map(|(field_ind, field)| { + if DROP_COLS.contains(&field.name().as_str()) { None } else { - Some((f.name().clone(), val.clone())) + let val = ScalarValue::try_from_array(rhs_sa.column(field_ind), 0).unwrap(); + Some((field.name().clone(), val)) } }) .collect(); diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 0e587325a..4b54cdec6 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -95,18 +95,26 @@ impl ToSqlExpr for Expr { "ArrowAt cannot be converted to SQL".to_string(), )) } - Operator::LikeMatch => return Err(VegaFusionError::internal( - "LikeMatch cannot be converted to SQL".to_string(), - )), - Operator::ILikeMatch => return Err(VegaFusionError::internal( - "ILikeMatch cannot be converted to SQL".to_string(), - )), - Operator::NotLikeMatch => return Err(VegaFusionError::internal( - "NotLikeMatch cannot be converted to SQL".to_string(), - )), - Operator::NotILikeMatch => return Err(VegaFusionError::internal( - "NotILikeMatch cannot be converted to SQL".to_string(), - )) + Operator::LikeMatch => { + return Err(VegaFusionError::internal( + "LikeMatch cannot be converted to SQL".to_string(), + )) + } + Operator::ILikeMatch => { + return Err(VegaFusionError::internal( + "ILikeMatch cannot be converted to SQL".to_string(), + )) + } + Operator::NotLikeMatch => { + return Err(VegaFusionError::internal( + "NotLikeMatch cannot be converted to SQL".to_string(), + )) + } + Operator::NotILikeMatch => { + return Err(VegaFusionError::internal( + "NotILikeMatch cannot be converted to SQL".to_string(), + )) + } }; Ok(SqlExpr::Nested(Box::new(SqlExpr::BinaryOp { left: Box::new(left.to_sql(dialect, schema)?), @@ -288,7 +296,7 @@ impl ToSqlExpr for Expr { Expr::ScalarFunction(fun) => { let fun_name = match fun.name().to_ascii_lowercase().as_str() { "power" => "pow".to_string(), - fun_name => fun_name.to_string() + fun_name => fun_name.to_string(), }; translate_scalar_function(&fun_name, &fun.args, dialect, schema) } From 48bc143bbf1b6db4a6278f352f41a937f3619453 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Feb 2024 08:39:50 -0500 Subject: [PATCH 07/44] Fix warnings --- vegafusion-common/src/data/scalar.rs | 1 - vegafusion-dataframe/src/dataframe.rs | 5 +---- vegafusion-runtime/src/transform/collect.rs | 3 +-- vegafusion-runtime/src/transform/identifier.rs | 6 +----- vegafusion-runtime/src/transform/window.rs | 2 +- vegafusion-sql/src/dataframe/mod.rs | 2 +- 6 files changed, 5 insertions(+), 14 deletions(-) diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 03daa018d..3a13621c1 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -4,7 +4,6 @@ use datafusion_common::DataFusionError; use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; -use datafusion_expr::col; #[cfg(feature = "json")] use { diff --git a/vegafusion-dataframe/src/dataframe.rs b/vegafusion-dataframe/src/dataframe.rs index 6d0519664..94bf86f81 100644 --- a/vegafusion-dataframe/src/dataframe.rs +++ b/vegafusion-dataframe/src/dataframe.rs @@ -4,10 +4,7 @@ use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::{ - expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, WindowFrameUnits, - WindowFunctionDefinition, -}; +use datafusion_expr::{expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFunctionDefinition}; use std::any::Any; use std::fmt::{Display, Formatter}; use std::sync::Arc; diff --git a/vegafusion-runtime/src/transform/collect.rs b/vegafusion-runtime/src/transform/collect.rs index ed43f06ee..8e067a09f 100644 --- a/vegafusion-runtime/src/transform/collect.rs +++ b/vegafusion-runtime/src/transform/collect.rs @@ -8,8 +8,7 @@ use vegafusion_core::error::{Result, ResultWithContext}; use vegafusion_core::proto::gen::transforms::{Collect, SortOrder}; use async_trait::async_trait; -use datafusion_common::scalar::ScalarValue; -use datafusion_expr::{BuiltInWindowFunction, WindowFrame, WindowFrameBound, WindowFrameUnits}; +use datafusion_expr::{BuiltInWindowFunction, WindowFrame}; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_core::task_graph::task_value::TaskValue; diff --git a/vegafusion-runtime/src/transform/identifier.rs b/vegafusion-runtime/src/transform/identifier.rs index 30b356a5d..1398f1e7b 100644 --- a/vegafusion-runtime/src/transform/identifier.rs +++ b/vegafusion-runtime/src/transform/identifier.rs @@ -2,13 +2,9 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; -use datafusion_expr::{ - expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, WindowFrameUnits, - WindowFunctionDefinition, -}; +use datafusion_expr::{expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; -use vegafusion_common::data::scalar::ScalarValue; use vegafusion_common::data::ORDER_COL; use vegafusion_common::error::Result; use vegafusion_core::proto::gen::transforms::Identifier; diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index f44bff5b0..05fc57a0d 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -11,7 +11,7 @@ use vegafusion_core::proto::gen::transforms::{ }; use vegafusion_core::task_graph::task_value::TaskValue; -use datafusion_expr::{window_frame, BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits}; +use datafusion_expr::{BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits}; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_common::datatypes::to_numeric; diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index e52cf69c3..ea0d6083f 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -11,7 +11,7 @@ use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{ abs, expr, is_null, lit, max, min, when, AggregateFunction, BuiltInWindowFunction, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, WindowFrame, - WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, + WindowFunctionDefinition, }; use sqlparser::ast::{ Cte, Expr as SqlExpr, GroupByExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, From 681ff18e4f0b320606f14ac825cf466439b02b80 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Feb 2024 18:29:21 -0500 Subject: [PATCH 08/44] Fix SQL list generation --- vegafusion-sql/src/compile/scalar.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 6547ce130..e6ff44e77 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -151,11 +151,12 @@ impl ToSqlScalar for ScalarValue { value: "make_list".to_string(), quote_style: None, }; - - let args = (0..array.len()) - .map(|i| { - let v = array.value(0).to_scalar_vec()?; - let sql_expr = v[i].to_sql(dialect)?; + let args = array + .value(0) + .to_scalar_vec()? + .into_iter() + .map(|v| { + let sql_expr = v.to_sql(dialect)?; Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr(sql_expr))) }) .collect::>>()?; From 8a208a576c7f40122d0cf40c74a83a7081520fa7 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Feb 2024 08:44:49 -0500 Subject: [PATCH 09/44] Update vegafusion-sql --- vegafusion-sql/src/compile/select.rs | 2 +- vegafusion-sql/tests/expected/select.toml | 48 +++---- vegafusion-sql/tests/test_aggregate.rs | 16 ++- vegafusion-sql/tests/test_select.rs | 160 ++++++++++++---------- vegafusion-sql/tests/test_window.rs | 135 ++++++++---------- 5 files changed, 186 insertions(+), 175 deletions(-) diff --git a/vegafusion-sql/src/compile/select.rs b/vegafusion-sql/src/compile/select.rs index 2294dc833..1517998fa 100644 --- a/vegafusion-sql/src/compile/select.rs +++ b/vegafusion-sql/src/compile/select.rs @@ -51,7 +51,7 @@ mod tests { #[test] pub fn test_select_wildcard() { - let expr = Expr::Wildcard; + let expr = Expr::Wildcard { qualifier: None }; let sql_expr = expr .to_sql_select(&Dialect::datafusion(), &schema()) .unwrap(); diff --git a/vegafusion-sql/tests/expected/select.toml b/vegafusion-sql/tests/expected/select.toml index 3db7689e9..df2139879 100644 --- a/vegafusion-sql/tests/expected/select.toml +++ b/vegafusion-sql/tests/expected/select.toml @@ -134,7 +134,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES ('0'), ('1')) AS "_values" ("a")), values1 AS (SELECT "a", CAST("a" AS TINYINT) AS "i8", CAST("a" AS SMALLINT) AS "u8", CAST("a" AS SMALLINT) AS "i16", CAST("a" AS INT) AS "u16", CAST("a" AS INT) AS "i32", CAST("a" AS BIGINT) AS "u32", CAST("a" AS BIGINT) AS "i64", CAST("a" AS DOUBLE) AS "f32", CAST("a" AS DOUBLE) AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, CAST(`a` AS INT) AS `i8`, CAST(`a` AS INT) AS `u8`, CAST(`a` AS INT) AS `i16`, CAST(`a` AS INT) AS `u16`, CAST(`a` AS INT) AS `i32`, CAST(`a` AS INT) AS `u32`, CAST(`a` AS INT) AS `i64`, CAST(`a` AS float64) AS `f32`, CAST(`a` AS float64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, CAST(`a` AS INT) AS `i8`, CAST(`a` AS INT) AS `u8`, CAST(`a` AS INT) AS `i16`, CAST(`a` AS INT) AS `u16`, CAST(`a` AS INT) AS `i32`, CAST(`a` AS INT) AS `u32`, CAST(`a` AS INT) AS `i64`, CAST(`a` AS FLOAT64) AS `f32`, CAST(`a` AS FLOAT64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT '0' AS "a" UNION ALL SELECT '1' AS "a"), values1 AS (SELECT "a", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TINYINT) ELSE NULL END AS "i8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "u8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "i16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "u16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "i32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "u32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "i64", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS FLOAT) ELSE NULL END AS "f32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS DOUBLE) ELSE NULL END AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST @@ -174,7 +174,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES ('0'), ('1')) AS "_values" ("a")), values1 AS (SELECT "a", TRY_CAST("a" AS TINYINT) AS "i8", TRY_CAST("a" AS SMALLINT) AS "u8", TRY_CAST("a" AS SMALLINT) AS "i16", TRY_CAST("a" AS INT) AS "u16", TRY_CAST("a" AS INT) AS "i32", TRY_CAST("a" AS BIGINT) AS "u32", TRY_CAST("a" AS BIGINT) AS "i64", TRY_CAST("a" AS DOUBLE) AS "f32", TRY_CAST("a" AS DOUBLE) AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, SAFE_CAST(`a` AS INT) AS `i8`, SAFE_CAST(`a` AS INT) AS `u8`, SAFE_CAST(`a` AS INT) AS `i16`, SAFE_CAST(`a` AS INT) AS `u16`, SAFE_CAST(`a` AS INT) AS `i32`, SAFE_CAST(`a` AS INT) AS `u32`, SAFE_CAST(`a` AS INT) AS `i64`, SAFE_CAST(`a` AS float64) AS `f32`, SAFE_CAST(`a` AS float64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, SAFE_CAST(`a` AS INT) AS `i8`, SAFE_CAST(`a` AS INT) AS `u8`, SAFE_CAST(`a` AS INT) AS `i16`, SAFE_CAST(`a` AS INT) AS `u16`, SAFE_CAST(`a` AS INT) AS `i32`, SAFE_CAST(`a` AS INT) AS `u32`, SAFE_CAST(`a` AS INT) AS `i64`, SAFE_CAST(`a` AS FLOAT64) AS `f32`, SAFE_CAST(`a` AS FLOAT64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT '0' AS "a" UNION ALL SELECT '1' AS "a"), values1 AS (SELECT "a", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TINYINT) ELSE NULL END AS "i8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "u8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "i16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "u16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "i32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "u32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "i64", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS FLOAT) ELSE NULL END AS "f32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS DOUBLE) ELSE NULL END AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST @@ -211,34 +211,34 @@ result = ''' [cast_string] athena = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, true AS `c`, 'A' AS `d`, NULL AS `b` UNION ALL SELECT 1 AS `a`, false AS `c`, 'BB' AS `d`, 1.5 AS `b` UNION ALL SELECT NULL AS `a`, NULL AS `c`, 'CCC' AS `d`, 2.25 AS `b`), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, NULL AS `b`, true AS `c`, 'A' AS `d` UNION ALL SELECT 1 AS `a`, 1.5 AS `b`, false AS `c`, 'BB' AS `d` UNION ALL SELECT NULL AS `a`, 2.25 AS `b`, NULL AS `c`, 'CCC' AS `d`), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ databricks = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ datafusion = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS STRING) AS "a", CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS STRING) AS "a", CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ duckdb = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ mysql = """ -WITH values0 AS (SELECT * FROM (VALUES ROW(0, true, 'A', NULL), ROW(1, false, 'BB', 1.5), ROW(NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC +WITH values0 AS (SELECT * FROM (VALUES ROW(0, NULL, true, 'A'), ROW(1, 1.5, false, 'BB'), ROW(NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC """ postgres = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ redshift = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ snowflake = """ -WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "c", "COLUMN3" AS "d", "COLUMN4" AS "b" FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c", "COLUMN4" AS "d" FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC'))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ result = ''' +---+------+-------+-----+ @@ -252,34 +252,34 @@ result = ''' [try_cast_string] athena = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, true AS `c`, 'A' AS `d`, NULL AS `b` UNION ALL SELECT 1 AS `a`, false AS `c`, 'BB' AS `d`, 1.5 AS `b` UNION ALL SELECT NULL AS `a`, NULL AS `c`, 'CCC' AS `d`, 2.25 AS `b`), values1 AS (SELECT SAFE_CAST(`a` AS STRING) AS `a`, SAFE_CAST(`b` AS STRING) AS `b`, SAFE_CAST(`c` AS STRING) AS `c`, SAFE_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, NULL AS `b`, true AS `c`, 'A' AS `d` UNION ALL SELECT 1 AS `a`, 1.5 AS `b`, false AS `c`, 'BB' AS `d` UNION ALL SELECT NULL AS `a`, 2.25 AS `b`, NULL AS `c`, 'CCC' AS `d`), values1 AS (SELECT SAFE_CAST(`a` AS STRING) AS `a`, SAFE_CAST(`b` AS STRING) AS `b`, SAFE_CAST(`c` AS STRING) AS `c`, SAFE_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ databricks = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT TRY_CAST(`a` AS STRING) AS `a`, TRY_CAST(`b` AS STRING) AS `b`, TRY_CAST(`c` AS STRING) AS `c`, TRY_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT TRY_CAST(`a` AS STRING) AS `a`, TRY_CAST(`b` AS STRING) AS `b`, TRY_CAST(`c` AS STRING) AS `c`, TRY_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ datafusion = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT TRY_CAST("a" AS STRING) AS "a", TRY_CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", TRY_CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT TRY_CAST("a" AS STRING) AS "a", TRY_CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", TRY_CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ duckdb = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ mysql = """ -WITH values0 AS (SELECT * FROM (VALUES ROW(0, true, 'A', NULL), ROW(1, false, 'BB', 1.5), ROW(NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC +WITH values0 AS (SELECT * FROM (VALUES ROW(0, NULL, true, 'A'), ROW(1, 1.5, false, 'BB'), ROW(NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC """ postgres = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ redshift = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ snowflake = """ -WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "c", "COLUMN3" AS "d", "COLUMN4" AS "b" FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c", "COLUMN4" AS "d" FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC'))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ result = ''' +---+------+-------+-----+ @@ -296,7 +296,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES (0)) AS "_values" ("a")), values1 AS (SELECT "a", NULL AS "ninf", NULL AS "nan", NULL AS "inf" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`), values1 AS (SELECT `a`, CAST('-inf' AS float64) AS `ninf`, CAST('NaN' AS float64) AS `nan`, CAST('inf' AS float64) AS `inf` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`), values1 AS (SELECT `a`, CAST('-inf' AS FLOAT64) AS `ninf`, CAST('NaN' AS FLOAT64) AS `nan`, CAST('inf' AS FLOAT64) AS `inf` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT 0 AS "a"), values1 AS (SELECT "a", CAST('-inf' AS DOUBLE) AS "ninf", CAST('NaN' AS DOUBLE) AS "nan", CAST('inf' AS DOUBLE) AS "inf" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST @@ -379,7 +379,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES (0, 0.0), (1, -1.5), (2, NULL), (3, NULL), (4, NULL)) AS "_values" ("a", "b")), values1 AS (SELECT "a", "b", "a" IS NOT NULL AS "f1", "b" IS NOT NULL AS "f2" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, 0.0 AS `b` UNION ALL SELECT 1 AS `a`, -1.5 AS `b` UNION ALL SELECT 2 AS `a`, CAST('-inf' AS float64) AS `b` UNION ALL SELECT 3 AS `a`, CAST('inf' AS float64) AS `b` UNION ALL SELECT 4 AS `a`, CAST('NaN' AS float64) AS `b`), values1 AS (SELECT `a`, `b`, CAST(`a` AS STRING) NOT IN (CAST(CAST('-inf' AS float64) AS STRING), CAST(CAST('inf' AS float64) AS STRING), CAST(CAST('NaN' AS float64) AS STRING)) AS `f1`, CAST(`b` AS STRING) NOT IN (CAST(CAST('-inf' AS float64) AS STRING), CAST(CAST('inf' AS float64) AS STRING), CAST(CAST('NaN' AS float64) AS STRING)) AS `f2` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, 0.0 AS `b` UNION ALL SELECT 1 AS `a`, -1.5 AS `b` UNION ALL SELECT 2 AS `a`, CAST('-inf' AS FLOAT64) AS `b` UNION ALL SELECT 3 AS `a`, CAST('inf' AS FLOAT64) AS `b` UNION ALL SELECT 4 AS `a`, CAST('NaN' AS FLOAT64) AS `b`), values1 AS (SELECT `a`, `b`, CAST(`a` AS STRING) NOT IN (CAST(CAST('-inf' AS FLOAT64) AS STRING), CAST(CAST('inf' AS FLOAT64) AS STRING), CAST(CAST('NaN' AS FLOAT64) AS STRING)) AS `f1`, CAST(`b` AS STRING) NOT IN (CAST(CAST('-inf' AS FLOAT64) AS STRING), CAST(CAST('inf' AS FLOAT64) AS STRING), CAST(CAST('NaN' AS FLOAT64) AS STRING)) AS `f2` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT 0 AS "a", 0.0 AS "b" UNION ALL SELECT 1 AS "a", -1.5 AS "b" UNION ALL SELECT 2 AS "a", CAST('-inf' AS DOUBLE) AS "b" UNION ALL SELECT 3 AS "a", CAST('inf' AS DOUBLE) AS "b" UNION ALL SELECT 4 AS "a", CAST('NaN' AS DOUBLE) AS "b"), values1 AS (SELECT "a", "b", isFinite("a") AS "f1", isFinite("b") AS "f2" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST diff --git a/vegafusion-sql/tests/test_aggregate.rs b/vegafusion-sql/tests/test_aggregate.rs index 6c296f08e..3ff95e2eb 100644 --- a/vegafusion-sql/tests/test_aggregate.rs +++ b/vegafusion-sql/tests/test_aggregate.rs @@ -72,6 +72,7 @@ mod test_simple_aggs { #[cfg(test)] mod test_median_agg { use crate::*; + use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -95,7 +96,7 @@ mod test_median_agg { vec![ count(flat_col("a")).alias("count_a"), Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Median, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Median), args: vec![flat_col("a")], distinct: false, filter: None, @@ -122,6 +123,7 @@ mod test_median_agg { #[cfg(test)] mod test_variance_aggs { use crate::*; + use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -144,7 +146,7 @@ mod test_variance_aggs { vec![flat_col("b")], vec![ round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Stddev, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Stddev), args: vec![flat_col("a")], distinct: false, filter: None, @@ -154,7 +156,9 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::StddevPop, + func_def: AggregateFunctionDefinition::BuiltIn( + AggregateFunction::StddevPop, + ), args: vec![flat_col("a")], distinct: false, filter: None, @@ -164,7 +168,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_pop_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Variance, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Variance), args: vec![flat_col("a")], distinct: false, filter: None, @@ -174,7 +178,9 @@ mod test_variance_aggs { .div(lit(100)) .alias("var_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::VariancePop, + func_def: AggregateFunctionDefinition::BuiltIn( + AggregateFunction::VariancePop, + ), args: vec![flat_col("a")], distinct: false, filter: None, diff --git a/vegafusion-sql/tests/test_select.rs b/vegafusion-sql/tests/test_select.rs index c1c52ff70..7132c2b14 100644 --- a/vegafusion-sql/tests/test_select.rs +++ b/vegafusion-sql/tests/test_select.rs @@ -485,12 +485,12 @@ mod test_non_finite_numbers { #[cfg(test)] mod test_scalar_math_functions { use crate::*; - use datafusion_expr::{expr, BuiltinScalarFunction, Expr}; + use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use vegafusion_common::column::flat_col; fn make_scalar_fn1(fun: BuiltinScalarFunction, arg: &str, alias: &str) -> Expr { Expr::ScalarFunction(expr::ScalarFunction { - fun, + func_def: ScalarFunctionDefinition::BuiltIn(fun), args: vec![flat_col(arg)], }) .alias(alias) @@ -498,7 +498,7 @@ mod test_scalar_math_functions { fn make_scalar_fn2(fun: BuiltinScalarFunction, arg1: &str, arg2: &str, alias: &str) -> Expr { Expr::ScalarFunction(expr::ScalarFunction { - fun, + func_def: ScalarFunctionDefinition::BuiltIn(fun), args: vec![flat_col(arg1), flat_col(arg2)], }) .alias(alias) @@ -577,7 +577,7 @@ mod test_is_finite { use arrow::array::{Float64Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, Expr}; + use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; @@ -613,13 +613,13 @@ mod test_is_finite { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ISFINITE_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ISFINITE_UDF.clone())), args: vec![flat_col("a")], }) .alias("f1"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ISFINITE_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ISFINITE_UDF.clone())), args: vec![flat_col("b")], }) .alias("f2"), @@ -650,7 +650,7 @@ mod test_is_finite { #[cfg(test)] mod test_str_to_utc_timestamp { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -674,8 +674,10 @@ mod test_str_to_utc_timestamp { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -712,7 +714,7 @@ mod test_str_to_utc_timestamp { #[cfg(test)] mod test_date_part_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; @@ -737,8 +739,10 @@ mod test_date_part_tz { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -750,18 +754,18 @@ mod test_date_part_tz { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), flat_col("b_utc"), lit("UTC")], }) .alias("hours_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), flat_col("b_utc"), lit("America/Los_Angeles")], }) .alias("hours_la"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), flat_col("b_utc"), lit("America/New_York")], }) .alias("hours_nyc"), @@ -801,7 +805,7 @@ mod test_date_part_tz { #[cfg(test)] mod test_date_trunc_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_trunc_tz::DATE_TRUNC_TZ_UDF; @@ -826,8 +830,10 @@ mod test_date_trunc_tz { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -839,18 +845,18 @@ mod test_date_trunc_tz { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit("day"), flat_col("b_utc"), lit("UTC")], }) .alias("day_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit("day"), flat_col("b_utc"), lit("America/Los_Angeles")], }) .alias("day_la"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit("day"), flat_col("b_utc"), lit("America/New_York")], }) .alias("day_nyc"), @@ -890,7 +896,7 @@ mod test_date_trunc_tz { #[cfg(test)] mod test_make_timestamp_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::make_utc_timestamp::MAKE_UTC_TIMESTAMP; @@ -912,8 +918,8 @@ mod test_make_timestamp_tz { let df_result = df .select(vec![ flat_col("a"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), args: vec![ flat_col("Y"), flat_col("M"), @@ -926,8 +932,8 @@ mod test_make_timestamp_tz { ], }) .alias("ts_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), args: vec![ flat_col("Y"), flat_col("M"), @@ -940,8 +946,8 @@ mod test_make_timestamp_tz { ], }) .alias("ts_nyc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), args: vec![ flat_col("Y"), flat_col("M"), @@ -987,7 +993,7 @@ mod test_make_timestamp_tz { #[cfg(test)] mod test_epoch_to_utc_timestamp { use crate::*; - use datafusion_expr::{expr, Expr}; + use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -1009,8 +1015,10 @@ mod test_epoch_to_utc_timestamp { .select(vec![ flat_col("a"), flat_col("t"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("t")], }) .alias("t_utc"), @@ -1047,7 +1055,7 @@ mod test_epoch_to_utc_timestamp { #[cfg(test)] mod test_utc_timestamp_to_epoch_ms { use crate::*; - use datafusion_expr::{expr, Expr}; + use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -1071,8 +1079,10 @@ mod test_utc_timestamp_to_epoch_ms { .select(vec![ flat_col("a"), flat_col("t"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("t")], }) .alias("t_utc"), @@ -1084,8 +1094,10 @@ mod test_utc_timestamp_to_epoch_ms { flat_col("a"), flat_col("t"), flat_col("t_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(UTC_TIMESTAMP_TO_EPOCH_MS.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + UTC_TIMESTAMP_TO_EPOCH_MS.clone(), + )), args: vec![flat_col("t_utc")], }) .alias("epoch_millis"), @@ -1125,7 +1137,7 @@ mod test_utc_timestamp_to_epoch_ms { #[cfg(test)] mod test_date_add_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -1149,8 +1161,10 @@ mod test_date_add_tz { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("UTC")], }) .alias("b_utc"), @@ -1162,13 +1176,13 @@ mod test_date_add_tz { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_ADD_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_ADD_TZ_UDF.clone())), args: vec![lit("month"), lit(1), flat_col("b_utc"), lit("UTC")], }) .alias("month_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_ADD_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_ADD_TZ_UDF.clone())), args: vec![ lit("month"), lit(1), @@ -1213,7 +1227,7 @@ mod test_date_add_tz { #[cfg(test)] mod test_utc_timestamp_to_str { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -1238,8 +1252,10 @@ mod test_utc_timestamp_to_str { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("UTC")], }) .alias("b_utc"), @@ -1251,13 +1267,17 @@ mod test_utc_timestamp_to_str { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(UTC_TIMESTAMP_TO_STR_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + UTC_TIMESTAMP_TO_STR_UDF.clone(), + )), args: vec![flat_col("b_utc"), lit("UTC")], }) .alias("str_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(UTC_TIMESTAMP_TO_STR_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + UTC_TIMESTAMP_TO_STR_UDF.clone(), + )), args: vec![flat_col("b_utc"), lit("America/New_York")], }) .alias("str_nyc"), @@ -1300,7 +1320,7 @@ mod test_date_to_utc_timestamp { use arrow::array::{ArrayRef, Date32Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_to_utc_timestamp::DATE_TO_UTC_TIMESTAMP_UDF; @@ -1332,8 +1352,10 @@ mod test_date_to_utc_timestamp { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + DATE_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -1373,7 +1395,7 @@ mod test_timestamp_to_utc_timestamp { use arrow::array::{ArrayRef, Int32Array, TimestampMillisecondArray}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::to_utc_timestamp::TO_UTC_TIMESTAMP_UDF; @@ -1404,8 +1426,8 @@ mod test_timestamp_to_utc_timestamp { df.select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(TO_UTC_TIMESTAMP_UDF.clone())), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -1445,7 +1467,7 @@ mod test_timestamp_to_utc_timestamp { #[cfg(test)] mod test_string_ops { use crate::*; - use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr}; + use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -1468,22 +1490,22 @@ mod test_string_ops { flat_col("b"), flat_col("c"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Substr, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Substr), args: vec![flat_col("b"), lit(2), lit(2)], }) .alias("b_substr"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Concat, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Concat), args: vec![flat_col("b"), lit(" "), flat_col("c")], }) .alias("bc_concat"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Upper, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Upper), args: vec![flat_col("b")], }) .alias("b_upper"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Lower, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Lower), args: vec![flat_col("b")], }) .alias("b_lower"), diff --git a/vegafusion-sql/tests/test_window.rs b/vegafusion-sql/tests/test_window.rs index 7792b0ade..524033b01 100644 --- a/vegafusion-sql/tests/test_window.rs +++ b/vegafusion-sql/tests/test_window.rs @@ -4,8 +4,8 @@ extern crate lazy_static; mod utils; use datafusion_common::ScalarValue; use datafusion_expr::{ - col, expr, lit, window_function, AggregateFunction, BuiltInWindowFunction, Expr, WindowFrame, - WindowFrameBound, WindowFrameUnits, + col, expr, lit, AggregateFunction, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, + WindowFrameUnits, }; use rstest::rstest; use rstest_reuse::{self, *}; @@ -17,6 +17,7 @@ use vegafusion_sql::dataframe::SqlDataFrame; #[cfg(test)] mod test_simple_aggs_unbounded { use crate::*; + use datafusion_expr::WindowFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -39,18 +40,14 @@ mod test_simple_aggs_unbounded { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ flat_col("a"), flat_col("b"), flat_col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -58,9 +55,7 @@ mod test_simple_aggs_unbounded { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -68,7 +63,7 @@ mod test_simple_aggs_unbounded { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -76,7 +71,7 @@ mod test_simple_aggs_unbounded { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -84,7 +79,7 @@ mod test_simple_aggs_unbounded { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -115,6 +110,7 @@ mod test_simple_aggs_unbounded { #[cfg(test)] mod test_simple_aggs_unbounded_groups { use crate::*; + use datafusion_expr::WindowFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -137,18 +133,18 @@ mod test_simple_aggs_unbounded_groups { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Groups, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Groups, + WindowFrameBound::Preceding(ScalarValue::Null), + WindowFrameBound::CurrentRow, + ); let df_result = df .select(vec![ flat_col("a"), flat_col("b"), flat_col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -156,9 +152,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -166,7 +160,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -174,7 +168,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -182,7 +176,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -214,6 +208,7 @@ mod test_simple_aggs_unbounded_groups { #[cfg(test)] mod test_simple_aggs_bounded { use crate::*; + use datafusion_expr::WindowFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -236,18 +231,18 @@ mod test_simple_aggs_bounded { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::from(1)), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Rows, + WindowFrameBound::Preceding(ScalarValue::from(1)), + WindowFrameBound::CurrentRow, + ); let df_result = df .select(vec![ flat_col("a"), flat_col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -255,9 +250,7 @@ mod test_simple_aggs_bounded { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -265,7 +258,7 @@ mod test_simple_aggs_bounded { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -273,7 +266,7 @@ mod test_simple_aggs_bounded { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -281,7 +274,7 @@ mod test_simple_aggs_bounded { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -313,6 +306,7 @@ mod test_simple_aggs_bounded { #[cfg(test)] mod test_simple_aggs_bounded_groups { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -334,18 +328,18 @@ mod test_simple_aggs_bounded_groups { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Groups, - start_bound: WindowFrameBound::Preceding(ScalarValue::from(1)), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Groups, + WindowFrameBound::Preceding(ScalarValue::from(1)), + WindowFrameBound::CurrentRow, + ); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -353,9 +347,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -363,7 +355,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -371,7 +363,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -379,7 +371,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -426,6 +418,7 @@ mod test_simple_aggs_bounded_groups { #[cfg(test)] mod test_simple_window_fns { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -447,18 +440,14 @@ mod test_simple_window_fns { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], @@ -468,7 +457,7 @@ mod test_simple_window_fns { }) .alias("row_num"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Rank, ), args: vec![], @@ -478,7 +467,7 @@ mod test_simple_window_fns { }) .alias("rank"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::DenseRank, ), args: vec![], @@ -488,7 +477,7 @@ mod test_simple_window_fns { }) .alias("d_rank"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::FirstValue, ), args: vec![col("b")], @@ -498,7 +487,7 @@ mod test_simple_window_fns { }) .alias("first"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::LastValue, ), args: vec![col("b")], @@ -532,6 +521,7 @@ mod test_simple_window_fns { #[cfg(test)] mod test_advanced_window_fns { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -553,18 +543,14 @@ mod test_advanced_window_fns { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::NthValue, ), args: vec![col("b"), lit(1)], @@ -574,7 +560,7 @@ mod test_advanced_window_fns { }) .alias("nth1"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::CumeDist, ), args: vec![], @@ -584,7 +570,7 @@ mod test_advanced_window_fns { }) .alias("cdist"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Lag, ), args: vec![col("b")], @@ -594,7 +580,7 @@ mod test_advanced_window_fns { }) .alias("lag_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Lead, ), args: vec![col("b")], @@ -604,7 +590,7 @@ mod test_advanced_window_fns { }) .alias("lead_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Ntile, ), args: vec![lit(2)], @@ -638,6 +624,7 @@ mod test_advanced_window_fns { #[cfg(test)] mod test_unordered_row_number { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -659,18 +646,14 @@ mod test_unordered_row_number { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], From 8b18f434f1f70753a041a73e9df0b7db49a88fc4 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Feb 2024 08:51:49 -0500 Subject: [PATCH 10/44] Update vegafusion-sql --- vegafusion-common/src/data/scalar.rs | 2 +- vegafusion-python-embed/src/connection.rs | 2 +- vegafusion-sql/src/connection/datafusion_py_datasource.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 3a13621c1..5addbd93d 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -2,12 +2,12 @@ use crate::error::{Result, VegaFusionError}; use arrow::array::{new_empty_array, Array, ArrayRef, ListArray}; use datafusion_common::DataFusionError; +use arrow::datatypes::DataType; use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; #[cfg(feature = "json")] use { - arrow::datatypes::DataType, serde_json::{Map, Value}, std::ops::Deref, std::sync::Arc, diff --git a/vegafusion-python-embed/src/connection.rs b/vegafusion-python-embed/src/connection.rs index 2a74d5f74..b14fd19cc 100644 --- a/vegafusion-python-embed/src/connection.rs +++ b/vegafusion-python-embed/src/connection.rs @@ -96,7 +96,7 @@ impl Connection for PySqlConnection { let mut tables: HashMap = HashMap::new(); for key in tables_dict.keys() { - let value = tables_dict.get_item(key).unwrap(); + let value = tables_dict.get_item(key)?.unwrap(); let key_string = key.extract::()?; let value_schema = Schema::from_pyarrow(value)?; tables.insert(key_string, value_schema); diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index 821d008cf..695bbf5c0 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -148,7 +148,7 @@ impl ExecutionPlan for PyDatasourceExec { )?)) } - fn statistics(&self) -> Statistics { - Statistics::default() + fn statistics(&self) -> datafusion_common::Result { + Ok(Statistics::new_unknown(self.schema().as_ref())) } } From e2163cb6a4be8ea02223f3d5f8091dd1469ed727 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 17 Feb 2024 07:44:32 -0500 Subject: [PATCH 11/44] Update to datafusion 36.0.0rc1 --- Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8fffc2706..61110e04f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,28 +28,28 @@ object_store = { version= "0.9.0" } # Post 35.0.0 [workspace.dependencies.datafusion] git = "https://github.com/apache/arrow-datafusion.git" -ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" +ref = "441a4356b889edde0802ec1b394f1f10c18283c3" [workspace.dependencies.datafusion-common] git = "https://github.com/apache/arrow-datafusion.git" default_features = false -ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" +ref = "441a4356b889edde0802ec1b394f1f10c18283c3" [workspace.dependencies.datafusion-expr] git = "https://github.com/apache/arrow-datafusion.git" -ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" +ref = "441a4356b889edde0802ec1b394f1f10c18283c3" [workspace.dependencies.datafusion-proto] git = "https://github.com/apache/arrow-datafusion.git" -ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" +ref = "441a4356b889edde0802ec1b394f1f10c18283c3" [workspace.dependencies.datafusion-physical-expr] git = "https://github.com/apache/arrow-datafusion.git" -ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" +ref = "441a4356b889edde0802ec1b394f1f10c18283c3" [workspace.dependencies.datafusion-optimizer] git = "https://github.com/apache/arrow-datafusion.git" -ref = "afb169cd069e0227fb0ef6d39f44d5eabbdc21a2" +ref = "441a4356b889edde0802ec1b394f1f10c18283c3" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization From 58a80aac78a94a5debcd96ab902c9c8d7e7faadb Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 17 Feb 2024 07:47:22 -0500 Subject: [PATCH 12/44] Bump manylinux to 2.17 --- .github/workflows/build_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 3db47b1e4..9a77f5bb6 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -108,7 +108,7 @@ jobs: uses: messense/maturin-action@9357a23898d2fca5ef8da7ac38aaaedab4ef218f # pin@v1.40.2 with: command: build - manylinux: 2014 + manylinux: manylinux_2_17 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip - name: Upload artifacts @@ -140,7 +140,7 @@ jobs: uses: messense/maturin-action@6d52485c3b3044e20b4c8ee6ce6f61e20a7645b0 # pin@v1 with: command: build - manylinux: 2014 + manylinux: manylinux_2_17 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu - name: Upload artifacts From 5251dac913b7777dd1ca93875551baf18f1602d9 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 20 Feb 2024 09:48:29 -0500 Subject: [PATCH 13/44] isnan was moved to datafusion-functions --- Cargo.lock | 76 +++++++++++++------ Cargo.toml | 23 +++--- vegafusion-runtime/Cargo.toml | 3 + .../src/expression/compiler/call.rs | 22 +++++- 4 files changed, 81 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e69d18d8..6be4ecc58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -810,9 +810,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -820,7 +820,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.0", + "windows-targets 0.52.0", ] [[package]] @@ -1239,8 +1239,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b360b692bf6c6d6e6b6dbaf41a3be0020daeceac0f406aed54c75331e50dbb" dependencies = [ "ahash", "arrow", @@ -1257,6 +1258,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -1287,8 +1289,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37f343ccc298f440e25aa38ff82678291a7acc24061c7370ba6c0ff5cc811412" dependencies = [ "ahash", "arrow", @@ -1307,8 +1310,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" dependencies = [ "arrow", "chrono", @@ -1327,8 +1331,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e204d89909e678846b6a95f156aafc1ee5b36cb6c9e37ec2e1449b078a38c818" dependencies = [ "ahash", "arrow", @@ -1342,8 +1347,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98f1c73f7801b2b8ba2297b3ad78ffcf6c1fc6b8171f502987eb9ad5cb244ee7" dependencies = [ "arrow", "base64", @@ -1354,10 +1360,25 @@ dependencies = [ "log", ] +[[package]] +name = "datafusion-functions-array" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d16a0ddf2c991526f6ffe2f47a72c6da0b7354d6c32411dd20631fe2e38937" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "log", + "paste", +] + [[package]] name = "datafusion-optimizer" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ae27e07bf1f04d327be5c2a293470879801ab5535204dc3b16b062fda195496" dependencies = [ "arrow", "async-trait", @@ -1373,8 +1394,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde620cd9ef76a3bca9c754fb68854bd2349c49f55baf97e08001f9e967f6d6b" dependencies = [ "ahash", "arrow", @@ -1408,8 +1430,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a4c75fba9ea99d64b2246cbd2fcae2e6fc973e6616b1015237a616036506dd4" dependencies = [ "ahash", "arrow", @@ -1438,8 +1461,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2292251d5441d88d92a90d1511d5a8c88759a6562ff38ac1711b1587e6bf19c4" dependencies = [ "arrow", "chrono", @@ -1452,8 +1476,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "35.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git#24bbae4529dd4b1470e9bf441f83bebee415f43c" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21474a95c3a62d113599d21b439fa15091b538bac06bd20be0bb2e7d22903c09" dependencies = [ "arrow", "arrow-schema", @@ -2814,9 +2839,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" @@ -4593,6 +4618,7 @@ dependencies = [ "criterion", "datafusion-common", "datafusion-expr", + "datafusion-functions", "datafusion-optimizer", "datafusion-physical-expr", "deterministic-hash", diff --git a/Cargo.toml b/Cargo.toml index 61110e04f..0645f107f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,31 +25,26 @@ prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } object_store = { version= "0.9.0" } -# Post 35.0.0 [workspace.dependencies.datafusion] -git = "https://github.com/apache/arrow-datafusion.git" -ref = "441a4356b889edde0802ec1b394f1f10c18283c3" +version = "36.0.0" [workspace.dependencies.datafusion-common] -git = "https://github.com/apache/arrow-datafusion.git" -default_features = false -ref = "441a4356b889edde0802ec1b394f1f10c18283c3" +version = "36.0.0" [workspace.dependencies.datafusion-expr] -git = "https://github.com/apache/arrow-datafusion.git" -ref = "441a4356b889edde0802ec1b394f1f10c18283c3" +version = "36.0.0" [workspace.dependencies.datafusion-proto] -git = "https://github.com/apache/arrow-datafusion.git" -ref = "441a4356b889edde0802ec1b394f1f10c18283c3" +version = "36.0.0" [workspace.dependencies.datafusion-physical-expr] -git = "https://github.com/apache/arrow-datafusion.git" -ref = "441a4356b889edde0802ec1b394f1f10c18283c3" +version = "36.0.0" [workspace.dependencies.datafusion-optimizer] -git = "https://github.com/apache/arrow-datafusion.git" -ref = "441a4356b889edde0802ec1b394f1f10c18283c3" +version = "36.0.0" + +[workspace.dependencies.datafusion-functions] +version = "36.0.0" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index d72a9cf27..5f8664713 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -100,6 +100,9 @@ workspace = true [dependencies.datafusion-optimizer] workspace = true +[dependencies.datafusion-functions] +workspace = true + [dependencies.tokio] workspace = true features = [ "macros", "rt-multi-thread", "fs",] diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 6a31429d8..42b9472bc 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -7,6 +7,7 @@ use crate::expression::compiler::builtin_functions::type_checking::isvalid::is_v use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, ScalarUDF}; +use datafusion_functions::expr_fn::isnan; use std::collections::HashMap; use std::ops::Deref; use std::str::FromStr; @@ -48,6 +49,7 @@ use crate::task_graph::timezone::RuntimeTzConfig; pub type MacroFn = Arc Result + Send + Sync>; pub type TransformFn = Arc Result + Send + Sync>; +pub type ScalarTransformFn = Arc Expr + Send + Sync>; pub type TzTransformFn = Arc Result + Send + Sync>; pub type DataFn = Arc< @@ -64,6 +66,9 @@ pub enum VegaFusionCallable { /// A function that operates on the compiled arguments and produces a new expression. Transform(TransformFn), + /// An infallible function that operates on a single compiled argument + UnaryTransform(ScalarTransformFn), + /// A function that uses the local timezone to operate on the compiled arguments and /// produces a new expression. LocalTransform(TzTransformFn), @@ -183,6 +188,18 @@ pub fn compile_call( let args = compile_scalar_arguments(node, config, schema, &None)?; callable(&args, schema) } + VegaFusionCallable::UnaryTransform(callable) => { + let mut args = compile_scalar_arguments(node, config, schema, &None)?; + if args.len() != 1 { + Err(VegaFusionError::internal(format!( + "The {} function requires 1 argument. Received {}", + &node.callee, + args.len() + ))) + } else { + Ok(callable(args.pop().unwrap())) + } + } VegaFusionCallable::LocalTransform(callable) => { let args = compile_scalar_arguments(node, config, schema, &None)?; let tz_config = config @@ -235,10 +252,7 @@ pub fn default_callables() -> HashMap { callables.insert( "isNaN".to_string(), - VegaFusionCallable::BuiltinScalarFunction { - function: BuiltinScalarFunction::Isnan, - cast: Some(DataType::Float64), - }, + VegaFusionCallable::UnaryTransform(Arc::new(isnan)), ); callables.insert( From d474eb2733332a41ce8d345ae9048bf9efcbdb0b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 20 Feb 2024 10:33:37 -0500 Subject: [PATCH 14/44] Update test --- vegafusion-common/src/data/json_writer.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index 0372800f4..0f382a957 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -799,8 +799,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"nanos":1542129070011,"micros":1542129070011,"millis":1542129070011,"secs":1542129070000,"name":"a"} -{"nanos":null,"micros":null,"millis":null,"secs":null,"name":"b"} + r#"{"micros":1542129070011,"millis":1542129070011,"name":"a","nanos":1542129070011,"secs":1542129070000} +{"micros":null,"millis":null,"name":"b","nanos":null,"secs":null} "# ); } @@ -888,8 +888,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"time32sec":"00:02:00","time32msec":"00:00:00.120","time64usec":"00:00:00.000120","time64nsec":"00:00:00.000000120","name":"a"} -{"time32sec":null,"time32msec":null,"time64usec":null,"time64nsec":null,"name":"b"} + r#"{"name":"a","time32msec":"00:00:00.120","time32sec":"00:02:00","time64nsec":"00:00:00.000000120","time64usec":"00:00:00.000120"} +{"name":"b","time32msec":null,"time32sec":null,"time64nsec":null,"time64usec":null} "# ); } @@ -931,8 +931,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"duration_sec":"PT120S","duration_msec":"PT0.120S","duration_usec":"PT0.000120S","duration_nsec":"PT0.000000120S","name":"a"} -{"duration_sec":null,"duration_msec":null,"duration_usec":null,"duration_nsec":null,"name":"b"} + r#"{"duration_msec":"PT0.12S","duration_nsec":"PT0.00000012S","duration_sec":"PT120S","duration_usec":"PT0.00012S","name":"a"} +{"duration_msec":null,"duration_nsec":null,"duration_sec":null,"duration_usec":null,"name":"b"} "# ); } From 4783a440a9a15c28cb198717a3c4492c7e0bfb1a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 20 Feb 2024 14:49:39 -0500 Subject: [PATCH 15/44] Revert "Update test" This reverts commit d474eb2733332a41ce8d345ae9048bf9efcbdb0b. --- vegafusion-common/src/data/json_writer.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index 0f382a957..0372800f4 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -799,8 +799,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"micros":1542129070011,"millis":1542129070011,"name":"a","nanos":1542129070011,"secs":1542129070000} -{"micros":null,"millis":null,"name":"b","nanos":null,"secs":null} + r#"{"nanos":1542129070011,"micros":1542129070011,"millis":1542129070011,"secs":1542129070000,"name":"a"} +{"nanos":null,"micros":null,"millis":null,"secs":null,"name":"b"} "# ); } @@ -888,8 +888,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"name":"a","time32msec":"00:00:00.120","time32sec":"00:02:00","time64nsec":"00:00:00.000000120","time64usec":"00:00:00.000120"} -{"name":"b","time32msec":null,"time32sec":null,"time64nsec":null,"time64usec":null} + r#"{"time32sec":"00:02:00","time32msec":"00:00:00.120","time64usec":"00:00:00.000120","time64nsec":"00:00:00.000000120","name":"a"} +{"time32sec":null,"time32msec":null,"time64usec":null,"time64nsec":null,"name":"b"} "# ); } @@ -931,8 +931,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"duration_msec":"PT0.12S","duration_nsec":"PT0.00000012S","duration_sec":"PT120S","duration_usec":"PT0.00012S","name":"a"} -{"duration_msec":null,"duration_nsec":null,"duration_sec":null,"duration_usec":null,"name":"b"} + r#"{"duration_sec":"PT120S","duration_msec":"PT0.120S","duration_usec":"PT0.000120S","duration_nsec":"PT0.000000120S","name":"a"} +{"duration_sec":null,"duration_msec":null,"duration_usec":null,"duration_nsec":null,"name":"b"} "# ); } From 78067352078758408007ea92372a77da120af37b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 20 Feb 2024 14:50:52 -0500 Subject: [PATCH 16/44] preserve order --- vegafusion-common/Cargo.toml | 2 +- vegafusion-common/src/data/json_writer.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vegafusion-common/Cargo.toml b/vegafusion-common/Cargo.toml index c886d2114..b49b723fc 100644 --- a/vegafusion-common/Cargo.toml +++ b/vegafusion-common/Cargo.toml @@ -7,7 +7,7 @@ license = "BSD-3-Clause" [features] pyarrow = [ "pyo3", "arrow/pyarrow",] -json = [ "serde_json", "arrow/json", "chrono",] +json = [ "serde_json/preserve_order", "arrow/json", "chrono",] prettyprint = [ "arrow/prettyprint",] [dependencies] diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index 0372800f4..75bf82012 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -931,7 +931,7 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"duration_sec":"PT120S","duration_msec":"PT0.120S","duration_usec":"PT0.000120S","duration_nsec":"PT0.000000120S","name":"a"} + r#"{"duration_sec":"PT120S","duration_msec":"PT0.12S","duration_usec":"PT0.00012S","duration_nsec":"PT0.00000012S","name":"a"} {"duration_sec":null,"duration_msec":null,"duration_usec":null,"duration_nsec":null,"name":"b"} "# ); From 3445206bdb2dbc9dc205382c3733c0aa82018caf Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 21 Feb 2024 19:31:33 -0500 Subject: [PATCH 17/44] fix cargo warning --- vegafusion-common/Cargo.toml | 1 - vegafusion-core/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/vegafusion-common/Cargo.toml b/vegafusion-common/Cargo.toml index eee35e1e7..d302a747c 100644 --- a/vegafusion-common/Cargo.toml +++ b/vegafusion-common/Cargo.toml @@ -32,7 +32,6 @@ features = [ "ipc",] [dependencies.datafusion-common] workspace = true -default_features = false [dependencies.datafusion-expr] workspace = true diff --git a/vegafusion-core/Cargo.toml b/vegafusion-core/Cargo.toml index 220d3a0b4..5363b1333 100644 --- a/vegafusion-core/Cargo.toml +++ b/vegafusion-core/Cargo.toml @@ -44,7 +44,6 @@ version = "1.6.4" [dependencies.datafusion-common] workspace = true -default_features = false [dependencies.pyo3] workspace = true From e5b84d0962655f38883ca6fc53a6b96c3bd4226d Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 21 Feb 2024 19:35:31 -0500 Subject: [PATCH 18/44] replace custom make_list udf with built-in --- .../src/udfs/array/constructor.rs | 87 ------------------- .../src/udfs/array/mod.rs | 1 - .../src/expression/compiler/array.rs | 5 +- .../src/expression/compiler/mod.rs | 29 ++----- .../src/connection/datafusion_conn.rs | 2 - 5 files changed, 9 insertions(+), 115 deletions(-) delete mode 100644 vegafusion-datafusion-udfs/src/udfs/array/constructor.rs diff --git a/vegafusion-datafusion-udfs/src/udfs/array/constructor.rs b/vegafusion-datafusion-udfs/src/udfs/array/constructor.rs deleted file mode 100644 index cd413a95c..000000000 --- a/vegafusion-datafusion-udfs/src/udfs/array/constructor.rs +++ /dev/null @@ -1,87 +0,0 @@ -use datafusion_physical_expr::functions::make_scalar_function; -use std::sync::Arc; -use std::vec; -use vegafusion_common::arrow::array::{ - Array, ArrayDataBuilder, ArrayRef, BooleanBufferBuilder, Int32Array, ListArray, -}; -use vegafusion_common::arrow::datatypes::{DataType, Field, Float64Type}; -use vegafusion_common::datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}; - -pub fn make_array_constructor_udf() -> ScalarUDF { - let array_constructor = |args: &[ArrayRef]| { - // Signature ensures arguments have the same type and length - let num_args = args.len(); - - // Return empty array as single row, empty array with Float64 data type - if num_args == 0 || num_args == 1 && args[0].data_type() == &DataType::Null { - let empty_row = vec![Some(Vec::>::new())]; - let empty = ListArray::from_iter_primitive::(empty_row); - return Ok(Arc::new(empty) as ArrayRef); - } - - let num_rows = args[0].len(); - let element_dtype = args[0].data_type(); - let array_dtype = DataType::List(Arc::new(Field::new("item", element_dtype.clone(), true))); - - // Concatenate arrays into single flat array (in column-major ordering) - let arrays: Vec<&dyn Array> = args.iter().map(|a| a.as_ref()).collect(); - let concatted = vegafusion_common::arrow::compute::concat(&arrays).unwrap(); - - // Transpose array into row-major ordering and compute offset indices - let mut indices_builder = Int32Array::builder(num_rows * num_args); - let mut flat_valid_builder = BooleanBufferBuilder::new(num_rows * num_args); - let mut offsets_builder = Int32Array::builder(num_rows + 1); - - offsets_builder.append_value(0); - - for r in 0..num_rows { - for (a, arg) in args.iter().enumerate() { - let col_major_idx = (a * num_rows + r) as i32; - indices_builder.append_value(col_major_idx); - flat_valid_builder.append(arg.is_valid(r)); - } - offsets_builder.append_value((num_args * (r + 1)) as i32); - } - - let flat_values = vegafusion_common::arrow::compute::take( - concatted.as_ref(), - &indices_builder.finish(), - Default::default(), - ) - .unwrap(); - let offsets = offsets_builder.finish(); - - // Build ListArray data - let list_array_data = ArrayDataBuilder::new(array_dtype) - .len(num_rows) - .null_bit_buffer(Some(flat_valid_builder.finish().into_inner())) - .add_buffer(offsets.to_data().buffers()[0].clone()) - .add_child_data(flat_values.to_data()) - .build()?; - - Ok(Arc::new(ListArray::from(list_array_data)) as ArrayRef) - }; - let array_constructor = make_scalar_function(array_constructor); - - let return_type: ReturnTypeFunction = Arc::new(move |args| { - Ok(Arc::new(DataType::List(Arc::new(Field::new( - "item", - if args.is_empty() { - DataType::Float64 - } else { - args[0].clone() - }, - true, - ))))) - }); - ScalarUDF::new( - "make_list", - &Signature::variadic_equal(Volatility::Immutable), - &return_type, - &array_constructor, - ) -} - -lazy_static! { - pub static ref ARRAY_CONSTRUCTOR_UDF: ScalarUDF = make_array_constructor_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/array/mod.rs b/vegafusion-datafusion-udfs/src/udfs/array/mod.rs index f0273cc8b..936329d15 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/mod.rs @@ -1,4 +1,3 @@ -pub mod constructor; pub mod indexof; pub mod length; pub mod span; diff --git a/vegafusion-runtime/src/expression/compiler/array.rs b/vegafusion-runtime/src/expression/compiler/array.rs index 6aa527325..94e1614a8 100644 --- a/vegafusion-runtime/src/expression/compiler/array.rs +++ b/vegafusion-runtime/src/expression/compiler/array.rs @@ -1,11 +1,10 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; +use datafusion_expr::{BuiltinScalarFunction, expr, Expr, ScalarFunctionDefinition}; use std::ops::Deref; use std::sync::Arc; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_core::error::Result; use vegafusion_core::proto::gen::expression::ArrayExpression; -use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; pub fn compile_array( node: &ArrayExpression, @@ -18,7 +17,7 @@ pub fn compile_array( elements.push(phys_expr); } Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: elements, })) } diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index c60ba3248..244f7c203 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -77,7 +77,6 @@ mod test_compile { use vegafusion_common::column::flat_col; use vegafusion_core::arrow::array::{new_empty_array, Float64Array}; use vegafusion_core::arrow::datatypes::Fields; - use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; use vegafusion_datafusion_udfs::udfs::object::make_object_constructor_udf; #[test] @@ -394,9 +393,7 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - ARRAY_CONSTRUCTOR_UDF.deref().clone(), - )), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(1.0), lit(2.0), lit(3.0)], }); println!("expr: {result_expr:?}"); @@ -419,19 +416,15 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - ARRAY_CONSTRUCTOR_UDF.deref().clone(), - )), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![], }); println!("expr: {result_expr:?}"); assert_eq!(result_expr, expected_expr); - // Check evaluated value. Empty array is given Float64 data type let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array( - &DataType::Float64, + &DataType::Null, )))); println!("value: {result_value:?}"); @@ -444,26 +437,18 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - ARRAY_CONSTRUCTOR_UDF.deref().clone(), - )), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![ Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - ARRAY_CONSTRUCTOR_UDF.deref().clone(), - )), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(1.0), lit(2.0)], }), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - ARRAY_CONSTRUCTOR_UDF.deref().clone(), - )), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(3.0), lit(4.0)], }), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - ARRAY_CONSTRUCTOR_UDF.deref().clone(), - )), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(5.0), lit(6.0)], }), ], diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index db0c4bfea..64024c221 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -33,7 +33,6 @@ use vegafusion_dataframe::connection::Connection; use vegafusion_dataframe::csv::CsvReadOptions; use vegafusion_dataframe::dataframe::DataFrame; use vegafusion_datafusion_udfs::udafs::{Q1_UDF, Q3_UDF}; -use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; @@ -515,7 +514,6 @@ pub fn make_datafusion_context() -> SessionContext { ctx.register_udf((*FORMAT_TIMESTAMP_UDF).clone()); // list - ctx.register_udf((*ARRAY_CONSTRUCTOR_UDF).clone()); ctx.register_udf((*LENGTH_UDF).clone()); ctx.register_udf((*INDEXOF_UDF).clone()); From c6d12228c655487639c579e4ed05d49adb09b62d Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 21 Feb 2024 20:03:33 -0500 Subject: [PATCH 19/44] use datafusion array and character length functions --- .../src/udfs/array/length.rs | 84 ------------------- .../src/udfs/array/mod.rs | 1 - .../src/udfs/array/span.rs | 23 +++-- .../compiler/builtin_functions/length.rs | 39 +++++++++ .../compiler/builtin_functions/mod.rs | 1 + .../src/expression/compiler/call.rs | 7 +- .../src/expression/compiler/member.rs | 14 ++-- .../src/expression/compiler/utils.rs | 2 +- .../src/connection/datafusion_conn.rs | 2 - vegafusion-sql/src/dialect/mod.rs | 3 +- 10 files changed, 65 insertions(+), 111 deletions(-) delete mode 100644 vegafusion-datafusion-udfs/src/udfs/array/length.rs create mode 100644 vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs diff --git a/vegafusion-datafusion-udfs/src/udfs/array/length.rs b/vegafusion-datafusion-udfs/src/udfs/array/length.rs deleted file mode 100644 index eb0da7603..000000000 --- a/vegafusion-datafusion-udfs/src/udfs/array/length.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::convert::TryFrom; -use std::sync::Arc; -use vegafusion_common::arrow::array::{new_null_array, Array, Float64Array, ListArray}; -use vegafusion_common::arrow::compute::kernels; -use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::datafusion_common::ScalarValue; -use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, -}; - -/// `length(array)` -/// -/// Returns the length of the input array or string -/// -/// See https://vega.github.io/vega/docs/expressions/#length -fn make_length_udf() -> ScalarUDF { - let length_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { - // Signature ensures there is a single argument - let arg = &args[0]; - Ok(match arg { - ColumnarValue::Scalar(value) => { - match value { - ScalarValue::List(arr) => { - let arr = arr.as_any().downcast_ref::().unwrap(); - ColumnarValue::Scalar(ScalarValue::from(arr.value(0).len() as f64)) - } - ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => { - ColumnarValue::Scalar(ScalarValue::from(s.len() as f64)) - } - _ => { - // Scalar f64 null - ColumnarValue::Scalar(ScalarValue::try_from(&DataType::Float64).unwrap()) - } - } - } - ColumnarValue::Array(array) => { - match array.data_type() { - DataType::Utf8 | DataType::LargeUtf8 => { - // String length - ColumnarValue::Array( - kernels::cast( - &kernels::length::length(array.as_ref()).unwrap(), - &DataType::Float64, - ) - .unwrap(), - ) - } - DataType::FixedSizeList(_, n) => { - // Use scalar length - ColumnarValue::Scalar(ScalarValue::from(*n as f64)) - } - DataType::List(_) => { - let array = array.as_any().downcast_ref::().unwrap(); - let offsets = array.value_offsets(); - let mut length_builder = Float64Array::builder(array.len()); - - for i in 0..array.len() { - length_builder.append_value((offsets[i + 1] - offsets[i]) as f64); - } - - ColumnarValue::Array(Arc::new(length_builder.finish())) - } - _ => { - // Array of f64 - ColumnarValue::Array(new_null_array(&DataType::Float64, array.len())) - } - } - } - }) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64))); - ScalarUDF::new( - "len", - &Signature::any(1, Volatility::Immutable), - &return_type, - &length_fn, - ) -} - -lazy_static! { - pub static ref LENGTH_UDF: ScalarUDF = make_length_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/array/mod.rs b/vegafusion-datafusion-udfs/src/udfs/array/mod.rs index 936329d15..72e68adb0 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/mod.rs @@ -1,3 +1,2 @@ pub mod indexof; -pub mod length; pub mod span; diff --git a/vegafusion-datafusion-udfs/src/udfs/array/span.rs b/vegafusion-datafusion-udfs/src/udfs/array/span.rs index 90319efdb..9c3f13227 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/span.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/span.rs @@ -30,12 +30,11 @@ fn make_span_udf() -> ScalarUDF { // Unwrap single element ListArray let arr = arr.as_any().downcast_ref::().unwrap(); let arr = arr.value(0); - match arr.data_type() { - DataType::Float64 => { - if arr.is_empty() { - // Span of empty array is 0 - ColumnarValue::Scalar(ScalarValue::from(0.0)) - } else { + if arr.is_empty() { + ColumnarValue::Scalar(ScalarValue::from(0.0)) + } else { + match arr.data_type() { + DataType::Float64 => { let first = ScalarValue::try_from_array(&arr, 0) .unwrap() .to_f64() @@ -46,12 +45,12 @@ fn make_span_udf() -> ScalarUDF { .unwrap(); ColumnarValue::Scalar(ScalarValue::from(last - first)) } - } - _ => { - return Err(DataFusionError::Internal(format!( - "Unexpected element type for span function: {}", - arr.data_type() - ))) + _ => { + return Err(DataFusionError::Internal(format!( + "Unexpected element type for span function: {}", + arr.data_type() + ))) + } } } } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs new file mode 100644 index 000000000..cd922cd1a --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs @@ -0,0 +1,39 @@ +use datafusion_common::DFSchema; +use datafusion_expr::{BuiltinScalarFunction, Expr, expr, ExprSchemable, lit, ScalarFunctionDefinition}; +use vegafusion_common::arrow::datatypes::DataType; +use vegafusion_common::error::{ResultWithContext, VegaFusionError}; + +pub fn length_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common::error::Result { + if args.len() == 1 { + let arg = args[0].clone(); + let dtype = arg + .get_type(schema) + .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; + + match dtype { + DataType::Utf8 | DataType::LargeUtf8 => Ok(Expr::Cast(expr::Cast { + expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), + args: vec![arg], + })), + data_type: DataType::Float64 + })), + DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => Ok(Expr::Cast(expr::Cast { + expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), + args: vec![arg], + })), + data_type: DataType::Float64 + })), + _ => Err(VegaFusionError::parse(format!( + "length function support array and string arguments. Received argument with type {:?}", + dtype + ))), + } + } else { + Err(VegaFusionError::parse(format!( + "length requires a single argument. Received {} arguments", + args.len() + ))) + } +} \ No newline at end of file diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs index a37c89bda..656529bf0 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs @@ -5,3 +5,4 @@ pub mod format; pub mod math; pub mod type_checking; pub mod type_coercion; +pub mod length; diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 42b9472bc..7ff09d535 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -21,7 +21,6 @@ use vegafusion_core::proto::gen::expression::{ expression, literal, CallExpression, Expression, Literal, }; use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; -use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; use vegafusion_datafusion_udfs::udfs::array::span::SPAN_UDF; use crate::expression::compiler::builtin_functions::data::data_fn::data_fn; @@ -40,6 +39,7 @@ use crate::expression::compiler::builtin_functions::date_time::date_parts::{ use crate::expression::compiler::builtin_functions::date_time::time::time_fn; use crate::expression::compiler::builtin_functions::date_time::time_offset::time_offset_fn; use crate::expression::compiler::builtin_functions::format::format_transform; +use crate::expression::compiler::builtin_functions::length::length_transform; use crate::expression::compiler::builtin_functions::math::isfinite::is_finite_fn; use crate::expression::compiler::builtin_functions::type_checking::isdate::is_date_fn; use crate::expression::compiler::builtin_functions::type_coercion::to_boolean::to_boolean_transform; @@ -272,10 +272,7 @@ pub fn default_callables() -> HashMap { callables.insert( "length".to_string(), - VegaFusionCallable::ScalarUDF { - udf: LENGTH_UDF.deref().clone(), - cast: None, - }, + VegaFusionCallable::Transform (Arc::new(length_transform)), ); callables.insert( diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index f6fc08e4e..a2f978802 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -13,7 +13,6 @@ use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; use vegafusion_common::datatypes::{data_type, is_numeric_datatype}; use vegafusion_core::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_core::proto::gen::expression::{Identifier, MemberExpression}; -use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; use vegafusion_datafusion_udfs::udfs::member::{make_get_element_udf, make_get_object_member_udf}; pub fn compile_member( @@ -86,11 +85,16 @@ pub fn compile_member( } } _ => { - if property_string == "length" { - // Special case to treat foo.length as length(foo) when foo is not an object - // make_length() + if property_string == "length" && matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { + // Special case to treat foo.length as length(foo) on a string Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(LENGTH_UDF.deref().clone())), + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), + args: vec![compiled_object], + }) + } else if property_string == "length" && matches!(dtype, DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)) { + // Special case to treat foo.length as length(foo) on an array + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), args: vec![compiled_object], }) } else if matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { diff --git a/vegafusion-runtime/src/expression/compiler/utils.rs b/vegafusion-runtime/src/expression/compiler/utils.rs index e5905050b..7a5a657c3 100644 --- a/vegafusion-runtime/src/expression/compiler/utils.rs +++ b/vegafusion-runtime/src/expression/compiler/utils.rs @@ -45,7 +45,7 @@ impl ExprHelpers for Expr { Ok(scalar) } else { Err(VegaFusionError::internal(format!( - "Failed to evaluate expression to scalar value: {self}" + "Failed to evaluate expression to scalar value: {self}\nsimplified to: {simplified_expr}\n" ))) } } diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index 64024c221..c05af7582 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -34,7 +34,6 @@ use vegafusion_dataframe::csv::CsvReadOptions; use vegafusion_dataframe::dataframe::DataFrame; use vegafusion_datafusion_udfs::udafs::{Q1_UDF, Q3_UDF}; use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; -use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; use vegafusion_datafusion_udfs::udfs::datetime::date_to_utc_timestamp::DATE_TO_UTC_TIMESTAMP_UDF; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -514,7 +513,6 @@ pub fn make_datafusion_context() -> SessionContext { ctx.register_udf((*FORMAT_TIMESTAMP_UDF).clone()); // list - ctx.register_udf((*LENGTH_UDF).clone()); ctx.register_udf((*INDEXOF_UDF).clone()); // q1/q3 aggregate functions diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index 06026fd4d..3d372f118 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -885,7 +885,8 @@ impl Dialect { "vega_timeunit", "format_timestamp", "make_list", - "len", + "array_length", + "character_length", "indexof", ] .iter() From ffc9547a8258424b5a303060d2420c154ebf4757 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 10:33:06 -0500 Subject: [PATCH 20/44] Move length into array module --- .../builtin_functions/array/length.rs | 39 +++++++++++++++++++ .../compiler/builtin_functions/array/mod.rs | 1 + .../compiler/builtin_functions/mod.rs | 2 +- .../src/expression/compiler/call.rs | 2 +- 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs create mode 100644 vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs new file mode 100644 index 000000000..cd922cd1a --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs @@ -0,0 +1,39 @@ +use datafusion_common::DFSchema; +use datafusion_expr::{BuiltinScalarFunction, Expr, expr, ExprSchemable, lit, ScalarFunctionDefinition}; +use vegafusion_common::arrow::datatypes::DataType; +use vegafusion_common::error::{ResultWithContext, VegaFusionError}; + +pub fn length_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common::error::Result { + if args.len() == 1 { + let arg = args[0].clone(); + let dtype = arg + .get_type(schema) + .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; + + match dtype { + DataType::Utf8 | DataType::LargeUtf8 => Ok(Expr::Cast(expr::Cast { + expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), + args: vec![arg], + })), + data_type: DataType::Float64 + })), + DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => Ok(Expr::Cast(expr::Cast { + expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), + args: vec![arg], + })), + data_type: DataType::Float64 + })), + _ => Err(VegaFusionError::parse(format!( + "length function support array and string arguments. Received argument with type {:?}", + dtype + ))), + } + } else { + Err(VegaFusionError::parse(format!( + "length requires a single argument. Received {} arguments", + args.len() + ))) + } +} \ No newline at end of file diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs new file mode 100644 index 000000000..93c9cc10a --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs @@ -0,0 +1 @@ +pub mod length; \ No newline at end of file diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs index 656529bf0..503bde5ff 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs @@ -5,4 +5,4 @@ pub mod format; pub mod math; pub mod type_checking; pub mod type_coercion; -pub mod length; +pub mod array; diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 7ff09d535..7d6518078 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -39,7 +39,7 @@ use crate::expression::compiler::builtin_functions::date_time::date_parts::{ use crate::expression::compiler::builtin_functions::date_time::time::time_fn; use crate::expression::compiler::builtin_functions::date_time::time_offset::time_offset_fn; use crate::expression::compiler::builtin_functions::format::format_transform; -use crate::expression::compiler::builtin_functions::length::length_transform; +use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::builtin_functions::math::isfinite::is_finite_fn; use crate::expression::compiler::builtin_functions::type_checking::isdate::is_date_fn; use crate::expression::compiler::builtin_functions::type_coercion::to_boolean::to_boolean_transform; From fa3dd14865de7a5a6552f2f3c33d6516a6086806 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 11:07:43 -0500 Subject: [PATCH 21/44] Add draft span implementation with DataFusion built-ins DataFusion doesn't know how to simplify it yet, which breaks our scalar value evaluation. --- .../compiler/builtin_functions/array/mod.rs | 5 +- .../compiler/builtin_functions/array/span.rs | 51 +++++++++++++++++++ .../tests/test_transform_formula.rs | 49 ++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs index 93c9cc10a..687393174 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs @@ -1 +1,4 @@ -pub mod length; \ No newline at end of file +pub mod length; + +// // Span transform not in use yet +// pub mod span; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs new file mode 100644 index 000000000..22fca919a --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs @@ -0,0 +1,51 @@ +use std::ops::Sub; +use datafusion_common::DFSchema; +use datafusion_expr::{BuiltinScalarFunction, Expr, expr, ExprSchemable, lit, ScalarFunctionDefinition, when}; +use vegafusion_common::arrow::datatypes::DataType; +use vegafusion_common::error::{ResultWithContext, VegaFusionError}; + +// Note: I believe this implementation of span, using built-in DataFusion functions, is correct. +// But the DataFusion simplifier doesn't seem to know how to simplify it, which is what we use for +// scalar evaluation, so we can't use it yet. +pub fn span_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common::error::Result { + if args.len() == 1 { + let arg = args[0].clone(); + let dtype = arg + .get_type(schema) + .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; + + match dtype { + DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) => { + if field.data_type().is_numeric() { + let len = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), + args: vec![arg.clone()], + }).cast_to(&DataType::Int32, schema)?; + + let first_el = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayElement), + args: vec![arg.clone(), lit(1)], + }); + + let last_el = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayElement), + args: vec![arg.clone(), len.clone()], + }); + + Ok(when(len.eq(lit(0)), lit(0.0)).otherwise(last_el.sub(first_el))?) + } else { + Ok(lit(0.0)) + } + }, + _ => { + // Span of non-array is zero + Ok(lit(0.0)) + }, + } + } else { + Err(VegaFusionError::parse(format!( + "span requires a single argument. Received {} arguments", + args.len() + ))) + } +} \ No newline at end of file diff --git a/vegafusion-runtime/tests/test_transform_formula.rs b/vegafusion-runtime/tests/test_transform_formula.rs index fc59daff4..9aeb07380 100644 --- a/vegafusion-runtime/tests/test_transform_formula.rs +++ b/vegafusion-runtime/tests/test_transform_formula.rs @@ -238,3 +238,52 @@ fn test_formula_recursive_ternary_doesnt_overflow_stack() { &eq_config, ); } + +#[test] +fn test_formula_span() { + let dataset = vega_json_dataset("penguins"); + + let transform_specs: Vec = serde_json::from_value(json!([ + { + "type": "formula", + "expr": "span([1, 2])", + "as": "span1" + }, + { + "type": "formula", + "expr": "span([])", + "as": "span2" + }, + { + "type": "formula", + "expr": "span(null)", + "as": "span3" + }, + { + "type": "formula", + "expr": "span(23)", + "as": "span4" + }, + { + "type": "formula", + "expr": "span([17])", + "as": "span5" + }, + { + "type": "formula", + "expr": "span([1, 2, 5, 6, 7])", + "as": "span6" + }, + ])) + .unwrap(); + + let comp_config = Default::default(); + let eq_config = Default::default(); + + check_transform_evaluation( + &dataset, + transform_specs.as_slice(), + &comp_config, + &eq_config, + ); +} From b0b9042cf0f2dece7f5e05e13248175e336befe6 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 11:08:06 -0500 Subject: [PATCH 22/44] fmt --- .../src/expression/compiler/array.rs | 2 +- .../compiler/builtin_functions/array/length.rs | 11 ++++++++--- .../expression/compiler/builtin_functions/mod.rs | 2 +- vegafusion-runtime/src/expression/compiler/call.rs | 4 ++-- .../src/expression/compiler/member.rs | 14 +++++++++++--- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/vegafusion-runtime/src/expression/compiler/array.rs b/vegafusion-runtime/src/expression/compiler/array.rs index 94e1614a8..c39bbb9dd 100644 --- a/vegafusion-runtime/src/expression/compiler/array.rs +++ b/vegafusion-runtime/src/expression/compiler/array.rs @@ -1,5 +1,5 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{BuiltinScalarFunction, expr, Expr, ScalarFunctionDefinition}; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use std::ops::Deref; use std::sync::Arc; use vegafusion_common::datafusion_common::DFSchema; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs index cd922cd1a..7ed10a77d 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs @@ -1,9 +1,14 @@ use datafusion_common::DFSchema; -use datafusion_expr::{BuiltinScalarFunction, Expr, expr, ExprSchemable, lit, ScalarFunctionDefinition}; +use datafusion_expr::{ + expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, +}; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::error::{ResultWithContext, VegaFusionError}; -pub fn length_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common::error::Result { +pub fn length_transform( + args: &[Expr], + schema: &DFSchema, +) -> vegafusion_common::error::Result { if args.len() == 1 { let arg = args[0].clone(); let dtype = arg @@ -36,4 +41,4 @@ pub fn length_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common:: args.len() ))) } -} \ No newline at end of file +} diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs index 503bde5ff..212f5503d 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs @@ -1,3 +1,4 @@ +pub mod array; pub mod control_flow; pub mod data; pub mod date_time; @@ -5,4 +6,3 @@ pub mod format; pub mod math; pub mod type_checking; pub mod type_coercion; -pub mod array; diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 7d6518078..ecf15caf1 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -23,6 +23,7 @@ use vegafusion_core::proto::gen::expression::{ use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; use vegafusion_datafusion_udfs::udfs::array::span::SPAN_UDF; +use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::builtin_functions::data::data_fn::data_fn; use crate::expression::compiler::builtin_functions::data::vl_selection_resolve::vl_selection_resolve_fn; use crate::expression::compiler::builtin_functions::data::vl_selection_test::vl_selection_test_fn; @@ -39,7 +40,6 @@ use crate::expression::compiler::builtin_functions::date_time::date_parts::{ use crate::expression::compiler::builtin_functions::date_time::time::time_fn; use crate::expression::compiler::builtin_functions::date_time::time_offset::time_offset_fn; use crate::expression::compiler::builtin_functions::format::format_transform; -use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::builtin_functions::math::isfinite::is_finite_fn; use crate::expression::compiler::builtin_functions::type_checking::isdate::is_date_fn; use crate::expression::compiler::builtin_functions::type_coercion::to_boolean::to_boolean_transform; @@ -272,7 +272,7 @@ pub fn default_callables() -> HashMap { callables.insert( "length".to_string(), - VegaFusionCallable::Transform (Arc::new(length_transform)), + VegaFusionCallable::Transform(Arc::new(length_transform)), ); callables.insert( diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index a2f978802..c3009aaaa 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -85,13 +85,21 @@ pub fn compile_member( } } _ => { - if property_string == "length" && matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { + if property_string == "length" && matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) + { // Special case to treat foo.length as length(foo) on a string Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::CharacterLength, + ), args: vec![compiled_object], }) - } else if property_string == "length" && matches!(dtype, DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)) { + } else if property_string == "length" + && matches!( + dtype, + DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) + ) + { // Special case to treat foo.length as length(foo) on an array Expr::ScalarFunction(expr::ScalarFunction { func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), From 9e2c2ce752ac505daa2d8452d9a22cc953001707 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 11:22:27 -0500 Subject: [PATCH 23/44] Update SpanUDF to use ScalaUDFImpl --- .../src/udfs/array/span.rs | 71 +++++++++++-------- .../src/expression/compiler/call.rs | 4 +- 2 files changed, 45 insertions(+), 30 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/array/span.rs b/vegafusion-datafusion-udfs/src/udfs/array/span.rs index 9c3f13227..a188c331e 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/span.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/span.rs @@ -1,12 +1,11 @@ +use std::any::Any; use std::sync::Arc; -use vegafusion_common::arrow::array::{Array, ListArray}; +use vegafusion_common::arrow::array::Array; +use vegafusion_common::arrow::array::ListArray; use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, -}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; /// `span(array)` /// @@ -14,8 +13,44 @@ use vegafusion_common::datafusion_expr::{ /// or array[array.length-1] - array[0]. /// /// See https://vega.github.io/vega/docs/expressions/#span -fn make_span_udf() -> ScalarUDF { - let span_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct SpanUDF { + signature: Signature, +} + +impl SpanUDF { + pub fn new() -> Self { + let signature = Signature::uniform( + 1, + vec![ + DataType::Float64, // For null + DataType::Null, // For null + DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for SpanUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "span" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // Signature ensures there is a single argument let arg = &args[0]; Ok(match arg { @@ -65,25 +100,5 @@ fn make_span_udf() -> ScalarUDF { todo!("Span on column not yet implemented") } }) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64))); - ScalarUDF::new( - "span", - &Signature::uniform( - 1, - vec![ - DataType::Float64, // For null - DataType::Null, // For null - DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), - ], - Volatility::Immutable, - ), - &return_type, - &span_fn, - ) -} - -lazy_static! { - pub static ref SPAN_UDF: ScalarUDF = make_span_udf(); + } } diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index ecf15caf1..50a10d2a9 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -21,7 +21,7 @@ use vegafusion_core::proto::gen::expression::{ expression, literal, CallExpression, Expression, Literal, }; use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; -use vegafusion_datafusion_udfs::udfs::array::span::SPAN_UDF; +use vegafusion_datafusion_udfs::udfs::array::span::SpanUDF; use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::builtin_functions::data::data_fn::data_fn; @@ -278,7 +278,7 @@ pub fn default_callables() -> HashMap { callables.insert( "span".to_string(), VegaFusionCallable::ScalarUDF { - udf: SPAN_UDF.deref().clone(), + udf: ScalarUDF::from(SpanUDF::new()), cast: None, }, ); From 045d4ce933a5526b8e4ab15e2ffaaa464930b4f3 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 11:29:35 -0500 Subject: [PATCH 24/44] Update IndexOfUDF to use ScalaUDFImpl --- .../src/udfs/array/indexof.rs | 53 ++++++++++++------- .../src/expression/compiler/call.rs | 4 +- .../src/connection/datafusion_conn.rs | 5 +- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs index f5c47aa80..3ddd70fcc 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs @@ -1,4 +1,5 @@ use ordered_float::NotNan; +use std::any::Any; use std::collections::HashMap; use std::sync::Arc; use vegafusion_common::arrow::array::{ @@ -8,10 +9,7 @@ use vegafusion_common::arrow::compute::cast; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, -}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; use vegafusion_common::datatypes::{is_numeric_datatype, is_string_datatype}; /// `indexof(array, value)` @@ -20,8 +18,36 @@ use vegafusion_common::datatypes::{is_numeric_datatype, is_string_datatype}; /// /// See https://vega.github.io/vega/docs/expressions/#indexof /// and https://vega.github.io/vega/docs/expressions/#string_indexof -fn make_indexof_udf() -> ScalarUDF { - let indexof_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct IndexOfUDF { + signature: Signature, +} + +impl IndexOfUDF { + pub fn new() -> Self { + let signature = Signature::any(2, Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for IndexOfUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "indexof" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int32) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // Signature ensures there is a single argument let (array, array_dtype) = match &args[0] { ColumnarValue::Scalar(ScalarValue::List(array)) => { @@ -112,16 +138,7 @@ fn make_indexof_udf() -> ScalarUDF { } } }) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int32))); - - ScalarUDF::new( - "indexof", - &Signature::any(2, Volatility::Immutable), - &return_type, - &indexof_fn, - ) + } } fn build_notnan_index_map(array: &[ScalarValue]) -> HashMap, i32> { @@ -138,7 +155,3 @@ fn build_notnan_index_map(array: &[ScalarValue]) -> HashMap, i32> { }) .collect::>() } - -lazy_static! { - pub static ref INDEXOF_UDF: ScalarUDF = make_indexof_udf(); -} diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 50a10d2a9..6e1cbcc44 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -20,7 +20,7 @@ use vegafusion_common::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_core::proto::gen::expression::{ expression, literal, CallExpression, Expression, Literal, }; -use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; +use vegafusion_datafusion_udfs::udfs::array::indexof::IndexOfUDF; use vegafusion_datafusion_udfs::udfs::array::span::SpanUDF; use crate::expression::compiler::builtin_functions::array::length::length_transform; @@ -286,7 +286,7 @@ pub fn default_callables() -> HashMap { callables.insert( "indexof".to_string(), VegaFusionCallable::ScalarUDF { - udf: INDEXOF_UDF.deref().clone(), + udf: ScalarUDF::from(IndexOfUDF::new()), cast: None, }, ); diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index c05af7582..4dd44bd28 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -14,6 +14,7 @@ use datafusion::optimizer::analyzer::type_coercion::TypeCoercion; use datafusion::prelude::{ CsvReadOptions as DfCsvReadOptions, ParquetReadOptions, SessionConfig, SessionContext, }; +use datafusion_expr::ScalarUDF; use log::Level; use object_store::aws::AmazonS3Builder; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; @@ -33,7 +34,7 @@ use vegafusion_dataframe::connection::Connection; use vegafusion_dataframe::csv::CsvReadOptions; use vegafusion_dataframe::dataframe::DataFrame; use vegafusion_datafusion_udfs::udafs::{Q1_UDF, Q3_UDF}; -use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; +use vegafusion_datafusion_udfs::udfs::array::indexof::IndexOfUDF; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; use vegafusion_datafusion_udfs::udfs::datetime::date_to_utc_timestamp::DATE_TO_UTC_TIMESTAMP_UDF; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -513,7 +514,7 @@ pub fn make_datafusion_context() -> SessionContext { ctx.register_udf((*FORMAT_TIMESTAMP_UDF).clone()); // list - ctx.register_udf((*INDEXOF_UDF).clone()); + ctx.register_udf(ScalarUDF::from(IndexOfUDF::new())); // q1/q3 aggregate functions ctx.register_udaf((*Q1_UDF).clone()); From 2e565387dfdf108c00bb3551cf7ec7d68f448f23 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 11:53:06 -0500 Subject: [PATCH 25/44] Update IsFiniteUDF to use ScalaUDFImpl --- .../src/udfs/math/isfinite.rs | 94 ++++++++++++------- vegafusion-datafusion-udfs/src/udfs/mod.rs | 1 + vegafusion-datafusion-udfs/src/udfs/util.rs | 38 ++++++++ .../builtin_functions/math/isfinite.rs | 9 +- .../tests/test_expression_evaluation.rs | 4 +- .../src/connection/datafusion_conn.rs | 4 +- 6 files changed, 109 insertions(+), 41 deletions(-) create mode 100644 vegafusion-datafusion-udfs/src/udfs/util.rs diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs index 631a90220..00729d4cd 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs @@ -1,46 +1,72 @@ -use datafusion_physical_expr::functions::make_scalar_function; +use std::any::Any; use std::sync::Arc; -use vegafusion_common::arrow::array::{ArrayRef, BooleanArray, Float32Array, Float64Array}; +use vegafusion_common::arrow::array::{BooleanArray, Float32Array, Float64Array}; use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}; +use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; /// `isFinite(value)` /// /// Returns true if value is a finite number. /// /// See: https://vega.github.io/vega/docs/expressions/#isFinite -fn make_is_finite_udf() -> ScalarUDF { - let is_finite = |args: &[ArrayRef]| { - // Signature ensures there is a single argument - let arg = &args[0]; - - let is_finite_array = match arg.data_type() { - DataType::Float32 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_finite()) - } - DataType::Float64 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_finite()) - } - _ => { - // No other type can be non-finite - BooleanArray::from(vec![true; arg.len()]) - } - }; - Ok(Arc::new(is_finite_array) as ArrayRef) - }; - let is_finite = make_scalar_function(is_finite); +#[derive(Debug, Clone)] +pub struct IsFiniteUDF { + signature: Signature, +} - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Boolean))); - ScalarUDF::new( - "isfinite", - &Signature::any(1, Volatility::Immutable), - &return_type, - &is_finite, - ) +impl IsFiniteUDF { + pub fn new() -> Self { + let signature = Signature::any(1, Volatility::Immutable); + Self { signature } + } } -lazy_static! { - pub static ref ISFINITE_UDF: ScalarUDF = make_is_finite_udf(); +impl ScalarUDFImpl for IsFiniteUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "isfinite" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Boolean) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { + // Signature ensures there is a single argument + match &args[0] { + ColumnarValue::Scalar(arg) => { + let res = match arg { + ScalarValue::Float32(Some(v)) => v.is_finite(), + ScalarValue::Float64(Some(v)) => v.is_finite(), + _ => true, + }; + Ok(ColumnarValue::Scalar(ScalarValue::from(res))) + } + ColumnarValue::Array(arg) => { + let is_finite_array = match arg.data_type() { + DataType::Float32 => { + let array = arg.as_any().downcast_ref::().unwrap(); + BooleanArray::from_unary(array, |a| a.is_finite()) + } + DataType::Float64 => { + let array = arg.as_any().downcast_ref::().unwrap(); + BooleanArray::from_unary(array, |a| a.is_finite()) + } + _ => { + // No other type can be non-finite + BooleanArray::from(vec![true; arg.len()]) + } + }; + Ok(ColumnarValue::Array(Arc::new(is_finite_array))) + } + } + } } diff --git a/vegafusion-datafusion-udfs/src/udfs/mod.rs b/vegafusion-datafusion-udfs/src/udfs/mod.rs index 6c7bfe80d..40f35087d 100644 --- a/vegafusion-datafusion-udfs/src/udfs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/mod.rs @@ -3,3 +3,4 @@ pub mod datetime; pub mod math; pub mod member; pub mod object; +pub mod util; diff --git a/vegafusion-datafusion-udfs/src/udfs/util.rs b/vegafusion-datafusion-udfs/src/udfs/util.rs new file mode 100644 index 000000000..9d3a745ea --- /dev/null +++ b/vegafusion-datafusion-udfs/src/udfs/util.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; +use vegafusion_common::arrow::array::ArrayRef; +use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarFunctionImplementation}; + +pub fn make_scalar_function(inner: F) -> ScalarFunctionImplementation +where + F: Fn(&[ArrayRef]) -> Result + Sync + Send + 'static, +{ + Arc::new(move |args: &[ColumnarValue]| { + // first, identify if any of the arguments is an Array. If yes, store its `len`, + // as any scalar will need to be converted to an array of len `len`. + let len = args + .iter() + .fold(Option::::None, |acc, arg| match arg { + ColumnarValue::Scalar(_) => acc, + ColumnarValue::Array(a) => Some(a.len()), + }); + + let is_scalar = len.is_none(); + + let inferred_length = len.unwrap_or(1); + let args = args + .iter() + .map(|arg| arg.clone().into_array(inferred_length)) + .collect::, DataFusionError>>()?; + + let result = (inner)(&args); + + if is_scalar { + // If all inputs are scalar, keeps output as scalar + let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); + result.map(ColumnarValue::Scalar) + } else { + result.map(ColumnarValue::Array) + } + }) +} diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs index 5c9f763ad..d01eec4ac 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs @@ -1,10 +1,10 @@ -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition, ScalarUDF}; use std::ops::Deref; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_common::error::{Result, ResultWithContext, VegaFusionError}; -use vegafusion_datafusion_udfs::udfs::math::isfinite::ISFINITE_UDF; +use vegafusion_datafusion_udfs::udfs::math::isfinite::IsFiniteUDF; /// `isFinite(value)` /// @@ -20,9 +20,10 @@ pub fn is_finite_fn(args: &[Expr], schema: &DFSchema) -> Result { Ok(match dtype { DataType::Float16 | DataType::Float32 | DataType::Float64 => { - let is_finite_udf = ISFINITE_UDF.deref().clone(); Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(is_finite_udf)), + func_def: ScalarFunctionDefinition::UDF(Arc::new(ScalarUDF::from( + IsFiniteUDF::new(), + ))), args: vec![arg], }) } diff --git a/vegafusion-runtime/tests/test_expression_evaluation.rs b/vegafusion-runtime/tests/test_expression_evaluation.rs index 8ba4ef8ec..7dc599206 100644 --- a/vegafusion-runtime/tests/test_expression_evaluation.rs +++ b/vegafusion-runtime/tests/test_expression_evaluation.rs @@ -289,7 +289,9 @@ mod test_math_functions { case("sqrt(4)"), case("tan(1)"), case("log(2.718281828)"), - case("pow(2, 3)") + case("pow(2, 3)"), + case("isFinite(2)"), + case("isFinite(NaN)") )] fn test(expr: &str) { check_scalar_evaluation(expr, &config_a()) diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index 4dd44bd28..019a8ac03 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -46,7 +46,7 @@ use vegafusion_datafusion_udfs::udfs::datetime::timeunit::TIMEUNIT_START_UDF; use vegafusion_datafusion_udfs::udfs::datetime::to_utc_timestamp::TO_UTC_TIMESTAMP_UDF; use vegafusion_datafusion_udfs::udfs::datetime::utc_timestamp_to_epoch::UTC_TIMESTAMP_TO_EPOCH_MS; use vegafusion_datafusion_udfs::udfs::datetime::utc_timestamp_to_str::UTC_TIMESTAMP_TO_STR_UDF; -use vegafusion_datafusion_udfs::udfs::math::isfinite::ISFINITE_UDF; +use vegafusion_datafusion_udfs::udfs::math::isfinite::IsFiniteUDF; #[cfg(feature = "pyarrow")] use {crate::connection::datafusion_py_datasource::PyDatasource, pyo3::PyObject}; @@ -494,7 +494,7 @@ pub fn make_datafusion_context() -> SessionContext { let ctx = SessionContext::new_with_state(session_state); // isFinite - ctx.register_udf((*ISFINITE_UDF).clone()); + ctx.register_udf(ScalarUDF::from(IsFiniteUDF::new())); // datetime ctx.register_udf((*DATE_PART_TZ_UDF).clone()); From 00277b8fbabedb97544ce2004b3bf23df2844ca7 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 13:32:10 -0500 Subject: [PATCH 26/44] delete old length --- .../compiler/builtin_functions/length.rs | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs deleted file mode 100644 index cd922cd1a..000000000 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/length.rs +++ /dev/null @@ -1,39 +0,0 @@ -use datafusion_common::DFSchema; -use datafusion_expr::{BuiltinScalarFunction, Expr, expr, ExprSchemable, lit, ScalarFunctionDefinition}; -use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::error::{ResultWithContext, VegaFusionError}; - -pub fn length_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common::error::Result { - if args.len() == 1 { - let arg = args[0].clone(); - let dtype = arg - .get_type(schema) - .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; - - match dtype { - DataType::Utf8 | DataType::LargeUtf8 => Ok(Expr::Cast(expr::Cast { - expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), - args: vec![arg], - })), - data_type: DataType::Float64 - })), - DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => Ok(Expr::Cast(expr::Cast { - expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), - args: vec![arg], - })), - data_type: DataType::Float64 - })), - _ => Err(VegaFusionError::parse(format!( - "length function support array and string arguments. Received argument with type {:?}", - dtype - ))), - } - } else { - Err(VegaFusionError::parse(format!( - "length requires a single argument. Received {} arguments", - args.len() - ))) - } -} \ No newline at end of file From 0143c39d29d89b9760e8275a7f77d60c9c83cd9c Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 13:33:48 -0500 Subject: [PATCH 27/44] Update member accessor UDFs to use ScalarUDFImpl --- .../src/udfs/member/mod.rs | 220 ++++++++++++------ 1 file changed, 154 insertions(+), 66 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs index b7e912878..938c788b8 100644 --- a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs @@ -1,62 +1,164 @@ -use datafusion_physical_expr::functions::make_scalar_function; -use std::sync::Arc; -use vegafusion_common::arrow::array::{ - new_null_array, Array, ArrayRef, Int32Array, ListArray, StructArray, -}; +use std::any::Any; +use vegafusion_common::arrow::array::{new_null_array, Array, Int32Array, ListArray, StructArray}; use vegafusion_common::arrow::compute::kernels; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; use vegafusion_common::error::{Result, VegaFusionError}; +#[derive(Debug, Clone)] +pub struct GetObjectMemberUDF { + field_type: DataType, + field_index: usize, + property_name: String, + signature: Signature, + name: String, +} + +impl GetObjectMemberUDF { + pub fn new(property_name: String, object_type: DataType) -> Result { + let signature = Signature::exact(vec![object_type.clone()], Volatility::Immutable); + let name = format!("get[{property_name}]"); + + let (field_index, field_type) = if let DataType::Struct(fields) = &object_type { + match fields + .iter() + .enumerate() + .find(|(_i, f)| f.name() == &property_name) + { + Some((field_index, field)) => (field_index, field.data_type().clone()), + None => { + return Err(VegaFusionError::compilation(format!( + "No object property named {property_name}" + ))) + } + } + } else { + return Err(VegaFusionError::compilation( + "Target of object property access is not a Struct type", + )); + }; + + Ok(Self { + field_type, + field_index, + property_name, + signature, + name, + }) + } +} + +impl ScalarUDFImpl for GetObjectMemberUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> std::result::Result { + Ok(self.field_type.clone()) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> std::result::Result { + // Signature ensures there is a single argument + let arg = &args[0]; + + match arg { + ColumnarValue::Scalar(ScalarValue::Struct(arg)) => { + let c = arg.column(self.field_index); + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(c, 0)?)) + } + ColumnarValue::Array(object) => { + let struct_array = object.as_any().downcast_ref::().unwrap(); + Ok(ColumnarValue::Array( + struct_array.column(self.field_index).clone(), + )) + } + _ => { + return Err(DataFusionError::Internal(format!( + "Unexpected object type for member access: {:?}", + arg.data_type() + ))); + } + } + } +} + pub fn make_get_object_member_udf( object_type: &DataType, property_name: &str, ) -> Result { - let (field_index, field_type) = if let DataType::Struct(fields) = object_type { - match fields - .iter() - .enumerate() - .find(|(_i, f)| f.name() == property_name) - { - Some((field_index, field)) => (field_index, field.data_type().clone()), - None => { - return Err(VegaFusionError::compilation(format!( - "No object property named {property_name}" - ))) - } - } - } else { - return Err(VegaFusionError::compilation( - "Target of object property access is not a Struct type", - )); - }; + Ok(ScalarUDF::from(GetObjectMemberUDF::new( + property_name.to_string(), + object_type.clone(), + )?)) +} - let get = move |args: &[ArrayRef]| { - // Signature ensures there is a single argument - let object = &args[0]; - - let struct_array = object.as_any().downcast_ref::().unwrap(); - Ok(struct_array.column(field_index).clone()) - }; - let get = make_scalar_function(get); - - let return_type: ReturnTypeFunction = - Arc::new(move |_dtype: &[DataType]| Ok(Arc::new(field_type.clone()))); - - Ok(ScalarUDF::new( - &format!("get[{property_name}]"), - &Signature::exact(vec![object_type.clone()], Volatility::Immutable), - &return_type, - &get, - )) +#[derive(Debug, Clone)] +pub struct ArrayElementUDF { + name: String, + signature: Signature, + index: i32, } -pub fn make_get_element_udf(index: i32) -> ScalarUDF { - let get_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +impl ArrayElementUDF { + pub fn new(index: i32) -> Self { + let signature = Signature::any(1, Volatility::Immutable); + Self { + name: format!("get[{index}]"), + signature, + index, + } + } +} + +impl ScalarUDFImpl for ArrayElementUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + arg_types: &[DataType], + ) -> std::result::Result { + let new_dtype = match &arg_types[0] { + DataType::List(field) => field.data_type().clone(), + dtype => { + return Err(DataFusionError::Internal(format!( + "Unsupported datatype for get index {dtype:?}" + ))) + } + }; + Ok(new_dtype) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> std::result::Result { // Signature ensures there is a single argument let arg = &args[0]; Ok(match arg { @@ -64,7 +166,7 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { match value { ScalarValue::List(arr) => { let arr = arr.as_any().downcast_ref::().unwrap(); - match ScalarValue::try_from_array(&arr.value(0), index as usize) { + match ScalarValue::try_from_array(&arr.value(0), self.index as usize) { Ok(element) => { // Scalar element of list ColumnarValue::Scalar(element.clone()) @@ -94,8 +196,8 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { for i in 0..array.len() { let el_start = offsets[i]; let el_stop = offsets[i + 1]; - if el_start + index < el_stop { - take_index_builder.append_value(el_start + index); + if el_start + self.index < el_stop { + take_index_builder.append_value(el_start + self.index); } else { take_index_builder.append_null(); } @@ -114,23 +216,9 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { } } }) - }); + } +} - let return_type: ReturnTypeFunction = Arc::new(move |dtype| { - let new_dtype = match &dtype[0] { - DataType::List(field) => field.data_type().clone(), - dtype => { - return Err(DataFusionError::Internal(format!( - "Unsupported datatype for get index {dtype:?}" - ))) - } - }; - Ok(Arc::new(new_dtype)) - }); - ScalarUDF::new( - &format!("get[{index}]"), - &Signature::any(1, Volatility::Immutable), - &return_type, - &get_fn, - ) +pub fn make_get_element_udf(index: i32) -> ScalarUDF { + ScalarUDF::from(ArrayElementUDF::new(index)) } From dd44fd358befb7255377c5e02df31106f0291a6a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 13:44:09 -0500 Subject: [PATCH 28/44] Update object constructor to use ScalarUDFImpl --- .../src/udfs/object/mod.rs | 95 +++++++++++++------ 1 file changed, 66 insertions(+), 29 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/object/mod.rs b/vegafusion-datafusion-udfs/src/udfs/object/mod.rs index 8924d568b..3ef227b82 100644 --- a/vegafusion-datafusion-udfs/src/udfs/object/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/object/mod.rs @@ -1,42 +1,79 @@ -use datafusion_physical_expr::functions::make_scalar_function; +use std::any::Any; +use crate::udfs::util::make_scalar_function; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, StructArray}; use vegafusion_common::arrow::datatypes::{DataType, Field, FieldRef, Fields}; -use vegafusion_common::datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}; +use vegafusion_common::datafusion_common::DataFusionError; +use vegafusion_common::datafusion_expr::{ + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, +}; -pub fn make_object_constructor_udf(keys: &[String], value_types: &[DataType]) -> ScalarUDF { - // Build fields vector - let fields: Vec<_> = keys - .iter() - .zip(value_types.iter()) - .map(|(k, dtype)| Field::new(k, dtype.clone(), true)) - .collect(); +#[derive(Debug, Clone)] +struct ObjectConstructorUDF { + signature: Signature, + fields: Vec, + struct_dtype: DataType, + name: String, +} - let struct_dtype = DataType::Struct(Fields::from(fields.clone())); +impl ObjectConstructorUDF { + pub fn new(keys: &[String], value_types: &[DataType]) -> Self { + // Build fields vector + let fields: Vec<_> = keys + .iter() + .zip(value_types.iter()) + .map(|(k, dtype)| Field::new(k, dtype.clone(), true)) + .collect(); + let struct_dtype = DataType::Struct(Fields::from(fields.clone())); - let object_constructor = move |args: &[ArrayRef]| { - let pairs: Vec<_> = fields + // Build name + let name_csv: Vec<_> = keys .iter() - .zip(args.iter()) - .map(|(f, v)| (FieldRef::new(f.clone()), v.clone())) + .zip(value_types) + .map(|(k, dtype)| format!("{k}: {dtype}")) .collect(); - Ok(Arc::new(StructArray::from(pairs)) as ArrayRef) - }; + let name = format!("object{{{}}}", name_csv.join(",")); + Self { + signature: Signature::any(keys.len(), Volatility::Immutable), + fields, + struct_dtype, + name, + } + } +} - let object_constructor = make_scalar_function(object_constructor); +impl ScalarUDFImpl for ObjectConstructorUDF { + fn as_any(&self) -> &dyn Any { + self + } - let return_type: ReturnTypeFunction = Arc::new(move |_args| Ok(Arc::new(struct_dtype.clone()))); + fn name(&self) -> &str { + self.name.as_str() + } - let name_csv: Vec<_> = keys - .iter() - .zip(value_types) - .map(|(k, dtype)| format!("{k}: {dtype}")) - .collect(); + fn signature(&self) -> &Signature { + &self.signature + } - ScalarUDF::new( - &format!("object{{{}}}", name_csv.join(",")), - &Signature::any(keys.len(), Volatility::Immutable), - &return_type, - &object_constructor, - ) + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(self.struct_dtype.clone()) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { + let fields = self.fields.clone(); + let object_constructor = move |args: &[ArrayRef]| { + let pairs: Vec<_> = fields + .iter() + .zip(args.iter()) + .map(|(f, v)| (FieldRef::new(f.clone()), v.clone())) + .collect(); + Ok(Arc::new(StructArray::from(pairs)) as ArrayRef) + }; + let object_constructor = make_scalar_function(object_constructor); + object_constructor(args) + } +} + +pub fn make_object_constructor_udf(keys: &[String], value_types: &[DataType]) -> ScalarUDF { + ScalarUDF::from(ObjectConstructorUDF::new(keys, value_types)) } From d1e56723578203439191007aac39590a43ff4345 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 13:47:45 -0500 Subject: [PATCH 29/44] Clean up warnings --- vegafusion-datafusion-udfs/src/udfs/member/mod.rs | 2 -- vegafusion-datafusion-udfs/src/udfs/object/mod.rs | 2 +- vegafusion-runtime/src/expression/compiler/array.rs | 2 -- .../src/expression/compiler/builtin_functions/array/length.rs | 4 +--- .../expression/compiler/builtin_functions/math/isfinite.rs | 1 - vegafusion-runtime/src/expression/compiler/member.rs | 1 - vegafusion-sql/src/compile/scalar.rs | 1 - 7 files changed, 2 insertions(+), 11 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs index 938c788b8..64d61e823 100644 --- a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs @@ -12,7 +12,6 @@ use vegafusion_common::error::{Result, VegaFusionError}; pub struct GetObjectMemberUDF { field_type: DataType, field_index: usize, - property_name: String, signature: Signature, name: String, } @@ -44,7 +43,6 @@ impl GetObjectMemberUDF { Ok(Self { field_type, field_index, - property_name, signature, name, }) diff --git a/vegafusion-datafusion-udfs/src/udfs/object/mod.rs b/vegafusion-datafusion-udfs/src/udfs/object/mod.rs index 3ef227b82..458464813 100644 --- a/vegafusion-datafusion-udfs/src/udfs/object/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/object/mod.rs @@ -1,5 +1,5 @@ -use std::any::Any; use crate::udfs::util::make_scalar_function; +use std::any::Any; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, StructArray}; use vegafusion_common::arrow::datatypes::{DataType, Field, FieldRef, Fields}; diff --git a/vegafusion-runtime/src/expression/compiler/array.rs b/vegafusion-runtime/src/expression/compiler/array.rs index c39bbb9dd..9732b2ec9 100644 --- a/vegafusion-runtime/src/expression/compiler/array.rs +++ b/vegafusion-runtime/src/expression/compiler/array.rs @@ -1,7 +1,5 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; -use std::ops::Deref; -use std::sync::Arc; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_core::error::Result; use vegafusion_core::proto::gen::expression::ArrayExpression; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs index 7ed10a77d..c89d07afb 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs @@ -1,7 +1,5 @@ use datafusion_common::DFSchema; -use datafusion_expr::{ - expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, -}; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition}; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::error::{ResultWithContext, VegaFusionError}; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs index d01eec4ac..269cde3bf 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs @@ -1,5 +1,4 @@ use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition, ScalarUDF}; -use std::ops::Deref; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index c3009aaaa..b9e0913f0 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -3,7 +3,6 @@ use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use std::convert::TryFrom; -use std::ops::Deref; use std::sync::Arc; use vegafusion_common::arrow::array::Int64Array; use vegafusion_common::arrow::compute::cast; diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index e6ff44e77..c8e8c1545 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -1,7 +1,6 @@ use crate::compile::data_type::ToSqlDataType; use crate::compile::expr::ToSqlExpr; use crate::dialect::Dialect; -use arrow::array::Array; use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; From 4ad196413394d9b161b907784258a9cf783887c6 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 13:52:12 -0500 Subject: [PATCH 30/44] Update data_add_tz to use ScalarUDFImpl --- .../src/udfs/datetime/date_add_tz.rs | 110 +++++++++++------- 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs index 390c292c0..14da33111 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs @@ -1,52 +1,80 @@ -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use std::any::Any; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, + ColumnarValue, ScalarUDF, Signature, Volatility, }, }; +use vegafusion_common::datafusion_common::DataFusionError; -fn make_date_add_tz_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DateAddTzUDF { + signature: Signature +} + +impl DateAddTzUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Date32, + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Date64, + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { + signature, + } + } +} + +impl ScalarUDFImpl for DateAddTzUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_add_tz" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> vegafusion_common::datafusion_common::Result { unimplemented!("date_add_tz function is not implemented by DataFusion") - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Date32, - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Date64, - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("date_add_tz", &signature, &return_type, &scalar_fn) + } +} + +fn make_date_add_tz_udf() -> ScalarUDF { + ScalarUDF::from(DateAddTzUDF::new()) } lazy_static! { From 65f226fe1ca731554e4b9996163f2191591be76a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 15:59:24 -0500 Subject: [PATCH 31/44] Convert more UDFs to use ScalarUDFImpl --- .../src/udfs/datetime/date_add_tz.rs | 20 +- .../src/udfs/datetime/date_part_tz.rs | 99 +++--- .../udfs/datetime/date_to_utc_timestamp.rs | 62 ++-- .../src/udfs/datetime/date_trunc_tz.rs | 106 +++--- .../udfs/datetime/epoch_to_utc_timestamp.rs | 63 ++-- .../src/udfs/datetime/format_timestamp.rs | 83 +++-- .../src/udfs/datetime/from_utc_timestamp.rs | 84 +++-- .../src/udfs/datetime/make_utc_timestamp.rs | 303 ++++++++++-------- .../src/udfs/datetime/str_to_utc_timestamp.rs | 56 +++- .../src/udfs/datetime/timeunit.rs | 110 ++++--- 10 files changed, 596 insertions(+), 390 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs index 14da33111..c0af94281 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs @@ -1,20 +1,17 @@ use std::any::Any; +use vegafusion_common::datafusion_common::DataFusionError; use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, - datafusion_expr::{ - ColumnarValue, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -use vegafusion_common::datafusion_common::DataFusionError; #[derive(Debug, Clone)] pub struct DateAddTzUDF { - signature: Signature + signature: Signature, } -impl DateAddTzUDF { +impl DateAddTzUDF { pub fn new() -> Self { let signature = Signature::one_of( vec![ @@ -45,9 +42,7 @@ impl DateAddTzUDF { ], Volatility::Immutable, ); - Self { - signature, - } + Self { signature } } } @@ -68,7 +63,10 @@ impl ScalarUDFImpl for DateAddTzUDF { Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) } - fn invoke(&self, _args: &[ColumnarValue]) -> vegafusion_common::datafusion_common::Result { + fn invoke( + &self, + _args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { unimplemented!("date_add_tz function is not implemented by DataFusion") } } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs index 77eaffd4a..99b63d2ea 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs @@ -1,20 +1,69 @@ use crate::udfs::datetime::from_utc_timestamp::from_utc_timestamp; use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; use datafusion_physical_expr::datetime_expressions; +use std::any::Any; use std::str::FromStr; -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, datafusion_common::DataFusionError, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_part_tz_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DatePartTzUDF { + signature: Signature, +} + +impl DatePartTzUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date32, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date64, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, // timezone + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DatePartTzUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_part_tz" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // [1] data array let timestamp_array = match &args[1] { ColumnarValue::Array(array) => array.clone(), @@ -48,39 +97,9 @@ fn make_date_part_tz_udf() -> ScalarUDF { args[0].clone(), // Part timestamp_in_tz, // Timestamp converted to timezone ]) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date32, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date64, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, // timezone - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("date_part_tz", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref DATE_PART_TZ_UDF: ScalarUDF = make_date_part_tz_udf(); + pub static ref DATE_PART_TZ_UDF: ScalarUDF = ScalarUDF::from(DatePartTzUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs index 921a7ddec..96a7933e3 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs @@ -1,6 +1,8 @@ use chrono::{NaiveDateTime, TimeZone}; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, Date32Array, TimestampMillisecondArray}, @@ -8,14 +10,42 @@ use vegafusion_common::{ datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_to_utc_timestamp() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DateToUtcTimestampUDF { + signature: Signature, +} + +impl DateToUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::exact( + vec![DataType::Date32, DataType::Utf8], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DateToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // [0] data array let date_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), @@ -61,24 +91,10 @@ fn make_date_to_utc_timestamp() -> ScalarUDF { } else { ScalarValue::try_from_array(×tamp_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = Signature::exact( - vec![DataType::Date32, DataType::Utf8], - Volatility::Immutable, - ); - - ScalarUDF::new( - "date_to_utc_timestamp", - &signature, - &return_type, - &scalar_fn, - ) + } } lazy_static! { - pub static ref DATE_TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_date_to_utc_timestamp(); + pub static ref DATE_TO_UTC_TIMESTAMP_UDF: ScalarUDF = + ScalarUDF::from(DateToUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs index 3314ce270..a2d68651a 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs @@ -1,50 +1,74 @@ -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use std::any::Any; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_trunc_tz_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DateTruncTzUDF { + signature: Signature, +} + +impl DateTruncTzUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date32, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date64, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, // timezone + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DateTruncTzUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_trunc_tz" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + _args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { unimplemented!("date_trunc_tz function is not implemented by DataFusion") - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date32, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date64, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, // timezone - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("date_trunc_tz", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref DATE_TRUNC_TZ_UDF: ScalarUDF = make_date_trunc_tz_udf(); + pub static ref DATE_TRUNC_TZ_UDF: ScalarUDF = ScalarUDF::from(DateTruncTzUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs index 7a3f041e4..20331e20b 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs @@ -1,18 +1,50 @@ -use std::sync::Arc; +use std::any::Any; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ compute::cast, datatypes::{DataType, TimeUnit}, }, datafusion_common::ScalarValue, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_epoch_to_utc_timestamp() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct EpochMsToUtcTimestampUDF { + signature: Signature, +} + +impl EpochMsToUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for EpochMsToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "epoch_ms_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), @@ -30,21 +62,10 @@ fn make_epoch_to_utc_timestamp() -> ScalarUDF { } else { ScalarValue::try_from_array(×tamp_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); - - ScalarUDF::new( - "epoch_ms_to_utc_timestamp", - &signature, - &return_type, - &scalar_fn, - ) + } } lazy_static! { - pub static ref EPOCH_MS_TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_epoch_to_utc_timestamp(); + pub static ref EPOCH_MS_TO_UTC_TIMESTAMP_UDF: ScalarUDF = + ScalarUDF::from(EpochMsToUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs index 4c2048b88..7c5d2882b 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs @@ -1,20 +1,67 @@ use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; use chrono::NaiveDateTime; +use std::any::Any; use std::sync::Arc; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, StringArray, TimestampMillisecondArray}, datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - TypeSignature, Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, TypeSignature, Volatility}, }; -fn make_format_timestamp_udf() -> ScalarUDF { - let time_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct FormatTimestampUDF { + signature: Signature, +} + +impl FormatTimestampUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for FormatTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "format_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Utf8) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // Argument order // [0] data array let data_array = match &args[0] { @@ -67,27 +114,7 @@ fn make_format_timestamp_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&formatted, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Utf8))); - - let signature: Signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), - TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("format_timestamp", &signature, &return_type, &time_fn) + } } fn convert_d3_format_string(d3_format_str: &str) -> String { @@ -99,5 +126,5 @@ fn convert_d3_format_string(d3_format_str: &str) -> String { } lazy_static! { - pub static ref FORMAT_TIMESTAMP_UDF: ScalarUDF = make_format_timestamp_udf(); + pub static ref FORMAT_TIMESTAMP_UDF: ScalarUDF = ScalarUDF::from(FormatTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs index e26c7b937..7d68c02d9 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs @@ -1,25 +1,72 @@ use chrono::NaiveDateTime; use chrono::TimeZone; use chrono_tz::Tz; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::Array; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, TimestampMillisecondArray}, datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - TypeSignature, Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, TypeSignature, Volatility}, }; use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; -fn make_from_utc_timestamp() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct FromUtcTimestampUDF { + signature: Signature, +} + +impl FromUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for FromUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "from_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), @@ -46,28 +93,7 @@ fn make_from_utc_timestamp() -> ScalarUDF { } else { ScalarValue::try_from_array(&result_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), - TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("from_utc_timestamp", &signature, &return_type, &scalar_fn) + } } pub fn from_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result { @@ -104,5 +130,5 @@ pub fn from_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result ScalarUDF { - let datetime_components: ScalarFunctionImplementation = - Arc::new(move |args: &[ColumnarValue]| { - let tz_str = if let ColumnarValue::Scalar(tz_scalar) = &args[7] { - tz_scalar.to_string() - } else { - return Err(DataFusionError::Internal( - "Expected timezone to be a scalar".to_string(), - )); - }; - - let input_tz = chrono_tz::Tz::from_str(&tz_str).map_err(|_err| { - DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) - })?; - - // first, identify if any of the arguments is an Array. If yes, store its `len`, - // as any scalar will need to be converted to an array of len `len`. - let len = args[..7] - .iter() - .fold(Option::::None, |acc, arg| match arg { - ColumnarValue::Scalar(_) => acc, - ColumnarValue::Array(a) => Some(a.len()), - }); - - // to arrays - let args = if let Some(len) = len { - args.iter() - .map(|arg| arg.clone().into_array(len)) - .collect::, DataFusionError>>()? +#[derive(Debug, Clone)] +pub struct MakeUtcTimestampUDF { + signature: Signature, +} + +impl MakeUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::exact( + vec![ + DataType::Float64, // year + DataType::Float64, // month + DataType::Float64, // date + DataType::Float64, // hour + DataType::Float64, // minute + DataType::Float64, // second + DataType::Float64, // millisecond + DataType::Utf8, // time zone + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for MakeUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "make_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { + let tz_str = if let ColumnarValue::Scalar(tz_scalar) = &args[7] { + tz_scalar.to_string() + } else { + return Err(DataFusionError::Internal( + "Expected timezone to be a scalar".to_string(), + )); + }; + + let input_tz = chrono_tz::Tz::from_str(&tz_str).map_err(|_err| { + DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) + })?; + + // first, identify if any of the arguments is an Array. If yes, store its `len`, + // as any scalar will need to be converted to an array of len `len`. + let len = args[..7] + .iter() + .fold(Option::::None, |acc, arg| match arg { + ColumnarValue::Scalar(_) => acc, + ColumnarValue::Array(a) => Some(a.len()), + }); + + // to arrays + let args = if let Some(len) = len { + args.iter() + .map(|arg| arg.clone().into_array(len)) + .collect::, DataFusionError>>()? + } else { + args.iter() + .map(|arg| arg.clone().into_array(1)) + .collect::, DataFusionError>>()? + }; + + // To int64 arrays + let years = cast(&args[0], &DataType::Int64).unwrap(); + let years = years.as_any().downcast_ref::().unwrap(); + + let months = cast(&args[1], &DataType::Int64).unwrap(); + let months = months.as_any().downcast_ref::().unwrap(); + + let days = cast(&args[2], &DataType::Int64).unwrap(); + let days = days.as_any().downcast_ref::().unwrap(); + + let hours = cast(&args[3], &DataType::Int64).unwrap(); + let hours = hours.as_any().downcast_ref::().unwrap(); + + let minutes = cast(&args[4], &DataType::Int64).unwrap(); + let minutes = minutes.as_any().downcast_ref::().unwrap(); + + let seconds = cast(&args[5], &DataType::Int64).unwrap(); + let seconds = seconds.as_any().downcast_ref::().unwrap(); + + let millis = cast(&args[6], &DataType::Int64).unwrap(); + let millis = millis.as_any().downcast_ref::().unwrap(); + + let num_rows = years.len(); + let mut datetime_builder = TimestampMillisecondBuilder::new(); + + for i in 0..num_rows { + if years.is_null(i) + || months.is_null(i) + || days.is_null(i) + || hours.is_null(i) + || minutes.is_null(i) + || seconds.is_null(i) + || millis.is_null(i) + { + // If any component is null, propagate null + datetime_builder.append_null(); } else { - args.iter() - .map(|arg| arg.clone().into_array(1)) - .collect::, DataFusionError>>()? - }; - - // To int64 arrays - let years = cast(&args[0], &DataType::Int64).unwrap(); - let years = years.as_any().downcast_ref::().unwrap(); - - let months = cast(&args[1], &DataType::Int64).unwrap(); - let months = months.as_any().downcast_ref::().unwrap(); - - let days = cast(&args[2], &DataType::Int64).unwrap(); - let days = days.as_any().downcast_ref::().unwrap(); - - let hours = cast(&args[3], &DataType::Int64).unwrap(); - let hours = hours.as_any().downcast_ref::().unwrap(); - - let minutes = cast(&args[4], &DataType::Int64).unwrap(); - let minutes = minutes.as_any().downcast_ref::().unwrap(); - - let seconds = cast(&args[5], &DataType::Int64).unwrap(); - let seconds = seconds.as_any().downcast_ref::().unwrap(); - - let millis = cast(&args[6], &DataType::Int64).unwrap(); - let millis = millis.as_any().downcast_ref::().unwrap(); - - let num_rows = years.len(); - let mut datetime_builder = TimestampMillisecondBuilder::new(); - - for i in 0..num_rows { - if years.is_null(i) - || months.is_null(i) - || days.is_null(i) - || hours.is_null(i) - || minutes.is_null(i) - || seconds.is_null(i) - || millis.is_null(i) - { - // If any component is null, propagate null - datetime_builder.append_null(); + let year = years.value(i); + let month = months.value(i); + let day = days.value(i); + let hour = hours.value(i); + let minute = minutes.value(i); + let second = seconds.value(i); + let milli = millis.value(i); + + // Treat 00-99 as 1900 to 1999 + let mut year = year; + if (0..100).contains(&year) { + year += 1900 + } + + let datetime: Option> = input_tz + .with_ymd_and_hms( + year as i32, + month as u32 + 1, + day as u32, + hour as u32, + minute as u32, + second as u32, + ) + .single() + .and_then(|date: DateTime<_>| date.with_nanosecond((milli * 1_000_000) as u32)); + + if let Some(datetime) = datetime { + // Always convert to UTC + let datetime = datetime.with_timezone(&chrono::Utc); + datetime_builder.append_value(datetime.timestamp_millis()); } else { - let year = years.value(i); - let month = months.value(i); - let day = days.value(i); - let hour = hours.value(i); - let minute = minutes.value(i); - let second = seconds.value(i); - let milli = millis.value(i); - - // Treat 00-99 as 1900 to 1999 - let mut year = year; - if (0..100).contains(&year) { - year += 1900 - } - - let datetime: Option> = input_tz - .with_ymd_and_hms( - year as i32, - month as u32 + 1, - day as u32, - hour as u32, - minute as u32, - second as u32, - ) - .single() - .and_then(|date: DateTime<_>| { - date.with_nanosecond((milli * 1_000_000) as u32) - }); - - if let Some(datetime) = datetime { - // Always convert to UTC - let datetime = datetime.with_timezone(&chrono::Utc); - datetime_builder.append_value(datetime.timestamp_millis()); - } else { - // Invalid date - datetime_builder.append_null(); - } + // Invalid date + datetime_builder.append_null(); } } + } - let result = Arc::new(datetime_builder.finish()) as ArrayRef; + let result = Arc::new(datetime_builder.finish()) as ArrayRef; - // maybe back to scalar - if len.is_some() { - Ok(ColumnarValue::Array(result)) - } else { - ScalarValue::try_from_array(&result, 0).map(ColumnarValue::Scalar) - } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature = Signature::exact( - vec![ - DataType::Float64, // year - DataType::Float64, // month - DataType::Float64, // date - DataType::Float64, // hour - DataType::Float64, // minute - DataType::Float64, // second - DataType::Float64, // millisecond - DataType::Utf8, // time zone - ], - Volatility::Immutable, - ); - ScalarUDF::new( - "make_utc_timestamp", - &signature, - &return_type, - &datetime_components, - ) + // maybe back to scalar + if len.is_some() { + Ok(ColumnarValue::Array(result)) + } else { + ScalarValue::try_from_array(&result, 0).map(ColumnarValue::Scalar) + } + } } lazy_static! { - pub static ref MAKE_UTC_TIMESTAMP: ScalarUDF = make_utc_timestamp_udf(); + pub static ref MAKE_UTC_TIMESTAMP: ScalarUDF = ScalarUDF::from(MakeUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs index 4bba48e40..7f9038887 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs @@ -3,13 +3,13 @@ use chrono::{ {DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, Utc}, }; use regex::Regex; +use std::any::Any; use std::{str::FromStr, sync::Arc}; use vegafusion_common::arrow::array::{ArrayRef, StringArray, TimestampMillisecondArray}; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; lazy_static! { @@ -329,8 +329,43 @@ pub fn datetime_strs_to_timestamp_millis( Arc::new(millis_array) as ArrayRef } -fn make_str_to_utc_timestamp_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct StrToUtcTimestampUDF { + signature: Signature, +} + +impl StrToUtcTimestampUDF { + pub fn new() -> Self { + let signature = + Signature::exact(vec![DataType::Utf8, DataType::Utf8], Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for StrToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "str_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let str_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), @@ -359,19 +394,12 @@ fn make_str_to_utc_timestamp_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(×tamp_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = - Signature::exact(vec![DataType::Utf8, DataType::Utf8], Volatility::Immutable); - - ScalarUDF::new("str_to_utc_timestamp", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref STR_TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_str_to_utc_timestamp_udf(); + pub static ref STR_TO_UTC_TIMESTAMP_UDF: ScalarUDF = + ScalarUDF::from(StrToUtcTimestampUDF::new()); } #[test] diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs index 71066586c..423fc4cf4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs @@ -1,5 +1,6 @@ use crate::udfs::datetime::process_input_datetime; use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, TimeZone, Timelike, Utc, Weekday}; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, Int64Array, TimestampMillisecondArray}; @@ -8,8 +9,7 @@ use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::arrow::temporal_conversions::date64_to_datetime; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - TypeSignature, Volatility, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; fn extract_bool(value: &ColumnarValue) -> std::result::Result { @@ -223,9 +223,71 @@ fn perform_timeunit_start_from_utc( dt_value } -fn make_timeunit_start_udf() -> ScalarUDF { - let timeunit_start: ScalarFunctionImplementation = Arc::new(|columns: &[ColumnarValue]| { - let (timestamp, tz, units_mask) = unpack_timeunit_udf_args(columns)?; +#[derive(Debug, Clone)] +pub struct TimeunitStartUDF { + signature: Signature, +} + +impl TimeunitStartUDF { + pub fn new() -> Self { + let make_sig = |timestamp_dtype: DataType| -> TypeSignature { + TypeSignature::Exact(vec![ + timestamp_dtype, // [0] timestamp + DataType::Utf8, // [1] timezone + DataType::Boolean, // [2] Year + DataType::Boolean, // [3] Quarter + DataType::Boolean, // [4] Month + DataType::Boolean, // [5] Date + DataType::Boolean, // [6] Week + DataType::Boolean, // [7] Day + DataType::Boolean, // [8] DayOfYear + DataType::Boolean, // [9] Hours + DataType::Boolean, // [10] Minutes + DataType::Boolean, // [11] Seconds + DataType::Boolean, // [12] Milliseconds + ]) + }; + + let signature = Signature::one_of( + vec![ + make_sig(DataType::Int64), + make_sig(DataType::Date64), + make_sig(DataType::Date32), + make_sig(DataType::Timestamp(TimeUnit::Millisecond, None)), + make_sig(DataType::Timestamp(TimeUnit::Nanosecond, None)), + ], + Volatility::Immutable, + ); + + Self { signature } + } +} + +impl ScalarUDFImpl for TimeunitStartUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "vega_timeunit" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { + let (timestamp, tz, units_mask) = unpack_timeunit_udf_args(args)?; let array = timestamp.as_any().downcast_ref::().unwrap(); let result_array: TimestampMillisecondArray = unary(array, |value| { @@ -233,43 +295,9 @@ fn make_timeunit_start_udf() -> ScalarUDF { }); Ok(ColumnarValue::Array(Arc::new(result_array) as ArrayRef)) - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_datatypes| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let make_sig = |timestamp_dtype: DataType| -> TypeSignature { - TypeSignature::Exact(vec![ - timestamp_dtype, // [0] timestamp - DataType::Utf8, // [1] timezone - DataType::Boolean, // [2] Year - DataType::Boolean, // [3] Quarter - DataType::Boolean, // [4] Month - DataType::Boolean, // [5] Date - DataType::Boolean, // [6] Week - DataType::Boolean, // [7] Day - DataType::Boolean, // [8] DayOfYear - DataType::Boolean, // [9] Hours - DataType::Boolean, // [10] Minutes - DataType::Boolean, // [11] Seconds - DataType::Boolean, // [12] Milliseconds - ]) - }; - - let signature: Signature = Signature::one_of( - vec![ - make_sig(DataType::Int64), - make_sig(DataType::Date64), - make_sig(DataType::Date32), - make_sig(DataType::Timestamp(TimeUnit::Millisecond, None)), - make_sig(DataType::Timestamp(TimeUnit::Nanosecond, None)), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("vega_timeunit", &signature, &return_type, &timeunit_start) + } } lazy_static! { - pub static ref TIMEUNIT_START_UDF: ScalarUDF = make_timeunit_start_udf(); + pub static ref TIMEUNIT_START_UDF: ScalarUDF = ScalarUDF::from(TimeunitStartUDF::new()); } From e2b368a89ac1681175a8ea4b6a8a9d11e8793511 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 16:22:49 -0500 Subject: [PATCH 32/44] Convert remaining UDFs to use ScalarUDFImpl --- .../src/udfs/datetime/str_to_utc_timestamp.rs | 2 +- .../src/udfs/datetime/to_utc_timestamp.rs | 62 ++++++++++---- .../udfs/datetime/utc_timestamp_to_epoch.rs | 78 +++++++++++------ .../src/udfs/datetime/utc_timestamp_to_str.rs | 85 ++++++++++++------- vegafusion-sql/src/compile/scalar.rs | 57 +++++++++---- 5 files changed, 192 insertions(+), 92 deletions(-) diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs index 7f9038887..981acb685 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs @@ -357,7 +357,7 @@ impl ScalarUDFImpl for StrToUtcTimestampUDF { fn return_type( &self, - arg_types: &[DataType], + _arg_types: &[DataType], ) -> vegafusion_common::datafusion_common::Result { Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs index 036cca048..1a88dd8c6 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs @@ -1,9 +1,11 @@ use chrono::TimeZone; use chrono::{NaiveDateTime, Timelike}; use chrono_tz::Tz; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::Array; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, TimestampMillisecondArray}, @@ -11,14 +13,48 @@ use vegafusion_common::{ datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_to_utc_timestamp_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct ToUtcTimestampUDF { + signature: Signature, +} + +impl ToUtcTimestampUDF { + pub fn new() -> Self { + // Signature should be (Timestamp, UTF8), but specifying Timestamp in the signature + // requires specifying the timezone explicitly, and DataFusion doesn't currently + // coerce between timezones. + let signature: Signature = Signature::any(2, Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for ToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), @@ -45,17 +81,7 @@ fn make_to_utc_timestamp_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&result_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - // Signature should be (Timestamp, UTF8), but specifying Timestamp in the signature - // requires specifying the timezone explicitly, and DataFusion doesn't currently - // coerce between timezones. - let signature: Signature = Signature::any(2, Volatility::Immutable); - - ScalarUDF::new("to_utc_timestamp", &signature, &return_type, &scalar_fn) + } } pub fn to_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result { @@ -127,5 +153,5 @@ pub fn to_timestamp_ms(array: &ArrayRef) -> Result { } lazy_static! { - pub static ref TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_to_utc_timestamp_udf(); + pub static ref TO_UTC_TIMESTAMP_UDF: ScalarUDF = ScalarUDF::from(ToUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs index 1fd4780e4..d48ae1b91 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs @@ -1,20 +1,59 @@ use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use std::any::Any; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::{ compute::cast, datatypes::{DataType, TimeUnit}, }, datafusion_common::ScalarValue, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_utc_timestamp_to_epoch_ms_udf() -> ScalarUDF { - let time_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct UtcTimestampToEpochUDF { + signature: Signature, +} + +impl UtcTimestampToEpochUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32]), + TypeSignature::Exact(vec![DataType::Date64]), + TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Millisecond, None)]), + TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Nanosecond, None)]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for UtcTimestampToEpochUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "utc_timestamp_to_epoch_ms" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Int64) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), @@ -31,27 +70,10 @@ fn make_utc_timestamp_to_epoch_ms_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&result_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32]), - TypeSignature::Exact(vec![DataType::Date64]), - TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Millisecond, None)]), - TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Nanosecond, None)]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new( - "utc_timestamp_to_epoch_ms", - &signature, - &return_type, - &time_fn, - ) + } } lazy_static! { - pub static ref UTC_TIMESTAMP_TO_EPOCH_MS: ScalarUDF = make_utc_timestamp_to_epoch_ms_udf(); + pub static ref UTC_TIMESTAMP_TO_EPOCH_MS: ScalarUDF = + ScalarUDF::from(UtcTimestampToEpochUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs index 88ae6749b..7df843c86 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs @@ -1,21 +1,67 @@ use crate::udfs::datetime::from_utc_timestamp::from_utc_timestamp; use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; use chrono::NaiveDateTime; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, StringArray, TimestampMillisecondArray}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::TypeSignature; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_utc_timestamp_to_str_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct UtcTimestampToStrUDF { + signature: Signature, +} + +impl UtcTimestampToStrUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for UtcTimestampToStrUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "utc_timestamp_to_str" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Utf8) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // Argument order // [0] data array let timestamp_array = match &args[0] { @@ -74,29 +120,10 @@ fn make_utc_timestamp_to_str_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&formatted, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Utf8))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), - TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("utc_timestamp_to_str", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref UTC_TIMESTAMP_TO_STR_UDF: ScalarUDF = make_utc_timestamp_to_str_udf(); + pub static ref UTC_TIMESTAMP_TO_STR_UDF: ScalarUDF = + ScalarUDF::from(UtcTimestampToStrUDF::new()); } diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index c8e8c1545..8b29f5c25 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -5,13 +5,14 @@ use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; use datafusion_expr::{ - expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionDefinition, - ScalarFunctionImplementation, ScalarUDF, Signature, Volatility, + expr, lit, ColumnarValue, Expr, ScalarFunctionDefinition, ScalarUDF, ScalarUDFImpl, Signature, + Volatility, }; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr, Ident, ObjectName as SqlObjectName, Value as SqlValue, }; +use std::any::Any; use std::ops::Add; use std::sync::Arc; use vegafusion_common::data::scalar::ArrayRefHelpers; @@ -260,26 +261,50 @@ impl ToSqlScalar for ScalarValue { fn ms_to_timestamp(v: i64, dialect: &Dialect) -> Result { // Hack to recursively transform the epoch_ms_to_utc_timestamp - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| { - panic!("Placeholder UDF implementation should not be called") - }); - - let udf = ScalarUDF::new( - "epoch_ms_to_utc_timestamp", - &signature, - &return_type, - &scalar_fn, - ); Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(udf)), + func_def: ScalarFunctionDefinition::UDF(Arc::new(ScalarUDF::from( + EpochMsToUtcTimestampUDF::new(), + ))), args: vec![lit(v)], }) .to_sql(dialect, &DFSchema::empty()) } +// Hack to recursively transform the epoch_ms_to_utc_timestamp +#[derive(Debug, Clone)] +pub struct EpochMsToUtcTimestampUDF { + signature: Signature, +} + +impl EpochMsToUtcTimestampUDF { + pub fn new() -> Self { + let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for EpochMsToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "epoch_ms_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> datafusion_common::Result { + panic!("Placeholder UDF implementation should not be called") + } +} + fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { let epoch = chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); match days { From e8acd9d344a101d14937787bda10dee9d430d565 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 16:32:25 -0500 Subject: [PATCH 33/44] clippy fix --- vegafusion-common/src/data/json_writer.rs | 2 +- vegafusion-common/src/data/scalar.rs | 2 +- vegafusion-common/src/data/table.rs | 6 +-- vegafusion-core/src/expression/visitors.rs | 6 +-- .../src/planning/lift_facet_aggregations.rs | 12 +++--- .../src/planning/projection_pushdown.rs | 2 +- .../src/planning/stringify_local_datetimes.rs | 4 +- vegafusion-core/src/spec/data.rs | 2 +- vegafusion-core/src/spec/transform/bin.rs | 2 +- vegafusion-core/src/spec/transform/fold.rs | 2 +- vegafusion-core/src/spec/transform/pivot.rs | 2 +- .../src/spec/transform/timeunit.rs | 2 +- vegafusion-core/src/transform/bin.rs | 2 +- vegafusion-core/src/transform/stack.rs | 2 +- vegafusion-core/src/transform/timeunit.rs | 2 +- vegafusion-dataframe/src/dataframe.rs | 2 +- .../src/udfs/array/indexof.rs | 6 +++ .../src/udfs/array/span.rs | 6 +++ .../src/udfs/datetime/date_add_tz.rs | 6 +++ .../src/udfs/datetime/date_part_tz.rs | 6 +++ .../udfs/datetime/date_to_utc_timestamp.rs | 6 +++ .../src/udfs/datetime/date_trunc_tz.rs | 6 +++ .../udfs/datetime/epoch_to_utc_timestamp.rs | 6 +++ .../src/udfs/datetime/format_timestamp.rs | 6 +++ .../src/udfs/datetime/from_utc_timestamp.rs | 6 +++ .../src/udfs/datetime/make_utc_timestamp.rs | 6 +++ .../src/udfs/datetime/mod.rs | 5 +-- .../src/udfs/datetime/str_to_utc_timestamp.rs | 6 +++ .../src/udfs/datetime/timeunit.rs | 6 +++ .../src/udfs/datetime/to_utc_timestamp.rs | 6 +++ .../udfs/datetime/utc_timestamp_to_epoch.rs | 6 +++ .../src/udfs/datetime/utc_timestamp_to_str.rs | 6 +++ .../src/udfs/math/isfinite.rs | 6 +++ .../src/udfs/member/mod.rs | 10 ++--- vegafusion-runtime/src/data/tasks.rs | 2 +- .../data/vl_selection_resolve.rs | 2 +- .../builtin_functions/date_time/date_parts.rs | 2 +- .../src/expression/compiler/call.rs | 2 +- .../src/expression/compiler/mod.rs | 2 +- .../destringify_selection_datetimes.rs | 6 +-- vegafusion-runtime/src/task_graph/runtime.rs | 2 +- vegafusion-runtime/src/transform/fold.rs | 2 +- vegafusion-runtime/src/transform/timeunit.rs | 24 ++++++------ vegafusion-sql/src/compile/scalar.rs | 6 +++ .../src/connection/datafusion_conn.rs | 6 +-- .../connection/datafusion_py_datasource.rs | 2 +- vegafusion-sql/src/dataframe/mod.rs | 2 +- vegafusion-sql/src/dialect/mod.rs | 38 +++++++++---------- vegafusion-wasm/src/lib.rs | 4 +- 49 files changed, 180 insertions(+), 85 deletions(-) diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index 75bf82012..6138e29eb 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -429,7 +429,7 @@ fn set_column_for_json_rows( let inner_objs = struct_array_to_jsonmap_array(as_struct_array(array), row_count)?; rows.iter_mut() .take(row_count) - .zip(inner_objs.into_iter()) + .zip(inner_objs) .for_each(|(row, obj)| { row.insert(col_name.to_string(), Value::Object(obj)); }); diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 5addbd93d..d0ffb6d13 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -237,7 +237,7 @@ impl ArrayRefHelpers for ArrayRef { /// Convert ArrayRef into vector of ScalarValues fn to_scalar_vec(&self) -> std::result::Result, DataFusionError> { (0..self.len()) - .map(|i| Ok(ScalarValue::try_from_array(self, i)?)) + .map(|i| ScalarValue::try_from_array(self, i)) .collect::, DataFusionError>>() } diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index 1f6b3fd8f..50568364b 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -174,7 +174,7 @@ impl VegaFusionTable { pub fn to_record_batch(&self) -> Result { let mut schema = self.schema.clone(); - if let Some(batch) = self.batches.get(0) { + if let Some(batch) = self.batches.first() { schema = batch.schema() } concat_batches(&schema, &self.batches) @@ -205,7 +205,7 @@ impl VegaFusionTable { if let serde_json::Value::Array(values) = value { // Handle special case where array elements are non-object scalars let mut values = Cow::Borrowed(values); - if let Some(first) = values.get(0) { + if let Some(first) = values.first() { if let Value::Object(props) = first { // Handle odd special case where vega will interpret // [{}, {}] as [{"datum": {}}, {"datum": {}}] @@ -298,7 +298,7 @@ impl VegaFusionTable { let batches_list = PyList::new(py, batch_objects); // Convert table's schema into pyarrow schema - let schema = if let Some(batch) = self.batches.get(0) { + let schema = if let Some(batch) = self.batches.first() { // Get schema from first batch if present batch.schema() } else { diff --git a/vegafusion-core/src/expression/visitors.rs b/vegafusion-core/src/expression/visitors.rs index 2c5100101..330f35e47 100644 --- a/vegafusion-core/src/expression/visitors.rs +++ b/vegafusion-core/src/expression/visitors.rs @@ -105,7 +105,7 @@ impl ExpressionVisitor for GetInputVariablesVisitor { /// Collect data and scale identifiers. These show up as a literal string as the first /// argument to a Data or Scale callable. fn visit_called_identifier(&mut self, node: &Identifier, args: &[Expression]) { - if let Some(arg0) = args.get(0) { + if let Some(arg0) = args.first() { if let Ok(arg0) = arg0.as_literal() { if let Value::String(arg0) = arg0.value() { // Check data callable @@ -157,7 +157,7 @@ impl UpdateVariablesExprVisitor { impl ExpressionVisitor for UpdateVariablesExprVisitor { fn visit_called_identifier(&mut self, node: &Identifier, args: &[Expression]) { if node.name == "modify" { - if let Some(arg0) = args.get(0) { + if let Some(arg0) = args.first() { if let Ok(arg0) = arg0.as_literal() { if let Value::String(arg0) = arg0.value() { // First arg is a string, which holds the name of the output dataset @@ -372,7 +372,7 @@ impl<'a> ExpressionVisitor for DatasetsColumnUsageVisitor<'a> { .. })), .. - }) = node.arguments.get(0) + }) = node.arguments.first() { // Resolve data variable let reference_data_var = Variable::new_data(reference_data_name); diff --git a/vegafusion-core/src/planning/lift_facet_aggregations.rs b/vegafusion-core/src/planning/lift_facet_aggregations.rs index 86d43f959..92d741e55 100644 --- a/vegafusion-core/src/planning/lift_facet_aggregations.rs +++ b/vegafusion-core/src/planning/lift_facet_aggregations.rs @@ -66,7 +66,7 @@ impl ExtractFacetAggregationsVisitor { } fn num_children_of_dataset(&self, name: &str, scope: &[u32]) -> usize { - let facet_dataset_var: ScopedVariable = (Variable::new_data(&name), Vec::from(scope)); + let facet_dataset_var: ScopedVariable = (Variable::new_data(name), Vec::from(scope)); let Some(facet_dataset_idx) = self.node_indexes.get(&facet_dataset_var) else { return 0; }; @@ -107,7 +107,7 @@ impl MutChartVisitor for ExtractFacetAggregationsVisitor { let mut lifted_transforms: Vec = Vec::new(); let agg = loop { - match child_dataset.transform.get(0).cloned() { + match child_dataset.transform.first().cloned() { None => { // End of transforms for this dataset, advance to child dataset if possible if self.num_children_of_dataset(&child_dataset.name, scope) != 1 { @@ -235,11 +235,9 @@ impl MutChartVisitor for ExtractFacetAggregationsVisitor { // Add lifted aggregate transform, potentially after the joinaggregate transform lifted_transforms.push(TransformSpec::Aggregate(agg)); - } else { - if lifted_transforms.is_empty() { - // No supported transforms found - return Ok(()); - } + } else if lifted_transforms.is_empty() { + // No supported transforms found + return Ok(()); } // Create facet dataset name and increment counter to keep names unique even if the same diff --git a/vegafusion-core/src/planning/projection_pushdown.rs b/vegafusion-core/src/planning/projection_pushdown.rs index d1c61cb99..75a414e00 100644 --- a/vegafusion-core/src/planning/projection_pushdown.rs +++ b/vegafusion-core/src/planning/projection_pushdown.rs @@ -701,8 +701,8 @@ impl<'a> MutChartVisitor for InsertProjectionVisitor<'a> { // so we can append a projection transform to limit the columns that are produced // Note: empty strings here seem to break vega, filter them out let proj_fields: Vec<_> = sorted(columns) + .filter(|&f| !f.is_empty()) .cloned() - .filter(|f| !f.is_empty()) .map(|f| escape_field(&f)) .collect(); diff --git a/vegafusion-core/src/planning/stringify_local_datetimes.rs b/vegafusion-core/src/planning/stringify_local_datetimes.rs index a9c8b769d..693b654fe 100644 --- a/vegafusion-core/src/planning/stringify_local_datetimes.rs +++ b/vegafusion-core/src/planning/stringify_local_datetimes.rs @@ -42,8 +42,8 @@ pub fn stringify_local_datetimes( let server_to_client_datasets: HashSet<_> = comm_plan .server_to_client .iter() + .filter(|&var| var.0.namespace == VariableNamespace::Data as i32) .cloned() - .filter(|var| var.0.namespace == VariableNamespace::Data as i32) .collect(); let mut visitor = CollectCandidateDatasetMapping::new( @@ -407,7 +407,7 @@ impl<'a> MutChartVisitor for StringifyLocalDatetimeFieldsVisitor<'a> { let source_resolved_var = (source_resolved.var, source_resolved.scope); if let Some(fields) = self.local_datetime_fields.get(&source_resolved_var) { for field in sorted(fields) { - let field = unescape_field(&field); + let field = unescape_field(field); let expr_str = format!("toDate(datum['{field}'], 'local')"); let transforms = &mut data.transform; let transform = FormulaTransformSpec { diff --git a/vegafusion-core/src/spec/data.rs b/vegafusion-core/src/spec/data.rs index 6202ab670..c3fa0f81c 100644 --- a/vegafusion-core/src/spec/data.rs +++ b/vegafusion-core/src/spec/data.rs @@ -85,7 +85,7 @@ impl DataSpec { } else if self.url.is_some() { DependencyNodeSupported::PartiallySupported } else { - match self.transform.get(0) { + match self.transform.first() { Some(tx) if tx.supported_and_allowed(planner_config, task_scope, scope) => { DependencyNodeSupported::PartiallySupported } diff --git a/vegafusion-core/src/spec/transform/bin.rs b/vegafusion-core/src/spec/transform/bin.rs index ec169165d..2524c5502 100644 --- a/vegafusion-core/src/spec/transform/bin.rs +++ b/vegafusion-core/src/spec/transform/bin.rs @@ -125,7 +125,7 @@ impl TransformSpecTrait for BinTransformSpec { let bin_start = self .as_ .as_ref() - .and_then(|as_| as_.get(0).cloned()) + .and_then(|as_| as_.first().cloned()) .unwrap_or_else(|| "bin0".to_string()); let bin_end = self .as_ diff --git a/vegafusion-core/src/spec/transform/fold.rs b/vegafusion-core/src/spec/transform/fold.rs index 6cac92cb2..a925773ef 100644 --- a/vegafusion-core/src/spec/transform/fold.rs +++ b/vegafusion-core/src/spec/transform/fold.rs @@ -24,7 +24,7 @@ impl FoldTransformSpec { pub fn as_(&self) -> Vec { let as_ = self.as_.clone().unwrap_or_default(); vec![ - as_.get(0).cloned().unwrap_or_else(|| "key".to_string()), + as_.first().cloned().unwrap_or_else(|| "key".to_string()), as_.get(1).cloned().unwrap_or_else(|| "value".to_string()), ] } diff --git a/vegafusion-core/src/spec/transform/pivot.rs b/vegafusion-core/src/spec/transform/pivot.rs index 7ba7c7a03..1ae11bd1e 100644 --- a/vegafusion-core/src/spec/transform/pivot.rs +++ b/vegafusion-core/src/spec/transform/pivot.rs @@ -67,7 +67,7 @@ impl TransformSpecTrait for PivotTransformSpec { .unwrap_or_default() .iter() .filter_map(|groupby_field| { - let unescaped = unescape_field(&groupby_field); + let unescaped = unescape_field(groupby_field); if input_local_datetime_columns.contains(&unescaped) { Some(unescaped) } else { diff --git a/vegafusion-core/src/spec/transform/timeunit.rs b/vegafusion-core/src/spec/transform/timeunit.rs index 6f9f30d2f..e9df4aa0d 100644 --- a/vegafusion-core/src/spec/transform/timeunit.rs +++ b/vegafusion-core/src/spec/transform/timeunit.rs @@ -67,7 +67,7 @@ impl TimeUnitTransformSpec { let as0 = self .as_ .clone() - .and_then(|as_| as_.get(0).cloned()) + .and_then(|as_| as_.first().cloned()) .unwrap_or_else(|| "unit0".to_string()); let as1 = self .as_ diff --git a/vegafusion-core/src/transform/bin.rs b/vegafusion-core/src/transform/bin.rs index 58359d5ae..6fe72d378 100644 --- a/vegafusion-core/src/transform/bin.rs +++ b/vegafusion-core/src/transform/bin.rs @@ -39,7 +39,7 @@ impl Bin { Ok(Self { field, extent: Some(extent_expr), - alias_0: as_.get(0).cloned(), + alias_0: as_.first().cloned(), alias_1: as_.get(1).cloned(), anchor: config.anchor, maxbins: config.maxbins, diff --git a/vegafusion-core/src/transform/stack.rs b/vegafusion-core/src/transform/stack.rs index a6b836745..6b9bd9a5c 100644 --- a/vegafusion-core/src/transform/stack.rs +++ b/vegafusion-core/src/transform/stack.rs @@ -47,7 +47,7 @@ impl Stack { // Extract aliases let alias0 = spec .as_() - .get(0) + .first() .cloned() .unwrap_or_else(|| "y0".to_string()); let alias1 = spec diff --git a/vegafusion-core/src/transform/timeunit.rs b/vegafusion-core/src/transform/timeunit.rs index 555b59330..a84bef850 100644 --- a/vegafusion-core/src/transform/timeunit.rs +++ b/vegafusion-core/src/transform/timeunit.rs @@ -16,7 +16,7 @@ impl TimeUnit { .collect(); let signal = transform.signal.clone(); - let alias_0 = transform.as_.as_ref().and_then(|v| v.get(0).cloned()); + let alias_0 = transform.as_.as_ref().and_then(|v| v.first().cloned()); let alias_1 = transform.as_.as_ref().and_then(|v| v.get(1).cloned()); let timezone = match &transform.timezone { diff --git a/vegafusion-dataframe/src/dataframe.rs b/vegafusion-dataframe/src/dataframe.rs index 94bf86f81..4a72ac6b0 100644 --- a/vegafusion-dataframe/src/dataframe.rs +++ b/vegafusion-dataframe/src/dataframe.rs @@ -30,7 +30,7 @@ pub trait DataFrame: Send + Sync + 'static { async fn collect_flat(&self) -> Result { let mut arrow_schema = Arc::new(self.schema()) as SchemaRef; let table = self.collect().await?; - if let Some(batch) = table.batches.get(0) { + if let Some(batch) = table.batches.first() { arrow_schema = batch.schema() } concat_batches(&arrow_schema, table.batches.as_slice()) diff --git a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs index 3ddd70fcc..4e8e6cbf5 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs @@ -23,6 +23,12 @@ pub struct IndexOfUDF { signature: Signature, } +impl Default for IndexOfUDF { + fn default() -> Self { + Self::new() + } +} + impl IndexOfUDF { pub fn new() -> Self { let signature = Signature::any(2, Volatility::Immutable); diff --git a/vegafusion-datafusion-udfs/src/udfs/array/span.rs b/vegafusion-datafusion-udfs/src/udfs/array/span.rs index a188c331e..5a6760ea2 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/span.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/span.rs @@ -18,6 +18,12 @@ pub struct SpanUDF { signature: Signature, } +impl Default for SpanUDF { + fn default() -> Self { + Self::new() + } +} + impl SpanUDF { pub fn new() -> Self { let signature = Signature::uniform( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs index c0af94281..971f29705 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs @@ -11,6 +11,12 @@ pub struct DateAddTzUDF { signature: Signature, } +impl Default for DateAddTzUDF { + fn default() -> Self { + Self::new() + } +} + impl DateAddTzUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs index 99b63d2ea..e322b92ad 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs @@ -15,6 +15,12 @@ pub struct DatePartTzUDF { signature: Signature, } +impl Default for DatePartTzUDF { + fn default() -> Self { + Self::new() + } +} + impl DatePartTzUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs index 96a7933e3..ff00ecb9d 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs @@ -18,6 +18,12 @@ pub struct DateToUtcTimestampUDF { signature: Signature, } +impl Default for DateToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl DateToUtcTimestampUDF { pub fn new() -> Self { let signature = Signature::exact( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs index a2d68651a..6035210bb 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs @@ -10,6 +10,12 @@ pub struct DateTruncTzUDF { signature: Signature, } +impl Default for DateTruncTzUDF { + fn default() -> Self { + Self::new() + } +} + impl DateTruncTzUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs index 20331e20b..57fc3903c 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs @@ -14,6 +14,12 @@ pub struct EpochMsToUtcTimestampUDF { signature: Signature, } +impl Default for EpochMsToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl EpochMsToUtcTimestampUDF { pub fn new() -> Self { let signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs index 7c5d2882b..e020f521b 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs @@ -17,6 +17,12 @@ pub struct FormatTimestampUDF { signature: Signature, } +impl Default for FormatTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl FormatTimestampUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs index 7d68c02d9..974e19020 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs @@ -22,6 +22,12 @@ pub struct FromUtcTimestampUDF { signature: Signature, } +impl Default for FromUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl FromUtcTimestampUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs index 52f365f30..cf139b824 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs @@ -18,6 +18,12 @@ pub struct MakeUtcTimestampUDF { signature: Signature, } +impl Default for MakeUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl MakeUtcTimestampUDF { pub fn new() -> Self { let signature = Signature::exact( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs index 13571862f..3c1a3d757 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs @@ -49,10 +49,7 @@ pub fn process_input_datetime( let array: Int64Array = unary(array, |v| (v as i64) * ms_per_day); Arc::new(array) as ArrayRef as _ } - DataType::Date64 => { - let int_array = cast(arg, &DataType::Int64).unwrap(); - int_array - } + DataType::Date64 => (cast(arg, &DataType::Int64).unwrap()) as _, DataType::Int64 => arg.clone(), DataType::Float64 => cast(arg, &DataType::Int64).expect("Failed to cast float to int"), _ => { diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs index 981acb685..bb8d633ee 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs @@ -334,6 +334,12 @@ pub struct StrToUtcTimestampUDF { signature: Signature, } +impl Default for StrToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl StrToUtcTimestampUDF { pub fn new() -> Self { let signature = diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs index 423fc4cf4..517d83ade 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs @@ -228,6 +228,12 @@ pub struct TimeunitStartUDF { signature: Signature, } +impl Default for TimeunitStartUDF { + fn default() -> Self { + Self::new() + } +} + impl TimeunitStartUDF { pub fn new() -> Self { let make_sig = |timestamp_dtype: DataType| -> TypeSignature { diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs index 1a88dd8c6..de4c8e8f4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs @@ -21,6 +21,12 @@ pub struct ToUtcTimestampUDF { signature: Signature, } +impl Default for ToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl ToUtcTimestampUDF { pub fn new() -> Self { // Signature should be (Timestamp, UTF8), but specifying Timestamp in the signature diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs index d48ae1b91..3a19ff217 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs @@ -15,6 +15,12 @@ pub struct UtcTimestampToEpochUDF { signature: Signature, } +impl Default for UtcTimestampToEpochUDF { + fn default() -> Self { + Self::new() + } +} + impl UtcTimestampToEpochUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs index 7df843c86..11fdaf907 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs @@ -17,6 +17,12 @@ pub struct UtcTimestampToStrUDF { signature: Signature, } +impl Default for UtcTimestampToStrUDF { + fn default() -> Self { + Self::new() + } +} + impl UtcTimestampToStrUDF { pub fn new() -> Self { let signature = Signature::one_of( diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs index 00729d4cd..367088c03 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs @@ -15,6 +15,12 @@ pub struct IsFiniteUDF { signature: Signature, } +impl Default for IsFiniteUDF { + fn default() -> Self { + Self::new() + } +} + impl IsFiniteUDF { pub fn new() -> Self { let signature = Signature::any(1, Volatility::Immutable); diff --git a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs index 64d61e823..24f9f47c9 100644 --- a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs @@ -87,12 +87,10 @@ impl ScalarUDFImpl for GetObjectMemberUDF { struct_array.column(self.field_index).clone(), )) } - _ => { - return Err(DataFusionError::Internal(format!( - "Unexpected object type for member access: {:?}", - arg.data_type() - ))); - } + _ => Err(DataFusionError::Internal(format!( + "Unexpected object type for member access: {:?}", + arg.data_type() + ))), } } } diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index a1b35aec0..ff7343ded 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -483,7 +483,7 @@ impl TaskCall for DataValuesTask { // (which generates its own data) if values_table.num_rows() == 0 { if let Some(pipeline) = &self.pipeline { - if let Some(first_tx) = pipeline.transforms.get(0) { + if let Some(first_tx) = pipeline.transforms.first() { if !matches!(first_tx.transform_kind(), TransformKind::Sequence(_)) { return Ok((TaskValue::Table(values_table), Default::default())); } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs index 024c3079f..cdfb61435 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs @@ -108,7 +108,7 @@ pub fn vl_selection_resolve_fn( } }; - let values = props.entry(field.field.clone()).or_insert_with(Vec::new); + let values = props.entry(field.field.clone()).or_default(); values.extend(value.clone()); } } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs index 7701bc7f9..1a6cade0f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -62,7 +62,7 @@ fn extract_timestamp_arg( schema: &DFSchema, default_input_tz: &str, ) -> Result { - if let Some(arg) = args.get(0) { + if let Some(arg) = args.first() { Ok(match arg.get_type(schema)? { DataType::Date32 => Expr::Cast(expr::Cast { expr: Box::new(arg.clone()), diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 6e1cbcc44..4c6b70461 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -150,7 +150,7 @@ pub fn compile_call( })) } VegaFusionCallable::Data(callee) => { - if let Some(v) = node.arguments.get(0) { + if let Some(v) = node.arguments.first() { match v.expr() { expression::Expr::Literal(Literal { value: Some(literal::Value::String(name)), diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 244f7c203..b77420a06 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -71,7 +71,7 @@ mod test_compile { }; use std::collections::HashMap; use std::convert::TryFrom; - use std::ops::Deref; + use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, Field, Schema}; use vegafusion_common::column::flat_col; diff --git a/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs b/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs index f83ac8ae0..a32183aa0 100644 --- a/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs +++ b/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs @@ -27,9 +27,9 @@ impl DestringifySelectionDatetimesVisitor { impl MutChartVisitor for DestringifySelectionDatetimesVisitor { fn visit_data(&mut self, data: &mut DataSpec, _scope: &[u32]) -> Result<()> { if let Some(Value::Array(values)) = &data.values { - if let Some(Value::Object(value0)) = values.get(0) { + if let Some(Value::Object(value0)) = values.first() { let columns: HashSet<_> = value0.keys().cloned().collect(); - let store_columns: HashSet<_> = vec!["unit", "fields", "values"] + let store_columns: HashSet<_> = ["unit", "fields", "values"] .iter() .map(|f| f.to_string()) .collect(); @@ -37,7 +37,7 @@ impl MutChartVisitor for DestringifySelectionDatetimesVisitor { // We have a selection store dataset with no transforms // Extract the values array if let Value::Array(values) = &value0["values"] { - if let Some(Value::Array(values)) = values.get(0) { + if let Some(Value::Array(values)) = values.first() { // Nested array, as in the case of an interval selection let is_date_str: Vec<_> = values .iter() diff --git a/vegafusion-runtime/src/task_graph/runtime.rs b/vegafusion-runtime/src/task_graph/runtime.rs index ced685d57..ce5d01e54 100644 --- a/vegafusion-runtime/src/task_graph/runtime.rs +++ b/vegafusion-runtime/src/task_graph/runtime.rs @@ -1023,8 +1023,8 @@ impl ChartState { // Filter to update nodes in the comm plan let indices: Vec<_> = indices .iter() + .filter(|&node| server_to_client.contains(node)) .cloned() - .filter(|node| server_to_client.contains(node)) .collect(); let response_task_values = runtime diff --git a/vegafusion-runtime/src/transform/fold.rs b/vegafusion-runtime/src/transform/fold.rs index 9e6a3da26..579a7a5df 100644 --- a/vegafusion-runtime/src/transform/fold.rs +++ b/vegafusion-runtime/src/transform/fold.rs @@ -21,7 +21,7 @@ impl TransformTrait for Fold { let key_col = unescape_field( &self .r#as - .get(0) + .first() .cloned() .unwrap_or_else(|| "key".to_string()), ); diff --git a/vegafusion-runtime/src/transform/timeunit.rs b/vegafusion-runtime/src/transform/timeunit.rs index 477b77fb1..ad99aa591 100644 --- a/vegafusion-runtime/src/transform/timeunit.rs +++ b/vegafusion-runtime/src/transform/timeunit.rs @@ -255,18 +255,18 @@ fn timeunit_custom_udf( default_input_tz: &String, local_tz: &Option, ) -> Result<(Expr, (i32, String))> { - let units_mask = vec![ - units_set.contains(&TimeUnitUnit::Year), // 0 - units_set.contains(&TimeUnitUnit::Quarter), // 1 - units_set.contains(&TimeUnitUnit::Month), // 2 - units_set.contains(&TimeUnitUnit::Date), // 3 - units_set.contains(&TimeUnitUnit::Week), // 4 - units_set.contains(&TimeUnitUnit::Day), // 5 - units_set.contains(&TimeUnitUnit::DayOfYear), // 6 - units_set.contains(&TimeUnitUnit::Hours), // 7 - units_set.contains(&TimeUnitUnit::Minutes), // 8 - units_set.contains(&TimeUnitUnit::Seconds), // 9 - units_set.contains(&TimeUnitUnit::Milliseconds), // 10 + let units_mask = [ + units_set.contains(&TimeUnitUnit::Year), // 0 + units_set.contains(&TimeUnitUnit::Quarter), // 1 + units_set.contains(&TimeUnitUnit::Month), // 2 + units_set.contains(&TimeUnitUnit::Date), // 3 + units_set.contains(&TimeUnitUnit::Week), // 4 + units_set.contains(&TimeUnitUnit::Day), // 5 + units_set.contains(&TimeUnitUnit::DayOfYear), // 6 + units_set.contains(&TimeUnitUnit::Hours), // 7 + units_set.contains(&TimeUnitUnit::Minutes), // 8 + units_set.contains(&TimeUnitUnit::Seconds), // 9 + units_set.contains(&TimeUnitUnit::Milliseconds), ]; let timeunit_start_udf = &TIMEUNIT_START_UDF; diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 8b29f5c25..94801421b 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -276,6 +276,12 @@ pub struct EpochMsToUtcTimestampUDF { signature: Signature, } +impl Default for EpochMsToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + impl EpochMsToUtcTimestampUDF { pub fn new() -> Self { let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index 019a8ac03..24de7dd3c 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -115,11 +115,11 @@ impl Connection for DataFusionConnection { async fn tables(&self) -> Result> { let catalog_names = self.ctx.catalog_names(); - let first_catalog_name = catalog_names.get(0).unwrap(); + let first_catalog_name = catalog_names.first().unwrap(); let catalog = self.ctx.catalog(first_catalog_name).unwrap(); let schema_provider_names = catalog.schema_names(); - let first_schema_provider_name = schema_provider_names.get(0).unwrap(); + let first_schema_provider_name = schema_provider_names.first().unwrap(); let schema_provider = catalog.schema(first_schema_provider_name).unwrap(); let mut tables: HashMap = HashMap::new(); @@ -141,7 +141,7 @@ impl Connection for DataFusionConnection { let batch_schema = if table.batches.is_empty() { None } else { - Some(table.batches.get(0).unwrap().schema()) + Some(table.batches.first().unwrap().schema()) }; // Create memtable diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index 695bbf5c0..23eb6d160 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -136,7 +136,7 @@ impl ExecutionPlan for PyDatasourceExec { .collect::>(); let args = PyTuple::new(py, vec![column_names.into_py(py)]); let pa_table = self.db.py_datasource.call_method1(py, "fetch", args)?; - let table = VegaFusionTable::from_pyarrow(py, &pa_table.as_ref(py))?; + let table = VegaFusionTable::from_pyarrow(py, pa_table.as_ref(py))?; Ok(table) }) .map_err(|err| DataFusionError::Execution(err.to_string()))?; diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index ea0d6083f..7d6eacdb8 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -1471,7 +1471,7 @@ fn query_chain_to_cte(queries: &[Query], prefix: &str) -> Query { fn parse_sql_query(query: &str, dialect: &Dialect) -> Result { let statements: Vec = Parser::parse_sql(dialect.parser_dialect().as_ref(), query)?; - if let Some(statement) = statements.get(0) { + if let Some(statement) = statements.first() { if let Statement::Query(box_query) = statement { let query: &Query = box_query.as_ref(); Ok(query.clone()) diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index 3d372f118..af7b15372 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -297,7 +297,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -311,7 +311,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -423,11 +423,11 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec!["min", "max", "count", "avg", "sum"] + aggregate_functions: ["min", "max", "count", "avg", "sum"] .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -568,11 +568,11 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec!["min", "max", "count", "avg", "sum", "median", "corr"] + aggregate_functions: ["min", "max", "count", "avg", "sum", "median", "corr"] .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -689,7 +689,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -704,7 +704,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -914,7 +914,7 @@ impl Dialect { .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1013,7 +1013,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1028,7 +1028,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1167,11 +1167,11 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec!["min", "max", "count", "avg", "sum", "var_pop", "stddev_pop"] + aggregate_functions: ["min", "max", "count", "avg", "sum", "var_pop", "stddev_pop"] .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1272,7 +1272,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1286,7 +1286,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1425,7 +1425,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1439,7 +1439,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1574,7 +1574,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1589,7 +1589,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", diff --git a/vegafusion-wasm/src/lib.rs b/vegafusion-wasm/src/lib.rs index c28304ae9..1e1c20b75 100644 --- a/vegafusion-wasm/src/lib.rs +++ b/vegafusion-wasm/src/lib.rs @@ -268,8 +268,8 @@ impl MsgReceiver { // Filter to update nodes in the comm plan let updated_nodes: Vec<_> = updated_nodes .iter() + .filter(|&node| server_to_client.contains(node)) .cloned() - .filter(|node| server_to_client.contains(node)) .collect(); let request_msg = QueryRequest { @@ -314,8 +314,8 @@ impl MsgReceiver { // Filter to update nodes in the comm plan let updated_nodes: Vec<_> = updated_nodes .iter() + .filter(|&node| server_to_client.contains(node)) .cloned() - .filter(|node| server_to_client.contains(node)) .collect(); if !updated_nodes.is_empty() { From 46b506689d2e76c8fcdb2cfb79b47248785de67f Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 16:46:10 -0500 Subject: [PATCH 34/44] Fix isFinite udf --- vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs index 367088c03..2769c9105 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs @@ -1,3 +1,4 @@ +use datafusion_physical_expr::udf::ScalarUDF; use std::any::Any; use std::sync::Arc; use vegafusion_common::arrow::array::{BooleanArray, Float32Array, Float64Array}; @@ -76,3 +77,7 @@ impl ScalarUDFImpl for IsFiniteUDF { } } } + +lazy_static! { + pub static ref ISFINITE_UDF: ScalarUDF = ScalarUDF::from(IsFiniteUDF::new()); +} From a8536dd1d272367d6b397562f359f80b242feecf Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 17:19:45 -0500 Subject: [PATCH 35/44] length should return f64 --- .../builtin_functions/array/length.rs | 6 +++-- .../src/expression/compiler/member.rs | 23 +++---------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs index c89d07afb..6bb710190 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs @@ -13,7 +13,7 @@ pub fn length_transform( .get_type(schema) .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; - match dtype { + let len_expr = match dtype { DataType::Utf8 | DataType::LargeUtf8 => Ok(Expr::Cast(expr::Cast { expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), @@ -32,7 +32,9 @@ pub fn length_transform( "length function support array and string arguments. Received argument with type {:?}", dtype ))), - } + }?; + + Ok(len_expr.cast_to(&DataType::Float64, schema)?) } else { Err(VegaFusionError::parse(format!( "length requires a single argument. Received {} arguments", diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index b9e0913f0..0b153a6a8 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -1,3 +1,4 @@ +use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; @@ -84,26 +85,8 @@ pub fn compile_member( } } _ => { - if property_string == "length" && matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) - { - // Special case to treat foo.length as length(foo) on a string - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn( - BuiltinScalarFunction::CharacterLength, - ), - args: vec![compiled_object], - }) - } else if property_string == "length" - && matches!( - dtype, - DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) - ) - { - // Special case to treat foo.length as length(foo) on an array - Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), - args: vec![compiled_object], - }) + if property_string == "length" { + length_transform(&[compiled_object], schema)? } else if matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { if let Some(index) = index { // SQL substr function is 1-indexed so add one From 201cc9d01bc182e2804d0140bdca07afe8c9277c Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 17:30:11 -0500 Subject: [PATCH 36/44] Bump manylinux to 2.24 --- .github/workflows/build_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 6d72e8f44..01a5be4e4 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -108,7 +108,7 @@ jobs: uses: messense/maturin-action@9357a23898d2fca5ef8da7ac38aaaedab4ef218f # pin@v1.40.2 with: command: build - manylinux: manylinux_2_17 + manylinux: manylinux_2_24 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip - name: Upload artifacts @@ -140,7 +140,7 @@ jobs: uses: messense/maturin-action@6d52485c3b3044e20b4c8ee6ce6f61e20a7645b0 # pin@v1 with: command: build - manylinux: manylinux_2_17 + manylinux: manylinux_2_24 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu - name: Upload artifacts @@ -318,7 +318,7 @@ jobs: run: | ls -la python -m pip install vegafusion-*.whl - python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl + python -m pip install vegafusion_python_embed-*manylinux_2_24_x86_64*.whl python -m pip install pytest vega-datasets polars duckdb==0.9.2 "vl-convert-python>=1.0.1rc1" scikit-image pandas==2.0 - name: Test vegafusion working-directory: python/vegafusion/ @@ -431,7 +431,7 @@ jobs: - name: install wheels run: | ls -la target/wheels/ - pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl + pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_24_x86_64*.whl pixi run pip install target/wheels/vegafusion-*.whl pixi run pip install target/wheels/vegafusion_jupyter-*.whl pixi run pip install chromedriver-binary-auto From fdfc6d591e4917cc9be3235e7094190a36008b37 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 17:34:27 -0500 Subject: [PATCH 37/44] Only bump manylinux for arm64 --- .github/workflows/build_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 01a5be4e4..b5dd98ca8 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -108,7 +108,7 @@ jobs: uses: messense/maturin-action@9357a23898d2fca5ef8da7ac38aaaedab4ef218f # pin@v1.40.2 with: command: build - manylinux: manylinux_2_24 + manylinux: 2014 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip - name: Upload artifacts @@ -140,7 +140,7 @@ jobs: uses: messense/maturin-action@6d52485c3b3044e20b4c8ee6ce6f61e20a7645b0 # pin@v1 with: command: build - manylinux: manylinux_2_24 + manylinux: 2_24 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu - name: Upload artifacts @@ -318,7 +318,7 @@ jobs: run: | ls -la python -m pip install vegafusion-*.whl - python -m pip install vegafusion_python_embed-*manylinux_2_24_x86_64*.whl + python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl python -m pip install pytest vega-datasets polars duckdb==0.9.2 "vl-convert-python>=1.0.1rc1" scikit-image pandas==2.0 - name: Test vegafusion working-directory: python/vegafusion/ @@ -431,7 +431,7 @@ jobs: - name: install wheels run: | ls -la target/wheels/ - pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_24_x86_64*.whl + pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl pixi run pip install target/wheels/vegafusion-*.whl pixi run pip install target/wheels/vegafusion_jupyter-*.whl pixi run pip install chromedriver-binary-auto From 0b9b8339f46ca97a2ab4004179af1237577ab002 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 18:42:05 -0500 Subject: [PATCH 38/44] cargo update --- Cargo.lock | 1806 +++++++++++++++++++++++++--------------------------- 1 file changed, 882 insertions(+), 924 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4c45d086..63515c9a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -19,22 +19,23 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" dependencies = [ "cfg-if", "const-random", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -56,9 +57,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "android-tzdata" @@ -83,49 +84,57 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.2.6" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", - "is-terminal", + "colorchoice", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "anstyle-wincon" -version = "0.2.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arrayref" @@ -135,9 +144,9 @@ checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" @@ -172,7 +181,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "half 2.2.1", + "half", "num", ] @@ -188,8 +197,8 @@ dependencies = [ "arrow-schema", "chrono", "chrono-tz", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", "num", ] @@ -200,7 +209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" dependencies = [ "bytes", - "half 2.2.1", + "half", "num", ] @@ -218,7 +227,7 @@ dependencies = [ "base64", "chrono", "comfy-table", - "half 2.2.1", + "half", "lexical-core", "num", ] @@ -250,7 +259,7 @@ checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" dependencies = [ "arrow-buffer", "arrow-schema", - "half 2.2.1", + "half", "num", ] @@ -281,8 +290,8 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "half 2.2.1", - "indexmap 2.0.0", + "half", + "indexmap 2.2.3", "lexical-core", "num", "serde", @@ -300,7 +309,7 @@ dependencies = [ "arrow-data", "arrow-schema", "arrow-select", - "half 2.2.1", + "half", "num", ] @@ -315,8 +324,8 @@ dependencies = [ "arrow-buffer", "arrow-data", "arrow-schema", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", ] [[package]] @@ -325,7 +334,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" dependencies = [ - "bitflags 2.1.0", + "bitflags 2.4.2", ] [[package]] @@ -355,14 +364,14 @@ dependencies = [ "arrow-select", "num", "regex", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] name = "assert_cmd" -version = "2.0.10" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b2340f55d9661d76793b2bfc2eb0e62689bd79d067a95707ea762afd5e9dd" +checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" dependencies = [ "anstyle", "bstr", @@ -385,20 +394,33 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 2.5.3", "futures-core", ] +[[package]] +name = "async-channel" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" +dependencies = [ + "concurrent-queue", + "event-listener 5.1.0", + "event-listener-strategy 0.5.0", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" -version = "0.4.0" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "bzip2", "flate2", @@ -408,36 +430,36 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.12.3+zstd.1.5.2", - "zstd-safe 6.0.4+zstd.1.5.4", + "zstd", + "zstd-safe", ] [[package]] name = "async-executor" -version = "1.5.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17adb73da160dfb475c183343c8cccd80721ea5a605d3eb57125f0a7b7a92d0b" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" dependencies = [ - "async-lock", + "async-lock 3.3.0", "async-task", "concurrent-queue", - "fastrand", - "futures-lite", + "fastrand 2.0.1", + "futures-lite 2.2.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel", + "async-channel 2.2.0", "async-executor", - "async-io", - "async-lock", + "async-io 2.3.1", + "async-lock 3.3.0", "blocking", - "futures-lite", + "futures-lite 2.2.0", "once_cell", ] @@ -447,27 +469,57 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ - "async-lock", + "async-lock 2.8.0", "autocfg", "cfg-if", "concurrent-queue", - "futures-lite", + "futures-lite 1.13.0", "log", "parking", - "polling", - "rustix", + "polling 2.8.0", + "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f97ab0c5b00a7cdbe5a371b9a782ee7be1316095885c8a4ea1daf490eb0ef65" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.2.0", + "parking", + "polling 3.5.0", + "rustix 0.38.31", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener", + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", + "pin-project-lite", ] [[package]] @@ -478,7 +530,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -488,15 +540,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ "async-attributes", - "async-channel", + "async-channel 1.9.0", "async-global-executor", - "async-io", - "async-lock", + "async-io 1.13.0", + "async-lock 2.8.0", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite", + "futures-lite 1.13.0", "gloo-timers", "kv-log-macro", "log", @@ -510,9 +562,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", @@ -521,37 +573,37 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "async-task" -version = "4.4.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "atomic-waker" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "atty" @@ -581,9 +633,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.12" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -609,9 +661,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", "bytes", @@ -626,24 +678,24 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", "cfg-if", "libc", - "miniz_oxide 0.7.1", + "miniz_oxide", "object", "rustc-demangle", ] [[package]] name = "base64" -version = "0.21.2" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -653,9 +705,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.1.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70beb79cbb5ce9c4f8e20849978f34225931f665bb49efa6982875a4d5facb3" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "blake2" @@ -668,16 +720,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.3.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "digest", ] [[package]] @@ -691,23 +742,25 @@ dependencies = [ [[package]] name = "blocking" -version = "1.3.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c67b173a56acffd6d2326fb7ab938ba0b00a71480e14902b2591c87bc5741e8" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel", - "async-lock", + "async-channel 2.2.0", + "async-lock 3.3.0", "async-task", - "atomic-waker", - "fastrand", - "futures-lite", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.2.0", + "piper", + "tracing", ] [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -716,9 +769,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -726,39 +779,38 @@ dependencies = [ [[package]] name = "bstr" -version = "1.4.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "once_cell", - "regex-automata 0.1.10", + "regex-automata", "serde", ] [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2" @@ -789,11 +841,11 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "3286b845d0fccbdd15af433f61c5970e711987036cb468f437ff6badd70f4e24" dependencies = [ - "jobserver", + "libc", ] [[package]] @@ -820,14 +872,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.0", + "windows-targets 0.52.3", ] [[package]] name = "chrono-tz" -version = "0.8.1" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa48fa079165080f11d7753fd0bc175b7d391f276b965fe4b55bfad67856e463" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", "chrono-tz-build", @@ -836,9 +888,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" dependencies = [ "parse-zoneinfo", "phf", @@ -847,9 +899,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -858,25 +910,25 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 1.8.2", + "half", ] [[package]] name = "clap" -version = "3.2.23" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", @@ -886,38 +938,36 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", - "bitflags 1.3.2", - "clap_lex 0.4.1", + "clap_lex 0.7.0", "strsim", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -931,19 +981,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "combine" @@ -957,35 +1003,20 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum 0.24.1", - "strum_macros 0.24.3", + "strum 0.25.0", + "strum_macros 0.25.3", "unicode-width", ] -[[package]] -name = "concolor-override" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] - [[package]] name = "concurrent-queue" -version = "2.1.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -1002,37 +1033,35 @@ dependencies = [ [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom", "once_cell", - "proc-macro-hack", "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1040,24 +1069,24 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1072,7 +1101,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 3.2.25", "criterion-plot", "futures", "itertools 0.10.5", @@ -1102,46 +1131,37 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.8.0", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1161,9 +1181,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -1173,68 +1193,24 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.31", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.31", -] - [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.3", "lock_api", "once_cell", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.9", ] [[package]] @@ -1266,9 +1242,9 @@ dependencies = [ "flate2", "futures", "glob", - "half 2.2.1", - "hashbrown 0.14.0", - "indexmap 2.0.0", + "half", + "hashbrown 0.14.3", + "indexmap 2.2.3", "itertools 0.12.1", "log", "num_cpus", @@ -1284,7 +1260,7 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.0", + "zstd", ] [[package]] @@ -1299,7 +1275,7 @@ dependencies = [ "arrow-buffer", "arrow-schema", "chrono", - "half 2.2.1", + "half", "libc", "num_cpus", "object_store", @@ -1320,7 +1296,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "log", "object_store", "parking_lot 0.12.1", @@ -1386,10 +1362,10 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "itertools 0.12.1", "log", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] @@ -1412,10 +1388,10 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", "hex", - "indexmap 2.0.0", + "indexmap 2.2.3", "itertools 0.12.1", "log", "md-5", @@ -1446,9 +1422,9 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "futures", - "half 2.2.1", - "hashbrown 0.14.0", - "indexmap 2.0.0", + "half", + "hashbrown 0.14.3", + "indexmap 2.2.3", "itertools 0.12.1", "log", "once_cell", @@ -1519,10 +1495,11 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dssim" -version = "3.2.4" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea54acfcd238d680c2c720a5eef56d210aaabc7582a48615602bcdf0c9a5894" +checksum = "4ddc15a051da6e0d2611493ba1b5d56e3739d6d9df8a36ec2406c2b6f5145f33" dependencies = [ + "crossbeam-channel", "dssim-core", "getopts", "imgref", @@ -1534,42 +1511,42 @@ dependencies = [ [[package]] name = "dssim-core" -version = "3.2.5" +version = "3.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0286ae9482caf2cc3fb8ce55c5ff7dc1986425f36fd8423f00e38a83b4169fb" +checksum = "fafad37c1f4f168243f3ac1b4cae0d358c528ac695670100337314e38d54b486" dependencies = [ "imgref", - "itertools 0.10.5", + "itertools 0.12.1", "rayon", "rgb", ] [[package]] name = "dunce" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "either" -version = "1.8.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -1586,36 +1563,67 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.0" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" dependencies = [ - "cc", - "libc", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] name = "event-listener" -version = "2.5.3" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "b7ad6fd685ce13acd6d9541a30f6db6567a7a24c9ffd4ba2955d29e3f22c8b27" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" +dependencies = [ + "event-listener 5.1.0", + "pin-project-lite", +] [[package]] name = "fallible_collections" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acf77205554f3cfeca94a4b910e159ad9824e8c2d164de02b3f12495cc1074d" +checksum = "a88c69768c0a15262df21899142bc6df9b9b823546d4b4b9a7bc2d6c448ec6fd" dependencies = [ "hashbrown 0.13.2", ] @@ -1629,6 +1637,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1637,9 +1651,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.1.21" +version = "23.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1647,12 +1661,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", - "miniz_oxide 0.6.2", + "miniz_oxide", ] [[package]] @@ -1688,7 +1702,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -1699,18 +1713,18 @@ checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1723,9 +1737,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1733,15 +1747,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1750,17 +1764,17 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -1769,40 +1783,53 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1837,9 +1864,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -1850,9 +1877,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -1874,9 +1901,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -1884,7 +1911,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.2.3", "slab", "tokio", "tokio-util", @@ -1893,16 +1920,11 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" - -[[package]] -name = "half" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" dependencies = [ + "cfg-if", "crunchy", "num-traits", ] @@ -1924,9 +1946,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ "ahash", "allocator-api2", @@ -1949,18 +1971,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "379dada1584ad501b383485dd706b8afb7a70fcbc7f4da7d780638a5a6124a60" [[package]] name = "hex" @@ -1968,11 +1981,20 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1981,9 +2003,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1992,9 +2014,9 @@ dependencies = [ [[package]] name = "http-range-header" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" @@ -2004,9 +2026,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -2016,9 +2038,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2031,7 +2053,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2066,33 +2088,32 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] name = "idna" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2100,9 +2121,9 @@ dependencies = [ [[package]] name = "imgref" -version = "1.9.4" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cf49df1085dcfb171460e4592597b84abe50d900fb83efb6e41b20fefd6c2c" +checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" [[package]] name = "indexmap" @@ -2116,12 +2137,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -2150,31 +2171,30 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi 0.3.8", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.7.2" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.6" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.45.0", + "hermit-abi 0.3.8", + "libc", + "windows-sys 0.52.0", ] [[package]] @@ -2206,9 +2226,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jni" @@ -2232,38 +2252,29 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - [[package]] name = "jpeg-decoder" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" dependencies = [ "rayon", ] [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] [[package]] name = "json-patch" -version = "1.0.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f54898088ccb91df1b492cc80029a6fdf1c48ca0db7c6822a8babad69c94658" +checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6" dependencies = [ "serde", "serde_json", @@ -2288,19 +2299,20 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lcms2" -version = "5.5.0" +version = "6.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e736cf1cff040514bc97fd74ab45d622bba780e8799e43efba3269cecaaa45b" +checksum = "2249d276363ddd7591b2e049d61d3d2be5efe8acabd670f4e9480e5fecbb5a60" dependencies = [ + "bytemuck", "foreign-types", "lcms2-sys", ] [[package]] name = "lcms2-sys" -version = "4.0.0" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908236fb4fa8f210c5aefbec04e0bd6fea22b7f2796d479130037244d39078a8" +checksum = "99c5fe876968a1685074a0978632afd4629d7ba40f0ac563388c1a296ce12536" dependencies = [ "cc", "dunce", @@ -2380,30 +2392,27 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libm" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] -name = "link-cplusplus" -version = "1.0.8" +name = "linux-raw-sys" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "load_image" -version = "3.0.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f87f0f354b901237de8e6d4c0f050a3aa85b83a458fea9796f993a08a67c3cc" +checksum = "20c73d7e075d05bdcc5ff4cf16b35b50b2be696b93c1be1b32201fb32c35a814" dependencies = [ "bytemuck", "imgref", @@ -2417,9 +2426,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -2427,9 +2436,9 @@ dependencies = [ [[package]] name = "lodepng" -version = "3.7.2" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0ad39f75bbaa4b10bb6f2316543632a8046a5bcf9c785488d79720b21f044f8" +checksum = "a42d298694b14401847de29abd44adf278b42e989e516deac7b72018400002d8" dependencies = [ "crc32fast", "fallible_collections", @@ -2453,14 +2462,14 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" dependencies = [ - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] name = "lz4_flex" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" dependencies = [ "twox-hash", ] @@ -2478,33 +2487,25 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -2533,18 +2534,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" -dependencies = [ - "adler", -] - -[[package]] -name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -2588,9 +2580,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -2599,28 +2591,27 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -2641,9 +2632,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -2651,19 +2642,19 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.8", "libc", ] [[package]] name = "object" -version = "0.31.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -2687,7 +2678,7 @@ dependencies = [ "quick-xml", "rand", "reqwest", - "ring 0.17.3", + "ring", "serde", "serde_json", "snafu", @@ -2699,9 +2690,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -2717,33 +2708,33 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] [[package]] name = "ordered-float" -version = "3.6.0" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a384337e997e6860ffbaa83708b2ef329fd8c54cb67a5f64d421e0f943254f" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" dependencies = [ "num-traits", ] [[package]] name = "os_str_bytes" -version = "6.5.0" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" [[package]] name = "parking" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -2763,7 +2754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.9", ] [[package]] @@ -2782,15 +2773,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.4.1", "smallvec", - "windows-sys 0.45.0", + "windows-targets 0.48.5", ] [[package]] @@ -2813,8 +2804,8 @@ dependencies = [ "chrono", "flate2", "futures", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", "lz4_flex", "num", "num-bigint", @@ -2825,7 +2816,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.0", + "zstd", ] [[package]] @@ -2845,34 +2836,34 @@ checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap 2.2.3", ] [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" dependencies = [ "phf_generator", "phf_shared", @@ -2880,9 +2871,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand", @@ -2890,31 +2881,31 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] @@ -2929,17 +2920,28 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -2950,24 +2952,24 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] [[package]] name = "polling" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e1f879b2998099c2d69ab9605d145d5b661195627eccc680002c4918a7fb6fa" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", @@ -2976,9 +2978,29 @@ dependencies = [ "libc", "log", "pin-project-lite", - "windows-sys 0.45.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.31", + "tracing", + "windows-sys 0.52.0", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2987,14 +3009,13 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c575290b64d24745b6c57a12a31465f0a66f3a4799686a6921526a33b0797965" +checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" dependencies = [ "anstyle", "difflib", "float-cmp", - "itertools 0.10.5", "normalize-line-endings", "predicates-core", "regex", @@ -3018,58 +3039,28 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" -dependencies = [ - "proc-macro2", - "syn 2.0.31", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "quote", - "version_check", + "syn 2.0.50", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", "prost-derive", @@ -3077,9 +3068,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck", @@ -3092,29 +3083,29 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.31", + "syn 2.0.50", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "prost-types" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ "prost", ] @@ -3130,15 +3121,16 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.9.0", + "memoffset", "parking_lot 0.12.1", + "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -3147,9 +3139,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" dependencies = [ "once_cell", "target-lexicon", @@ -3157,9 +3149,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" dependencies = [ "libc", "pyo3-build-config", @@ -3167,26 +3159,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "pyo3-macros-backend" -version = "0.20.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" dependencies = [ "heck", "proc-macro2", + "pyo3-build-config", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -3217,9 +3210,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -3256,9 +3249,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -3266,14 +3259,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -3287,48 +3278,36 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.5" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.8", - "regex-syntax 0.7.5", + "regex-automata", + "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - -[[package]] -name = "regex-automata" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - [[package]] name = "regex-syntax" version = "0.8.2" @@ -3337,15 +3316,15 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "relative-path" -version = "1.9.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" +checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64", "bytes", @@ -3371,6 +3350,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls", @@ -3387,9 +3367,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c50db2c7ccd815f976473dd7d0bde296f8c3b77c383acf4fc021cdcf10852b" +checksum = "88a3e86aa6053e59030e7ce2d2a3b258dd08fc2d337d52f73f6cb480f5858690" dependencies = [ "anyhow", "async-trait", @@ -3425,9 +3405,9 @@ dependencies = [ [[package]] name = "retry-policies" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a715dc4d0e8aea3085a9a94d76e79c79c7df7c9f6be609da841a6d2489ca3687" +checksum = "17dd00bff1d737c40dbcd47d4375281bf4c17933f9eef0a185fc7bacca23ecbd" dependencies = [ "anyhow", "chrono", @@ -3445,40 +3425,26 @@ dependencies = [ [[package]] name = "rgb" -version = "0.8.36" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" dependencies = [ "bytemuck", ] [[package]] name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - -[[package]] -name = "ring" -version = "0.17.3" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babe80d5c16becf6594aa32ad2be8fe08498e7ae60b77de8df700e67f191d7e" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys 0.48.0", + "spin", + "untrusted", + "windows-sys 0.52.0", ] [[package]] @@ -3506,7 +3472,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.31", + "syn 2.0.50", "unicode-ident", ] @@ -3519,7 +3485,7 @@ dependencies = [ "quote", "rand", "rustc_version", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -3539,26 +3505,39 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.6" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys 0.45.0", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.16.20", + "ring", "rustls-webpki", "sct", ] @@ -3577,34 +3556,34 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring", + "untrusted", ] [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -3632,24 +3611,18 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.5" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring", + "untrusted", ] [[package]] @@ -3677,43 +3650,43 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "seq-macro" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.2.3", "itoa", "ryu", "serde", @@ -3721,9 +3694,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -3742,9 +3715,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -3753,30 +3726,30 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "snafu" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", "snafu-derive", @@ -3784,9 +3757,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck", "proc-macro2", @@ -3796,15 +3769,15 @@ dependencies = [ [[package]] name = "snap" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -3812,20 +3785,14 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -3859,7 +3826,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -3870,15 +3837,15 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "strum" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" [[package]] name = "strum" @@ -3891,15 +3858,15 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] @@ -3912,14 +3879,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -3934,9 +3901,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" dependencies = [ "proc-macro2", "quote", @@ -3972,9 +3939,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "task-local-extensions" @@ -3987,22 +3954,21 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", - "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", + "fastrand 2.0.1", + "rustix 0.38.31", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -4015,63 +3981,61 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "test-case" -version = "3.1.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a1d6e7bde536b0412f20765b76e921028059adfd1b90d8974d33fd3c91b25df" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" dependencies = [ "test-case-macros", ] [[package]] name = "test-case-core" -version = "3.1.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10394d5d1e27794f772b6fc854c7e91a2dc26e2cbf807ad523370c2a59c0cee" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" dependencies = [ "cfg-if", - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "test-case-macros" -version = "3.1.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb9a44b1c6a54c1ba58b152797739dba2a83ca74e18168a68c980eb142f9404" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", "test-case-core", ] [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -4082,7 +4046,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float 2.10.0", + "ordered-float 2.10.1", ] [[package]] @@ -4131,7 +4095,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.6", "tokio-macros", "windows-sys 0.48.0", ] @@ -4154,7 +4118,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -4169,9 +4133,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -4180,9 +4144,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -4194,9 +4158,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", @@ -4206,20 +4170,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.2.3", "serde", "serde_spanned", "toml_datetime", @@ -4266,7 +4230,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -4311,11 +4275,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "bytes", "futures-core", "futures-util", @@ -4341,11 +4305,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4353,38 +4316,38 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "treediff" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" +checksum = "4d127780145176e2b5d16611cc25a900150e86e9fd79d3bde6ff3a37359c9cb5" dependencies = [ "serde_json", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" @@ -4398,51 +4361,51 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unindent" @@ -4450,12 +4413,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -4464,9 +4421,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.3.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -4481,18 +4438,18 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", ] [[package]] name = "value-bag" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" +checksum = "126e423afe2dd9ac52142e7e9d5ce4135d7e13776c529d27fd6bc49f19e3280b" [[package]] name = "vegafusion-common" @@ -4524,7 +4481,7 @@ dependencies = [ "json-patch", "lazy_static", "num-complex", - "ordered-float 3.6.0", + "ordered-float 3.9.2", "petgraph", "prost", "prost-build", @@ -4562,7 +4519,7 @@ dependencies = [ "chrono-tz", "datafusion-physical-expr", "lazy_static", - "ordered-float 3.6.0", + "ordered-float 3.9.2", "regex", "vegafusion-common", ] @@ -4608,7 +4565,7 @@ dependencies = [ name = "vegafusion-runtime" version = "1.6.4" dependencies = [ - "async-lock", + "async-lock 2.8.0", "async-recursion", "async-trait", "base64", @@ -4634,7 +4591,7 @@ dependencies = [ "lru", "num-traits", "object_store", - "ordered-float 3.6.0", + "ordered-float 3.9.2", "prost", "prost-types", "regex", @@ -4662,7 +4619,7 @@ name = "vegafusion-server" version = "1.6.4" dependencies = [ "assert_cmd", - "clap 4.2.1", + "clap 4.5.1", "futures-util", "h2", "predicates", @@ -4754,15 +4711,15 @@ dependencies = [ [[package]] name = "waker-fn" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -4770,11 +4727,10 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -4786,9 +4742,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4796,24 +4752,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -4823,9 +4779,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4833,28 +4789,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "wasm-bindgen-test" -version = "0.3.34" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" +checksum = "143ddeb4f833e2ed0d252e618986e18bfc7b0e52f2d28d77d05b2f045dd8eb61" dependencies = [ "console_error_panic_hook", "js-sys", @@ -4866,19 +4822,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.34" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0734759ae6b3b1717d661fe4f016efcfb9828f5edb4520c18eaee05af3b43be9" +checksum = "a5211b7550606857312bba1d978a8ec75692eae187becc5e680444fffc5e6f89" dependencies = [ "proc-macro2", "quote", + "syn 2.0.50", ] [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -4904,9 +4861,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -4914,19 +4871,20 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.31", ] [[package]] @@ -4947,9 +4905,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -4961,12 +4919,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.52.3", ] [[package]] @@ -4984,7 +4942,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", ] [[package]] @@ -4993,7 +4951,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.3", ] [[package]] @@ -5013,32 +4971,32 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.3", + "windows_aarch64_msvc 0.52.3", + "windows_i686_gnu 0.52.3", + "windows_i686_msvc 0.52.3", + "windows_x86_64_gnu 0.52.3", + "windows_x86_64_gnullvm 0.52.3", + "windows_x86_64_msvc 0.52.3", ] [[package]] @@ -5049,15 +5007,15 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" [[package]] name = "windows_aarch64_msvc" @@ -5067,15 +5025,15 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" [[package]] name = "windows_i686_gnu" @@ -5085,15 +5043,15 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" [[package]] name = "windows_i686_msvc" @@ -5103,15 +5061,15 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" [[package]] name = "windows_x86_64_gnu" @@ -5121,15 +5079,15 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" [[package]] name = "windows_x86_64_gnullvm" @@ -5139,15 +5097,15 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" [[package]] name = "windows_x86_64_msvc" @@ -5157,21 +5115,21 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" [[package]] name = "winnow" -version = "0.4.1" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -5196,31 +5154,32 @@ dependencies = [ ] [[package]] -name = "zstd" -version = "0.12.3+zstd.1.5.2" +name = "zerocopy" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ - "zstd-safe 6.0.4+zstd.1.5.4", + "zerocopy-derive", ] [[package]] -name = "zstd" -version = "0.13.0" +name = "zerocopy-derive" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ - "zstd-safe 7.0.0", + "proc-macro2", + "quote", + "syn 2.0.50", ] [[package]] -name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +name = "zstd" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] @@ -5234,11 +5193,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] From 2b1981938f94293d4cd414702d2eaa8983bce455 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 18:52:25 -0500 Subject: [PATCH 39/44] reverts --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63515c9a8..46d06d13e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4871,9 +4871,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.4" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "which" From 491df151b98d5979b324590a84f41c52cce32471 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 24 Feb 2024 18:55:12 -0500 Subject: [PATCH 40/44] bump maturin action and manylinux for aarch64 --- .github/workflows/build_test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index b5dd98ca8..c3fed0693 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -105,7 +105,7 @@ jobs: with: cache-on-failure: True - name: Build wheels (Linux) - uses: messense/maturin-action@9357a23898d2fca5ef8da7ac38aaaedab4ef218f # pin@v1.40.2 + uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 with: command: build manylinux: 2014 @@ -137,10 +137,10 @@ jobs: run: | rustup target add aarch64-unknown-linux-gnu - name: Build arm64 wheels - uses: messense/maturin-action@6d52485c3b3044e20b4c8ee6ce6f61e20a7645b0 # pin@v1 + uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 with: command: build - manylinux: 2_24 + manylinux: 2_28 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu - name: Upload artifacts From 3d2214be5df0fe7cf3130a32b5aa8641559974b1 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 26 Feb 2024 08:27:52 -0500 Subject: [PATCH 41/44] Make use of protobuf-src optional in VegaFusion server --- vegafusion-server/Cargo.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vegafusion-server/Cargo.toml b/vegafusion-server/Cargo.toml index e42fe666e..3af027a72 100644 --- a/vegafusion-server/Cargo.toml +++ b/vegafusion-server/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" description = "VegaFusion Server" [features] -protobuf-src = [ "vegafusion-core/protobuf-src",] +protobuf-src = [ "vegafusion-core/protobuf-src", "dep:protobuf-src"] [dependencies] futures-util = "0.3.21" @@ -25,6 +25,7 @@ predicates = "3.0.2" [build-dependencies] tonic-build = "0.10.2" prost-build = "0.12.1" +protobuf-src = { version="1.1.0", optional = true } [dependencies.prost] workspace = true @@ -61,6 +62,3 @@ features = [ "tls",] [dependencies.clap] version = "4.2.1" features = [ "derive",] - -[target."cfg(target_family = \"unix\")".build-dependencies] -protobuf-src = "1.1.0" From 3eea54725bbbd1c6518202ef19ee00aeea2688e2 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 26 Feb 2024 08:33:47 -0500 Subject: [PATCH 42/44] fix feature flag switch --- vegafusion-server/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vegafusion-server/build.rs b/vegafusion-server/build.rs index cb0b48499..b40107494 100644 --- a/vegafusion-server/build.rs +++ b/vegafusion-server/build.rs @@ -1,5 +1,5 @@ fn main() -> Result<(), Box> { - #[cfg(target_family = "unix")] + #[cfg(feature = "protobuf-src")] std::env::set_var("PROTOC", protobuf_src::protoc()); tonic_build::compile_protos("proto/helloworld.proto")?; From ae39ee0dafd279c763c3f0332f54d11c9bd8ac8b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 26 Feb 2024 09:57:43 -0500 Subject: [PATCH 43/44] switch from dssim to pixelmatch crate for test image comparison --- Cargo.lock | 270 +++++++++--------- vegafusion-runtime/Cargo.toml | 2 +- .../tests/test_image_comparison.rs | 4 +- .../tests/util/vegajs_runtime/mod.rs | 70 ++--- 4 files changed, 159 insertions(+), 187 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46d06d13e..4402b26fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.8.9" @@ -686,7 +692,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.2", "object", "rustc-demangle", ] @@ -985,6 +991,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.0" @@ -1129,15 +1141,6 @@ dependencies = [ "itertools 0.10.5", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -1464,6 +1467,16 @@ dependencies = [ "sqlparser 0.43.1", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "deterministic-hash" version = "1.0.1" @@ -1493,40 +1506,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dssim" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddc15a051da6e0d2611493ba1b5d56e3739d6d9df8a36ec2406c2b6f5145f33" -dependencies = [ - "crossbeam-channel", - "dssim-core", - "getopts", - "imgref", - "load_image", - "lodepng", - "rayon", - "rgb", -] - -[[package]] -name = "dssim-core" -version = "3.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fafad37c1f4f168243f3ac1b4cae0d358c528ac695670100337314e38d54b486" -dependencies = [ - "imgref", - "itertools 0.12.1", - "rayon", - "rgb", -] - -[[package]] -name = "dunce" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" - [[package]] name = "either" version = "1.10.0" @@ -1666,7 +1645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.7.2", ] [[package]] @@ -1684,33 +1663,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" -dependencies = [ - "foreign-types-macros", - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-macros" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.50", -] - -[[package]] -name = "foreign-types-shared" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1853,15 +1805,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - [[package]] name = "getrandom" version = "0.2.12" @@ -1875,6 +1818,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gif" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.28.1" @@ -2120,10 +2073,23 @@ dependencies = [ ] [[package]] -name = "imgref" -version = "1.10.1" +name = "image" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" +checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational 0.3.2", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] [[package]] name = "indexmap" @@ -2254,9 +2220,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "jpeg-decoder" -version = "0.3.1" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" dependencies = [ "rayon", ] @@ -2297,29 +2263,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lcms2" -version = "6.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2249d276363ddd7591b2e049d61d3d2be5efe8acabd670f4e9480e5fecbb5a60" -dependencies = [ - "bytemuck", - "foreign-types", - "lcms2-sys", -] - -[[package]] -name = "lcms2-sys" -version = "4.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c5fe876968a1685074a0978632afd4629d7ba40f0ac563388c1a296ce12536" -dependencies = [ - "cc", - "dunce", - "libc", - "pkg-config", -] - [[package]] name = "lexical-core" version = "0.8.5" @@ -2408,22 +2351,6 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" -[[package]] -name = "load_image" -version = "3.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20c73d7e075d05bdcc5ff4cf16b35b50b2be696b93c1be1b32201fb32c35a814" -dependencies = [ - "bytemuck", - "imgref", - "jpeg-decoder", - "lcms2", - "lodepng", - "quick-error", - "rexif", - "rgb", -] - [[package]] name = "lock_api" version = "0.4.11" @@ -2532,6 +2459,25 @@ dependencies = [ "unicase", ] +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -2574,7 +2520,7 @@ dependencies = [ "num-complex", "num-integer", "num-iter", - "num-rational", + "num-rational 0.4.1", "num-traits", ] @@ -2618,6 +2564,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.1" @@ -2931,6 +2888,15 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pixelmatch" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b15774a2acbbefbf12db95ef5582f80a3c9628dc48b6397e563c08640799764" +dependencies = [ + "image", +] + [[package]] name = "pkg-config" version = "0.3.30" @@ -2965,6 +2931,18 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "polling" version = "2.8.0" @@ -3192,12 +3170,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quick-error" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - [[package]] name = "quick-xml" version = "0.31.0" @@ -3414,15 +3386,6 @@ dependencies = [ "rand", ] -[[package]] -name = "rexif" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49352965b70522af9085d7a8c2a6df7494713c67ac58b9af02bcff7fb4ca1483" -dependencies = [ - "num", -] - [[package]] name = "rgb" version = "0.8.37" @@ -3609,6 +3572,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.2.0" @@ -4049,6 +4018,17 @@ dependencies = [ "ordered-float 2.10.1", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.4", + "weezl", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -4579,7 +4559,6 @@ dependencies = [ "datafusion-optimizer", "datafusion-physical-expr", "deterministic-hash", - "dssim", "env_logger", "float-cmp", "futures", @@ -4592,6 +4571,7 @@ dependencies = [ "num-traits", "object_store", "ordered-float 3.9.2", + "pixelmatch", "prost", "prost-types", "regex", @@ -4875,6 +4855,12 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "which" version = "4.4.2" diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index 482a0b969..6d0fe00f6 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -42,7 +42,7 @@ futures-util = "0.3.21" rstest = "0.18.2" test-case = "3.1.0" base64 = "0.21.0" -dssim = "3.1.0" +pixelmatch = "0.1.0" rgb = "0.8.32" lodepng = "3.6.1" diff --git a/vegafusion-runtime/tests/test_image_comparison.rs b/vegafusion-runtime/tests/test_image_comparison.rs index 261919ae0..62c8d359c 100644 --- a/vegafusion-runtime/tests/test_image_comparison.rs +++ b/vegafusion-runtime/tests/test_image_comparison.rs @@ -754,8 +754,8 @@ mod test_vegalite_specs { case("vegalite/point_no_axis_domain_grid", 0.001), // (random function is non-deterministic so use higher tolerance) - case("vegalite/point_offset_random", 0.3), - case("vegalite/point_ordinal_bin_offset_random", 0.3), + case("vegalite/point_offset_random", 0.8), + case("vegalite/point_ordinal_bin_offset_random", 0.8), case("vegalite/point_ordinal_color", 0.001), case("vegalite/point_overlap", 0.001), diff --git a/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs b/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs index 1af836671..e32152bb7 100644 --- a/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs +++ b/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs @@ -1,5 +1,4 @@ use datafusion_common::ScalarValue; -use dssim::{Dssim, DssimImage}; use regex::Regex; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::{json, Value}; @@ -7,6 +6,7 @@ use std::collections::HashMap; use self::super::estree_expression::ESTreeExpression; use itertools::Itertools; +use pixelmatch::{pixelmatch, Options}; use std::io::{Read, Write}; use std::ops::Deref; use std::process::{Child, Command, Stdio}; @@ -471,51 +471,37 @@ impl ExportImage { Ok(path) } - pub fn to_dssim(&self, attr: &Dssim) -> Result> { - if !matches!(self, ExportImage::Png(_)) { - return Err(VegaFusionError::internal("Only PNG image supported")); + pub fn as_png_bytes(&self) -> Result> { + match self { + ExportImage::Png(png_b64) => { + #[allow(deprecated)] + let png_bytes = base64::decode(png_b64) + .external("Failed to decdode base64 encoded png image")?; + Ok(png_bytes) + } + _ => Err(VegaFusionError::internal("Only PNG image supported")), } - let tmpfile = tempfile::NamedTempFile::new().unwrap(); - let tmppath = tmpfile.path().to_str().unwrap(); - self.save(tmppath, false)?; - - let img = dssim::load_image(attr, tmppath) - .external("Failed to create DSSIM image for comparison")?; - Ok(img) } pub fn compare(&self, other: &Self) -> Result<(f64, Option>)> { - let mut attr = Dssim::new(); - attr.set_save_ssim_maps(1); - let this_img = self.to_dssim(&attr)?; - let other_img = other.to_dssim(&attr)?; - let (diff, ssim_maps) = attr.compare(&this_img, other_img); - // println!("ssim_map: {:?}", ssim_map); - - if diff > 0.0 { - let map_meta = ssim_maps[0].clone(); - let avgssim = map_meta.ssim as f32; - - let out: Vec<_> = map_meta - .map - .pixels() - .map(|ssim| { - let max = 1_f32 - ssim; - let maxsq = max * max; - rgb::RGBA8 { - r: to_byte(maxsq * 16.0), - g: to_byte(max * 3.0), - b: to_byte(max / ((1_f32 - avgssim) * 4_f32)), - a: 255, - } - }) - .collect(); - let png_res = - lodepng::encode32(&out, map_meta.map.width(), map_meta.map.height()).unwrap(); - Ok((diff.into(), Some(png_res))) - } else { - Ok((diff.into(), None)) - } + let self_png = self.as_png_bytes()?; + let other_png = other.as_png_bytes()?; + let mut img_out = Vec::new(); + let res = pixelmatch( + self_png.as_slice(), + other_png.as_slice(), + Some(&mut img_out), + None, + None, + Some(Options { + threshold: 0.1, + ..Default::default() + }), + ) + .expect("pixelmatch failed"); + + let diff_factor = (res as f64) / 10000.0; + Ok((diff_factor, Some(img_out))) } } From a2fc81696675a956e338639a5e2f8042e6c40d3d Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 26 Feb 2024 12:02:19 -0500 Subject: [PATCH 44/44] Update datafusion protobuf --- python/vegafusion/proto/datafusion.proto | 411 +++++++-- .../vegafusion/proto/datafusion_pb2.py | 782 ++++++++++-------- 2 files changed, 796 insertions(+), 397 deletions(-) diff --git a/python/vegafusion/proto/datafusion.proto b/python/vegafusion/proto/datafusion.proto index a1caa4c62..7707ed388 100644 --- a/python/vegafusion/proto/datafusion.proto +++ b/python/vegafusion/proto/datafusion.proto @@ -73,6 +73,8 @@ message LogicalPlanNode { CustomTableScanNode custom_scan = 25; PrepareNode prepare = 26; DropViewNode drop_view = 27; + DistinctOnNode distinct_on = 28; + CopyToNode copy_to = 29; } } @@ -88,6 +90,10 @@ message ProjectionColumns { message CsvFormat { bool has_header = 1; string delimiter = 2; + string quote = 3; + oneof optional_escape { + string escape = 4; + } } message ParquetFormat { @@ -176,6 +182,25 @@ message EmptyRelationNode { bool produce_one_row = 1; } +message PrimaryKeyConstraint{ + repeated uint64 indices = 1; +} + +message UniqueConstraint{ + repeated uint64 indices = 1; +} + +message Constraint{ + oneof constraint_mode{ + PrimaryKeyConstraint primary_key = 1; + UniqueConstraint unique = 2; + } +} + +message Constraints{ + repeated Constraint constraints = 1; +} + message CreateExternalTableNode { reserved 1; // was string name OwnedTableReference name = 12; @@ -187,10 +212,13 @@ message CreateExternalTableNode { bool if_not_exists = 7; string delimiter = 8; string definition = 9; - string file_compression_type = 10; + reserved 10; // was string file_compression_type + CompressionTypeVariant file_compression_type = 17; repeated LogicalExprNodeCollection order_exprs = 13; bool unbounded = 14; map options = 11; + Constraints constraints = 15; + map column_defaults = 16; } message PrepareNode { @@ -284,6 +312,32 @@ message DistinctNode { LogicalPlanNode input = 1; } +message DistinctOnNode { + repeated LogicalExprNode on_expr = 1; + repeated LogicalExprNode select_expr = 2; + repeated LogicalExprNode sort_expr = 3; + LogicalPlanNode input = 4; +} + +message CopyToNode { + LogicalPlanNode input = 1; + string output_url = 2; + oneof CopyOptions { + SQLOptions sql_options = 4; + FileTypeWriterOptions writer_options = 5; + } + string file_type = 6; +} + +message SQLOptions { + repeated SQLOption option = 1; +} + +message SQLOption { + string key = 1; + string value = 2; +} + message UnionNode { repeated LogicalPlanNode inputs = 1; } @@ -339,7 +393,7 @@ message LogicalExprNode { SortExprNode sort = 12; NegativeNode negative = 13; InListNode in_list = 14; - bool wildcard = 15; + Wildcard wildcard = 15; ScalarFunctionNode scalar_function = 16; TryCastNode try_cast = 17; @@ -372,9 +426,15 @@ message LogicalExprNode { PlaceholderNode placeholder = 34; + Unnest unnest = 35; + } } +message Wildcard { + string qualifier = 1; +} + message PlaceholderNode { string id = 1; ArrowType data_type = 2; @@ -396,11 +456,27 @@ message RollupNode { repeated LogicalExprNode expr = 1; } +message NamedStructField { + ScalarValue name = 1; +} + +message ListIndex { + LogicalExprNode key = 1; +} +message ListRange { + LogicalExprNode start = 1; + LogicalExprNode stop = 2; + LogicalExprNode stride = 3; +} message GetIndexedField { LogicalExprNode expr = 1; - ScalarValue key = 2; + oneof field { + NamedStructField named_struct_field = 2; + ListIndex list_index = 3; + ListRange list_range = 4; + } } message IsNull { @@ -442,6 +518,7 @@ message Not { message AliasNode { LogicalExprNode expr = 1; string alias = 2; + repeated OwnedTableReference relation = 3; } message BinaryExprNode { @@ -456,6 +533,10 @@ message NegativeNode { LogicalExprNode expr = 1; } +message Unnest { + repeated LogicalExprNode exprs = 1; +} + message InListNode { LogicalExprNode expr = 1; repeated LogicalExprNode list = 2; @@ -499,7 +580,7 @@ enum ScalarFunction { Lower = 33; Ltrim = 34; MD5 = 35; - NullIf = 36; + // 36 was NullIf OctetLength = 37; Random = 38; RegexpReplace = 39; @@ -552,7 +633,7 @@ enum ScalarFunction { ArrayAppend = 86; ArrayConcat = 87; ArrayDims = 88; - ArrayFill = 89; + ArrayRepeat = 89; ArrayLength = 90; ArrayNdims = 91; ArrayPosition = 92; @@ -560,13 +641,44 @@ enum ScalarFunction { ArrayPrepend = 94; ArrayRemove = 95; ArrayReplace = 96; - ArrayToString = 97; + // 97 was ArrayToString Cardinality = 98; - TrimArray = 99; - ArrayContains = 100; - Encode = 101; - Decode = 102; + ArrayElement = 99; + ArraySlice = 100; Cot = 103; + ArrayHas = 104; + ArrayHasAny = 105; + ArrayHasAll = 106; + ArrayRemoveN = 107; + ArrayReplaceN = 108; + ArrayRemoveAll = 109; + ArrayReplaceAll = 110; + Nanvl = 111; + Flatten = 112; + // 113 was IsNan + Iszero = 114; + ArrayEmpty = 115; + ArrayPopBack = 116; + StringToArray = 117; + ToTimestampNanos = 118; + ArrayIntersect = 119; + ArrayUnion = 120; + OverLay = 121; + Range = 122; + ArrayExcept = 123; + ArrayPopFront = 124; + Levenshtein = 125; + SubstrIndex = 126; + FindInSet = 127; + ArraySort = 128; + ArrayDistinct = 129; + ArrayResize = 130; + EndsWith = 131; + InStr = 132; + MakeDate = 133; + ArrayReverse = 134; + RegexpLike = 135; + ToChar = 136; } message ScalarFunctionNode { @@ -603,6 +715,17 @@ enum AggregateFunction { // we append "_AGG" to obey name scoping rules. FIRST_VALUE_AGG = 24; LAST_VALUE_AGG = 25; + REGR_SLOPE = 26; + REGR_INTERCEPT = 27; + REGR_COUNT = 28; + REGR_R2 = 29; + REGR_AVGX = 30; + REGR_AVGY = 31; + REGR_SXX = 32; + REGR_SYY = 33; + REGR_SXY = 34; + STRING_AGG = 35; + NTH_VALUE_AGG = 36; } message AggregateExprNode { @@ -752,6 +875,8 @@ message Field { // for complex data types like structs, unions repeated Field children = 4; map metadata = 5; + int64 dict_id = 6; + bool dict_ordered = 7; } message FixedSizeBinary{ @@ -821,12 +946,11 @@ message Union{ repeated int32 type_ids = 3; } -message ScalarListValue{ - // encode null explicitly to distinguish a list with a null value - // from a list with no values) - bool is_null = 3; - Field field = 1; - repeated ScalarValue values = 2; +// Used for List/FixedSizeList/LargeList/Struct +message ScalarNestedValue { + bytes ipc_message = 1; + bytes arrow_data = 2; + Schema schema = 3; } message ScalarTime32Value { @@ -864,14 +988,6 @@ message IntervalMonthDayNanoValue { int64 nanos = 3; } -message StructValue { - // Note that a null struct value must have one or more fields, so we - // encode a null StructValue as one witth an empty field_values - // list. - repeated ScalarValue field_values = 2; - repeated Field fields = 3; -} - message ScalarFixedSizeBinary{ bytes values = 1; int32 length = 2; @@ -902,10 +1018,14 @@ message ScalarValue{ // Literal Date32 value always has a unit of day int32 date_32_value = 14; ScalarTime32Value time32_value = 15; - ScalarListValue list_value = 17; - //WAS: ScalarType null_list_value = 18; + ScalarNestedValue large_list_value = 16; + ScalarNestedValue list_value = 17; + ScalarNestedValue fixed_size_list_value = 18; + ScalarNestedValue struct_value = 32; Decimal128 decimal128_value = 20; + Decimal256 decimal256_value = 39; + int64 date_64_value = 21; int32 interval_yearmonth_value = 24; int64 interval_daytime_value = 25; @@ -921,7 +1041,6 @@ message ScalarValue{ bytes large_binary_value = 29; ScalarTime64Value time64_value = 30; IntervalMonthDayNanoValue interval_month_day_nano = 31; - StructValue struct_value = 32; ScalarFixedSizeBinary fixed_size_binary_value = 34; } } @@ -932,6 +1051,12 @@ message Decimal128{ int64 s = 3; } +message Decimal256{ + bytes value = 1; + int64 p = 2; + int64 s = 3; +} + // Serialized data type message ArrowType{ oneof arrow_type_enum { @@ -1002,8 +1127,10 @@ message PlanType { OptimizedLogicalPlanType OptimizedLogicalPlan = 2; EmptyMessage FinalLogicalPlan = 3; EmptyMessage InitialPhysicalPlan = 4; + EmptyMessage InitialPhysicalPlanWithStats = 9; OptimizedPhysicalPlanType OptimizedPhysicalPlan = 5; EmptyMessage FinalPhysicalPlan = 6; + EmptyMessage FinalPhysicalPlanWithStats = 10; } } @@ -1061,9 +1188,122 @@ message PhysicalPlanNode { ExplainExecNode explain = 20; SortPreservingMergeExecNode sort_preserving_merge = 21; NestedLoopJoinExecNode nested_loop_join = 22; + AnalyzeExecNode analyze = 23; + JsonSinkExecNode json_sink = 24; + SymmetricHashJoinExecNode symmetric_hash_join = 25; + InterleaveExecNode interleave = 26; + PlaceholderRowExecNode placeholder_row = 27; + CsvSinkExecNode csv_sink = 28; + ParquetSinkExecNode parquet_sink = 29; } } +enum CompressionTypeVariant { + GZIP = 0; + BZIP2 = 1; + XZ = 2; + ZSTD = 3; + UNCOMPRESSED = 4; +} + +message PartitionColumn { + string name = 1; + ArrowType arrow_type = 2; +} + +message FileTypeWriterOptions { + oneof FileType { + JsonWriterOptions json_options = 1; + ParquetWriterOptions parquet_options = 2; + CsvWriterOptions csv_options = 3; + ArrowWriterOptions arrow_options = 4; + } +} + +message JsonWriterOptions { + CompressionTypeVariant compression = 1; +} + +message ParquetWriterOptions { + WriterProperties writer_properties = 1; +} + +message CsvWriterOptions { + // Compression type + CompressionTypeVariant compression = 1; + // Optional column delimiter. Defaults to `b','` + string delimiter = 2; + // Whether to write column names as file headers. Defaults to `true` + bool has_header = 3; + // Optional date format for date arrays + string date_format = 4; + // Optional datetime format for datetime arrays + string datetime_format = 5; + // Optional timestamp format for timestamp arrays + string timestamp_format = 6; + // Optional time format for time arrays + string time_format = 7; + // Optional value to represent null + string null_value = 8; +} + +message ArrowWriterOptions {} + +message WriterProperties { + uint64 data_page_size_limit = 1; + uint64 dictionary_page_size_limit = 2; + uint64 data_page_row_count_limit = 3; + uint64 write_batch_size = 4; + uint64 max_row_group_size = 5; + string writer_version = 6; + string created_by = 7; +} + +message FileSinkConfig { + reserved 6; // writer_mode + + string object_store_url = 1; + repeated PartitionedFile file_groups = 2; + repeated string table_paths = 3; + Schema output_schema = 4; + repeated PartitionColumn table_partition_cols = 5; + bool overwrite = 8; + FileTypeWriterOptions file_type_writer_options = 9; +} + +message JsonSink { + FileSinkConfig config = 1; +} + +message JsonSinkExecNode { + PhysicalPlanNode input = 1; + JsonSink sink = 2; + Schema sink_schema = 3; + PhysicalSortExprNodeCollection sort_order = 4; +} + +message CsvSink { + FileSinkConfig config = 1; +} + +message CsvSinkExecNode { + PhysicalPlanNode input = 1; + CsvSink sink = 2; + Schema sink_schema = 3; + PhysicalSortExprNodeCollection sort_order = 4; +} + +message ParquetSink { + FileSinkConfig config = 1; +} + +message ParquetSinkExecNode { + PhysicalPlanNode input = 1; + ParquetSink sink = 2; + Schema sink_schema = 3; + PhysicalSortExprNodeCollection sort_order = 4; +} + message PhysicalExtensionNode { bytes node = 1; repeated PhysicalPlanNode inputs = 2; @@ -1071,6 +1311,9 @@ message PhysicalExtensionNode { // physical expressions message PhysicalExprNode { + // Was date_time_interval_expr + reserved 17; + oneof ExprType { // column references PhysicalColumn column = 1; @@ -1101,8 +1344,6 @@ message PhysicalExprNode { PhysicalScalarUdfNode scalar_udf = 16; - PhysicalDateTimeIntervalExprNode date_time_interval_expr = 17; - PhysicalLikeExprNode like_expr = 18; PhysicalGetIndexedFieldExprNode get_indexed_field_expr = 19; @@ -1131,7 +1372,11 @@ message PhysicalWindowExprNode { BuiltInWindowFunction built_in_function = 2; // udaf = 3 } - PhysicalExprNode expr = 4; + repeated PhysicalExprNode args = 4; + repeated PhysicalExprNode partition_by = 5; + repeated PhysicalSortExprNode order_by = 6; + WindowFrame window_frame = 7; + string name = 8; } message PhysicalIsNull { @@ -1217,6 +1462,7 @@ message PhysicalNegativeNode { message FilterExecNode { PhysicalPlanNode input = 1; PhysicalExprNode expr = 2; + uint32 default_filter_selectivity = 3; } message FileGroup { @@ -1259,6 +1505,10 @@ message CsvScanExecNode { FileScanExecConf base_conf = 1; bool has_header = 2; string delimiter = 3; + string quote = 4; + oneof optional_escape { + string escape = 5; + } } message AvroScanExecNode { @@ -1281,6 +1531,27 @@ message HashJoinExecNode { JoinFilter filter = 8; } +enum StreamPartitionMode { + SINGLE_PARTITION = 0; + PARTITIONED_EXEC = 1; +} + +message SymmetricHashJoinExecNode { + PhysicalPlanNode left = 1; + PhysicalPlanNode right = 2; + repeated JoinOn on = 3; + JoinType join_type = 4; + StreamPartitionMode partition_mode = 6; + bool null_equals_null = 7; + JoinFilter filter = 8; + repeated PhysicalSortExprNode left_sort_exprs = 9; + repeated PhysicalSortExprNode right_sort_exprs = 10; +} + +message InterleaveExecNode { + repeated PhysicalPlanNode inputs = 1; +} + message UnionExecNode { repeated PhysicalPlanNode inputs = 1; } @@ -1291,6 +1562,13 @@ message ExplainExecNode { bool verbose = 3; } +message AnalyzeExecNode { + bool verbose = 1; + bool show_statistics = 2; + PhysicalPlanNode input = 3; + Schema schema = 4; +} + message CrossJoinExecNode { PhysicalPlanNode left = 1; PhysicalPlanNode right = 2; @@ -1302,13 +1580,16 @@ message PhysicalColumn { } message JoinOn { - PhysicalColumn left = 1; - PhysicalColumn right = 2; + PhysicalExprNode left = 1; + PhysicalExprNode right = 2; } message EmptyExecNode { - bool produce_one_row = 1; - Schema schema = 2; + Schema schema = 1; +} + +message PlaceholderRowExecNode { + Schema schema = 1; } message ProjectionExecNode { @@ -1325,11 +1606,20 @@ enum AggregateMode { SINGLE_PARTITIONED = 4; } +message PartiallySortedInputOrderMode { + repeated uint64 columns = 6; +} + message WindowAggExecNode { PhysicalPlanNode input = 1; - repeated PhysicalExprNode window_expr = 2; - repeated string window_expr_name = 3; - Schema input_schema = 4; + repeated PhysicalWindowExprNode window_expr = 2; + repeated PhysicalExprNode partition_keys = 5; + // Set optional to `None` for `BoundedWindowAggExec`. + oneof input_order_mode { + EmptyMessage linear = 7; + PartiallySortedInputOrderMode partially_sorted = 8; + EmptyMessage sorted = 9; + } } message MaybeFilter { @@ -1352,7 +1642,6 @@ message AggregateExecNode { repeated PhysicalExprNode null_expr = 8; repeated bool groups = 9; repeated MaybeFilter filter_expr = 10; - repeated MaybePhysicalSortExprs order_by_expr = 11; } message GlobalLimitExecNode { @@ -1449,21 +1738,49 @@ message PartitionStats { repeated ColumnStats column_stats = 4; } +message Precision{ + PrecisionInfo precision_info = 1; + ScalarValue val = 2; +} + +enum PrecisionInfo { + EXACT = 0; + INEXACT = 1; + ABSENT = 2; +} + message Statistics { - int64 num_rows = 1; - int64 total_byte_size = 2; + Precision num_rows = 1; + Precision total_byte_size = 2; repeated ColumnStats column_stats = 3; - bool is_exact = 4; } message ColumnStats { - ScalarValue min_value = 1; - ScalarValue max_value = 2; - uint32 null_count = 3; - uint32 distinct_count = 4; + Precision min_value = 1; + Precision max_value = 2; + Precision null_count = 3; + Precision distinct_count = 4; +} + +message NamedStructFieldExpr { + ScalarValue name = 1; +} + +message ListIndexExpr { + PhysicalExprNode key = 1; +} + +message ListRangeExpr { + PhysicalExprNode start = 1; + PhysicalExprNode stop = 2; + PhysicalExprNode stride = 3; } message PhysicalGetIndexedFieldExprNode { PhysicalExprNode arg = 1; - ScalarValue key = 2; -} + oneof field { + NamedStructFieldExpr named_struct_field_expr = 2; + ListIndexExpr list_index_expr = 3; + ListRangeExpr list_range_expr = 4; + } +} \ No newline at end of file diff --git a/python/vegafusion/vegafusion/proto/datafusion_pb2.py b/python/vegafusion/vegafusion/proto/datafusion_pb2.py index 6cd5755e2..530601492 100644 --- a/python/vegafusion/vegafusion/proto/datafusion_pb2.py +++ b/python/vegafusion/vegafusion/proto/datafusion_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16proto/datafusion.proto\x12\ndatafusion\"\"\n\x0e\x43olumnRelation\x12\x10\n\x08relation\x18\x01 \x01(\t\"D\n\x06\x43olumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\x08relation\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"Z\n\x07\x44\x66\x46ield\x12 \n\x05\x66ield\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12-\n\tqualifier\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"\x97\x01\n\x08\x44\x66Schema\x12$\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x13.datafusion.DfField\x12\x34\n\x08metadata\x18\x02 \x03(\x0b\x32\".datafusion.DfSchema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb3\n\n\x0fLogicalPlanNode\x12\x38\n\x0clisting_scan\x18\x01 \x01(\x0b\x32 .datafusion.ListingTableScanNodeH\x00\x12\x30\n\nprojection\x18\x03 \x01(\x0b\x32\x1a.datafusion.ProjectionNodeH\x00\x12.\n\tselection\x18\x04 \x01(\x0b\x32\x19.datafusion.SelectionNodeH\x00\x12&\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.LimitNodeH\x00\x12.\n\taggregate\x18\x06 \x01(\x0b\x32\x19.datafusion.AggregateNodeH\x00\x12$\n\x04join\x18\x07 \x01(\x0b\x32\x14.datafusion.JoinNodeH\x00\x12$\n\x04sort\x18\x08 \x01(\x0b\x32\x14.datafusion.SortNodeH\x00\x12\x32\n\x0brepartition\x18\t \x01(\x0b\x32\x1b.datafusion.RepartitionNodeH\x00\x12\x37\n\x0e\x65mpty_relation\x18\n \x01(\x0b\x32\x1d.datafusion.EmptyRelationNodeH\x00\x12\x44\n\x15\x63reate_external_table\x18\x0b \x01(\x0b\x32#.datafusion.CreateExternalTableNodeH\x00\x12*\n\x07\x65xplain\x18\x0c \x01(\x0b\x32\x17.datafusion.ExplainNodeH\x00\x12(\n\x06window\x18\r \x01(\x0b\x32\x16.datafusion.WindowNodeH\x00\x12*\n\x07\x61nalyze\x18\x0e \x01(\x0b\x32\x17.datafusion.AnalyzeNodeH\x00\x12/\n\ncross_join\x18\x0f \x01(\x0b\x32\x19.datafusion.CrossJoinNodeH\x00\x12(\n\x06values\x18\x10 \x01(\x0b\x32\x16.datafusion.ValuesNodeH\x00\x12\x35\n\textension\x18\x11 \x01(\x0b\x32 .datafusion.LogicalExtensionNodeH\x00\x12\x44\n\x15\x63reate_catalog_schema\x18\x12 \x01(\x0b\x32#.datafusion.CreateCatalogSchemaNodeH\x00\x12&\n\x05union\x18\x13 \x01(\x0b\x32\x15.datafusion.UnionNodeH\x00\x12\x37\n\x0e\x63reate_catalog\x18\x14 \x01(\x0b\x32\x1d.datafusion.CreateCatalogNodeH\x00\x12\x37\n\x0esubquery_alias\x18\x15 \x01(\x0b\x32\x1d.datafusion.SubqueryAliasNodeH\x00\x12\x31\n\x0b\x63reate_view\x18\x16 \x01(\x0b\x32\x1a.datafusion.CreateViewNodeH\x00\x12,\n\x08\x64istinct\x18\x17 \x01(\x0b\x32\x18.datafusion.DistinctNodeH\x00\x12\x32\n\tview_scan\x18\x18 \x01(\x0b\x32\x1d.datafusion.ViewTableScanNodeH\x00\x12\x36\n\x0b\x63ustom_scan\x18\x19 \x01(\x0b\x32\x1f.datafusion.CustomTableScanNodeH\x00\x12*\n\x07prepare\x18\x1a \x01(\x0b\x32\x17.datafusion.PrepareNodeH\x00\x12-\n\tdrop_view\x18\x1b \x01(\x0b\x32\x18.datafusion.DropViewNodeH\x00\x42\x11\n\x0fLogicalPlanType\"Q\n\x14LogicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"$\n\x11ProjectionColumns\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\"2\n\tCsvFormat\x12\x12\n\nhas_header\x18\x01 \x01(\x08\x12\x11\n\tdelimiter\x18\x02 \x01(\t\"\x15\n\rParquetFormatJ\x04\x08\x01\x10\x02\"\x0c\n\nAvroFormat\"T\n\x19LogicalExprNodeCollection\x12\x37\n\x12logical_expr_nodes\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x9a\x04\n\x14ListingTableScanNode\x12\x33\n\ntable_name\x18\x0e \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\r\n\x05paths\x18\x02 \x03(\t\x12\x16\n\x0e\x66ile_extension\x18\x03 \x01(\t\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x05 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x14\n\x0c\x63ollect_stat\x18\x08 \x01(\x08\x12\x19\n\x11target_partitions\x18\t \x01(\r\x12$\n\x03\x63sv\x18\n \x01(\x0b\x32\x15.datafusion.CsvFormatH\x00\x12,\n\x07parquet\x18\x0b \x01(\x0b\x32\x19.datafusion.ParquetFormatH\x00\x12&\n\x04\x61vro\x18\x0c \x01(\x0b\x32\x16.datafusion.AvroFormatH\x00\x12>\n\x0f\x66ile_sort_order\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollectionB\x10\n\x0e\x46ileFormatTypeJ\x04\x08\x01\x10\x02\"\xe5\x01\n\x11ViewTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\x12\n\ndefinition\x18\x05 \x01(\tJ\x04\x08\x01\x10\x02\"\xf0\x01\n\x13\x43ustomTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x31\n\nprojection\x18\x02 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x19\n\x11\x63ustom_table_data\x18\x05 \x01(\x0cJ\x04\x08\x01\x10\x02\"\x8a\x01\n\x0eProjectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x05\x61lias\x18\x03 \x01(\tH\x00\x42\x10\n\x0eoptional_alias\"f\n\rSelectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"p\n\x08SortNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\x95\x01\n\x0fRepartitionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12+\n\x04hash\x18\x03 \x01(\x0b\x32\x1b.datafusion.HashRepartitionH\x00\x42\x12\n\x10partition_method\"Z\n\x0fHashRepartition\x12.\n\thash_expr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\",\n\x11\x45mptyRelationNode\x12\x17\n\x0fproduce_one_row\x18\x01 \x01(\x08\"\xea\x03\n\x17\x43reateExternalTableNode\x12-\n\x04name\x18\x0c \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x10\n\x08location\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x12\n\nhas_header\x18\x04 \x01(\x08\x12$\n\x06schema\x18\x05 \x01(\x0b\x32\x14.datafusion.DfSchema\x12\x1c\n\x14table_partition_cols\x18\x06 \x03(\t\x12\x15\n\rif_not_exists\x18\x07 \x01(\x08\x12\x11\n\tdelimiter\x18\x08 \x01(\t\x12\x12\n\ndefinition\x18\t \x01(\t\x12\x1d\n\x15\x66ile_compression_type\x18\n \x01(\t\x12:\n\x0border_exprs\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollection\x12\x11\n\tunbounded\x18\x0e \x01(\x08\x12\x41\n\x07options\x18\x0b \x03(\x0b\x32\x30.datafusion.CreateExternalTableNode.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x01\x10\x02\"r\n\x0bPrepareNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\ndata_types\x18\x02 \x03(\x0b\x32\x15.datafusion.ArrowType\x12*\n\x05input\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"k\n\x17\x43reateCatalogSchemaNode\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"f\n\x11\x43reateCatalogNode\x12\x14\n\x0c\x63\x61talog_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"v\n\x0c\x44ropViewNode\x12-\n\x04name\x18\x01 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x11\n\tif_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"\x99\x01\n\x0e\x43reateViewNode\x12-\n\x04name\x18\x05 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x12\n\nor_replace\x18\x03 \x01(\x08\x12\x12\n\ndefinition\x18\x04 \x01(\tJ\x04\x08\x01\x10\x02\"N\n\nValuesNode\x12\x0e\n\x06n_cols\x18\x01 \x01(\x04\x12\x30\n\x0bvalues_list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"J\n\x0b\x41nalyzeNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"J\n\x0b\x45xplainNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"\x9c\x01\n\rAggregateNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12/\n\ngroup_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\taggr_expr\x18\x03 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\nWindowNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x30\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xef\x02\n\x08JoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x33\n\x0fjoin_constraint\x18\x04 \x01(\x0e\x32\x1a.datafusion.JoinConstraint\x12\x32\n\rleft_join_key\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x33\n\x0eright_join_key\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12+\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\":\n\x0c\x44istinctNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"8\n\tUnionNode\x12+\n\x06inputs\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"f\n\rCrossJoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"T\n\tLimitNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\x03\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\">\n\x11SelectionExecNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"u\n\x11SubqueryAliasNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12.\n\x05\x61lias\x18\x03 \x01(\x0b\x32\x1f.datafusion.OwnedTableReferenceJ\x04\x08\x02\x10\x03\"\xad\x0c\n\x0fLogicalExprNode\x12$\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x12.datafusion.ColumnH\x00\x12&\n\x05\x61lias\x18\x02 \x01(\x0b\x32\x15.datafusion.AliasNodeH\x00\x12*\n\x07literal\x18\x03 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x31\n\x0b\x62inary_expr\x18\x04 \x01(\x0b\x32\x1a.datafusion.BinaryExprNodeH\x00\x12\x37\n\x0e\x61ggregate_expr\x18\x05 \x01(\x0b\x32\x1d.datafusion.AggregateExprNodeH\x00\x12*\n\x0cis_null_expr\x18\x06 \x01(\x0b\x32\x12.datafusion.IsNullH\x00\x12\x31\n\x10is_not_null_expr\x18\x07 \x01(\x0b\x32\x15.datafusion.IsNotNullH\x00\x12#\n\x08not_expr\x18\x08 \x01(\x0b\x32\x0f.datafusion.NotH\x00\x12*\n\x07\x62\x65tween\x18\t \x01(\x0b\x32\x17.datafusion.BetweenNodeH\x00\x12%\n\x05\x63\x61se_\x18\n \x01(\x0b\x32\x14.datafusion.CaseNodeH\x00\x12$\n\x04\x63\x61st\x18\x0b \x01(\x0b\x32\x14.datafusion.CastNodeH\x00\x12(\n\x04sort\x18\x0c \x01(\x0b\x32\x18.datafusion.SortExprNodeH\x00\x12,\n\x08negative\x18\r \x01(\x0b\x32\x18.datafusion.NegativeNodeH\x00\x12)\n\x07in_list\x18\x0e \x01(\x0b\x32\x16.datafusion.InListNodeH\x00\x12\x12\n\x08wildcard\x18\x0f \x01(\x08H\x00\x12\x39\n\x0fscalar_function\x18\x10 \x01(\x0b\x32\x1e.datafusion.ScalarFunctionNodeH\x00\x12+\n\x08try_cast\x18\x11 \x01(\x0b\x32\x17.datafusion.TryCastNodeH\x00\x12\x31\n\x0bwindow_expr\x18\x12 \x01(\x0b\x32\x1a.datafusion.WindowExprNodeH\x00\x12>\n\x12\x61ggregate_udf_expr\x18\x13 \x01(\x0b\x32 .datafusion.AggregateUDFExprNodeH\x00\x12\x38\n\x0fscalar_udf_expr\x18\x14 \x01(\x0b\x32\x1d.datafusion.ScalarUDFExprNodeH\x00\x12\x38\n\x11get_indexed_field\x18\x15 \x01(\x0b\x32\x1b.datafusion.GetIndexedFieldH\x00\x12\x33\n\x0cgrouping_set\x18\x16 \x01(\x0b\x32\x1b.datafusion.GroupingSetNodeH\x00\x12$\n\x04\x63ube\x18\x17 \x01(\x0b\x32\x14.datafusion.CubeNodeH\x00\x12(\n\x06rollup\x18\x18 \x01(\x0b\x32\x16.datafusion.RollupNodeH\x00\x12%\n\x07is_true\x18\x19 \x01(\x0b\x32\x12.datafusion.IsTrueH\x00\x12\'\n\x08is_false\x18\x1a \x01(\x0b\x32\x13.datafusion.IsFalseH\x00\x12+\n\nis_unknown\x18\x1b \x01(\x0b\x32\x15.datafusion.IsUnknownH\x00\x12,\n\x0bis_not_true\x18\x1c \x01(\x0b\x32\x15.datafusion.IsNotTrueH\x00\x12.\n\x0cis_not_false\x18\x1d \x01(\x0b\x32\x16.datafusion.IsNotFalseH\x00\x12\x32\n\x0eis_not_unknown\x18\x1e \x01(\x0b\x32\x18.datafusion.IsNotUnknownH\x00\x12$\n\x04like\x18\x1f \x01(\x0b\x32\x14.datafusion.LikeNodeH\x00\x12&\n\x05ilike\x18 \x01(\x0b\x32\x15.datafusion.ILikeNodeH\x00\x12/\n\nsimilar_to\x18! \x01(\x0b\x32\x19.datafusion.SimilarToNodeH\x00\x12\x32\n\x0bplaceholder\x18\" \x01(\x0b\x32\x1b.datafusion.PlaceholderNodeH\x00\x42\n\n\x08\x45xprType\"G\n\x0fPlaceholderNode\x12\n\n\x02id\x18\x01 \x01(\t\x12(\n\tdata_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"<\n\x0fLogicalExprList\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"<\n\x0fGroupingSetNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprList\"5\n\x08\x43ubeNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nRollupNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"b\n\x0fGetIndexedField\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12$\n\x03key\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"3\n\x06IsNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"3\n\x06IsTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"4\n\x07IsFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nIsNotFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"9\n\x0cIsNotUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"0\n\x03Not\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"E\n\tAliasNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"K\n\x0e\x42inaryExprNode\x12-\n\x08operands\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"9\n\x0cNegativeNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"s\n\nInListNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"h\n\x12ScalarFunctionNode\x12\'\n\x03\x66un\x18\x01 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xe2\x01\n\x11\x41ggregateExprNode\x12\x34\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunction\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x12+\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xaf\x01\n\x14\x41ggregateUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12+\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"P\n\x11ScalarUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xf7\x02\n\x0eWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12\x0e\n\x04udaf\x18\x03 \x01(\tH\x00\x12\x0e\n\x04udwf\x18\t \x01(\tH\x00\x12)\n\x04\x65xpr\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x31\n\x0cpartition_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x0cwindow_frame\x18\x08 \x01(\x0b\x32\x17.datafusion.WindowFrameB\x11\n\x0fwindow_function\"\x9e\x01\n\x0b\x42\x65tweenNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x02 \x01(\x08\x12(\n\x03low\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04high\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x89\x01\n\x08LikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8a\x01\n\tILikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8e\x01\n\rSimilarToNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x93\x01\n\x08\x43\x61seNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x14.datafusion.WhenThen\x12.\n\telse_expr\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\x08WhenThen\x12.\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"`\n\x08\x43\x61stNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"c\n\x0bTryCastNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"[\n\x0cSortExprNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"\xb6\x01\n\x0bWindowFrame\x12\x38\n\x12window_frame_units\x18\x01 \x01(\x0e\x32\x1c.datafusion.WindowFrameUnits\x12\x31\n\x0bstart_bound\x18\x02 \x01(\x0b\x32\x1c.datafusion.WindowFrameBound\x12-\n\x05\x62ound\x18\x03 \x01(\x0b\x32\x1c.datafusion.WindowFrameBoundH\x00\x42\x0b\n\tend_bound\"\x83\x01\n\x10WindowFrameBound\x12\x41\n\x17window_frame_bound_type\x18\x01 \x01(\x0e\x32 .datafusion.WindowFrameBoundType\x12,\n\x0b\x62ound_value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"\x91\x01\n\x06Schema\x12\"\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12\x32\n\x08metadata\x18\x02 \x03(\x0b\x32 .datafusion.Schema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xdb\x01\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\x12\x10\n\x08nullable\x18\x03 \x01(\x08\x12#\n\x08\x63hildren\x18\x04 \x03(\x0b\x32\x11.datafusion.Field\x12\x31\n\x08metadata\x18\x05 \x03(\x0b\x32\x1f.datafusion.Field.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"!\n\x0f\x46ixedSizeBinary\x12\x0e\n\x06length\x18\x01 \x01(\x05\"F\n\tTimestamp\x12\'\n\ttime_unit\x18\x01 \x01(\x0e\x32\x14.datafusion.TimeUnit\x12\x10\n\x08timezone\x18\x02 \x01(\t\"7\n\x07\x44\x65\x63imal\x12\x11\n\tprecision\x18\x03 \x01(\r\x12\r\n\x05scale\x18\x04 \x01(\x05J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03\"-\n\x04List\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\"I\n\rFixedSizeList\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x11\n\tlist_size\x18\x02 \x01(\x05\"V\n\nDictionary\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"4\n\x06Struct\x12*\n\x0fsub_field_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\"A\n\x03Map\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x13\n\x0bkeys_sorted\x18\x02 \x01(\x08\"l\n\x05Union\x12&\n\x0bunion_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12)\n\nunion_mode\x18\x02 \x01(\x0e\x32\x15.datafusion.UnionMode\x12\x10\n\x08type_ids\x18\x03 \x03(\x05\"m\n\x0fScalarListValue\x12\x0f\n\x07is_null\x18\x03 \x01(\x08\x12 \n\x05\x66ield\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\'\n\x06values\x18\x02 \x03(\x0b\x32\x17.datafusion.ScalarValue\"_\n\x11ScalarTime32Value\x12\x1d\n\x13time32_second_value\x18\x01 \x01(\x05H\x00\x12\"\n\x18time32_millisecond_value\x18\x02 \x01(\x05H\x00\x42\x07\n\x05value\"c\n\x11ScalarTime64Value\x12\"\n\x18time64_microsecond_value\x18\x01 \x01(\x03H\x00\x12!\n\x17time64_nanosecond_value\x18\x02 \x01(\x03H\x00\x42\x07\n\x05value\"\xb3\x01\n\x14ScalarTimestampValue\x12 \n\x16time_microsecond_value\x18\x01 \x01(\x03H\x00\x12\x1f\n\x15time_nanosecond_value\x18\x02 \x01(\x03H\x00\x12\x1b\n\x11time_second_value\x18\x03 \x01(\x03H\x00\x12 \n\x16time_millisecond_value\x18\x04 \x01(\x03H\x00\x12\x10\n\x08timezone\x18\x05 \x01(\tB\x07\n\x05value\"j\n\x15ScalarDictionaryValue\x12)\n\nindex_type\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"H\n\x19IntervalMonthDayNanoValue\x12\x0e\n\x06months\x18\x01 \x01(\x05\x12\x0c\n\x04\x64\x61ys\x18\x02 \x01(\x05\x12\r\n\x05nanos\x18\x03 \x01(\x03\"_\n\x0bStructValue\x12-\n\x0c\x66ield_values\x18\x02 \x03(\x0b\x32\x17.datafusion.ScalarValue\x12!\n\x06\x66ields\x18\x03 \x03(\x0b\x32\x11.datafusion.Field\"7\n\x15ScalarFixedSizeBinary\x12\x0e\n\x06values\x18\x01 \x01(\x0c\x12\x0e\n\x06length\x18\x02 \x01(\x05\"\xcf\t\n\x0bScalarValue\x12+\n\nnull_value\x18! \x01(\x0b\x32\x15.datafusion.ArrowTypeH\x00\x12\x14\n\nbool_value\x18\x01 \x01(\x08H\x00\x12\x14\n\nutf8_value\x18\x02 \x01(\tH\x00\x12\x1a\n\x10large_utf8_value\x18\x03 \x01(\tH\x00\x12\x14\n\nint8_value\x18\x04 \x01(\x05H\x00\x12\x15\n\x0bint16_value\x18\x05 \x01(\x05H\x00\x12\x15\n\x0bint32_value\x18\x06 \x01(\x05H\x00\x12\x15\n\x0bint64_value\x18\x07 \x01(\x03H\x00\x12\x15\n\x0buint8_value\x18\x08 \x01(\rH\x00\x12\x16\n\x0cuint16_value\x18\t \x01(\rH\x00\x12\x16\n\x0cuint32_value\x18\n \x01(\rH\x00\x12\x16\n\x0cuint64_value\x18\x0b \x01(\x04H\x00\x12\x17\n\rfloat32_value\x18\x0c \x01(\x02H\x00\x12\x17\n\rfloat64_value\x18\r \x01(\x01H\x00\x12\x17\n\rdate_32_value\x18\x0e \x01(\x05H\x00\x12\x35\n\x0ctime32_value\x18\x0f \x01(\x0b\x32\x1d.datafusion.ScalarTime32ValueH\x00\x12\x31\n\nlist_value\x18\x11 \x01(\x0b\x32\x1b.datafusion.ScalarListValueH\x00\x12\x32\n\x10\x64\x65\x63imal128_value\x18\x14 \x01(\x0b\x32\x16.datafusion.Decimal128H\x00\x12\x17\n\rdate_64_value\x18\x15 \x01(\x03H\x00\x12\"\n\x18interval_yearmonth_value\x18\x18 \x01(\x05H\x00\x12 \n\x16interval_daytime_value\x18\x19 \x01(\x03H\x00\x12\x1f\n\x15\x64uration_second_value\x18# \x01(\x03H\x00\x12$\n\x1a\x64uration_millisecond_value\x18$ \x01(\x03H\x00\x12$\n\x1a\x64uration_microsecond_value\x18% \x01(\x03H\x00\x12#\n\x19\x64uration_nanosecond_value\x18& \x01(\x03H\x00\x12;\n\x0ftimestamp_value\x18\x1a \x01(\x0b\x32 .datafusion.ScalarTimestampValueH\x00\x12=\n\x10\x64ictionary_value\x18\x1b \x01(\x0b\x32!.datafusion.ScalarDictionaryValueH\x00\x12\x16\n\x0c\x62inary_value\x18\x1c \x01(\x0cH\x00\x12\x1c\n\x12large_binary_value\x18\x1d \x01(\x0cH\x00\x12\x35\n\x0ctime64_value\x18\x1e \x01(\x0b\x32\x1d.datafusion.ScalarTime64ValueH\x00\x12H\n\x17interval_month_day_nano\x18\x1f \x01(\x0b\x32%.datafusion.IntervalMonthDayNanoValueH\x00\x12/\n\x0cstruct_value\x18 \x01(\x0b\x32\x17.datafusion.StructValueH\x00\x12\x44\n\x17\x66ixed_size_binary_value\x18\" \x01(\x0b\x32!.datafusion.ScalarFixedSizeBinaryH\x00\x42\x07\n\x05valueJ\x04\x08\x13\x10\x14\"1\n\nDecimal128\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\t\n\x01p\x18\x02 \x01(\x03\x12\t\n\x01s\x18\x03 \x01(\x03\"\x94\x0b\n\tArrowType\x12(\n\x04NONE\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04\x42OOL\x18\x02 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05UINT8\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04INT8\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT16\x18\x05 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT16\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT32\x18\x07 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT32\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT64\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT64\x18\n \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT16\x18\x0b \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT32\x18\x0c \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT64\x18\r \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04UTF8\x18\x0e \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12.\n\nLARGE_UTF8\x18 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x42INARY\x18\x0f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x1b\n\x11\x46IXED_SIZE_BINARY\x18\x10 \x01(\x05H\x00\x12\x30\n\x0cLARGE_BINARY\x18\x1f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE32\x18\x11 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE64\x18\x12 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x08\x44URATION\x18\x13 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12*\n\tTIMESTAMP\x18\x14 \x01(\x0b\x32\x15.datafusion.TimestampH\x00\x12&\n\x06TIME32\x18\x15 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12&\n\x06TIME64\x18\x16 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12,\n\x08INTERVAL\x18\x17 \x01(\x0e\x32\x18.datafusion.IntervalUnitH\x00\x12&\n\x07\x44\x45\x43IMAL\x18\x18 \x01(\x0b\x32\x13.datafusion.DecimalH\x00\x12 \n\x04LIST\x18\x19 \x01(\x0b\x32\x10.datafusion.ListH\x00\x12&\n\nLARGE_LIST\x18\x1a \x01(\x0b\x32\x10.datafusion.ListH\x00\x12\x34\n\x0f\x46IXED_SIZE_LIST\x18\x1b \x01(\x0b\x32\x19.datafusion.FixedSizeListH\x00\x12$\n\x06STRUCT\x18\x1c \x01(\x0b\x32\x12.datafusion.StructH\x00\x12\"\n\x05UNION\x18\x1d \x01(\x0b\x32\x11.datafusion.UnionH\x00\x12,\n\nDICTIONARY\x18\x1e \x01(\x0b\x32\x16.datafusion.DictionaryH\x00\x12\x1e\n\x03MAP\x18! \x01(\x0b\x32\x0f.datafusion.MapH\x00\x42\x11\n\x0f\x61rrow_type_enum\"\x0e\n\x0c\x45mptyMessage\"0\n\x17\x41nalyzedLogicalPlanType\x12\x15\n\ranalyzer_name\x18\x01 \x01(\t\"2\n\x18OptimizedLogicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"3\n\x19OptimizedPhysicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"\x8a\x04\n\x08PlanType\x12\x36\n\x12InitialLogicalPlan\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x42\n\x13\x41nalyzedLogicalPlan\x18\x07 \x01(\x0b\x32#.datafusion.AnalyzedLogicalPlanTypeH\x00\x12<\n\x18\x46inalAnalyzedLogicalPlan\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x44\n\x14OptimizedLogicalPlan\x18\x02 \x01(\x0b\x32$.datafusion.OptimizedLogicalPlanTypeH\x00\x12\x34\n\x10\x46inalLogicalPlan\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x37\n\x13InitialPhysicalPlan\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x46\n\x15OptimizedPhysicalPlan\x18\x05 \x01(\x0b\x32%.datafusion.OptimizedPhysicalPlanTypeH\x00\x12\x35\n\x11\x46inalPhysicalPlan\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x42\x10\n\x0eplan_type_enum\"H\n\x0fStringifiedPlan\x12\'\n\tplan_type\x18\x01 \x01(\x0b\x32\x14.datafusion.PlanType\x12\x0c\n\x04plan\x18\x02 \x01(\t\"#\n\x12\x42\x61reTableReference\x12\r\n\x05table\x18\x01 \x01(\t\"6\n\x15PartialTableReference\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\r\n\x05table\x18\x02 \x01(\t\"D\n\x12\x46ullTableReference\x12\x0f\n\x07\x63\x61talog\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\xc3\x01\n\x13OwnedTableReference\x12.\n\x04\x62\x61re\x18\x01 \x01(\x0b\x32\x1e.datafusion.BareTableReferenceH\x00\x12\x34\n\x07partial\x18\x02 \x01(\x0b\x32!.datafusion.PartialTableReferenceH\x00\x12.\n\x04\x66ull\x18\x03 \x01(\x0b\x32\x1e.datafusion.FullTableReferenceH\x00\x42\x16\n\x14table_reference_enum\"\x8a\t\n\x10PhysicalPlanNode\x12\x37\n\x0cparquet_scan\x18\x01 \x01(\x0b\x32\x1f.datafusion.ParquetScanExecNodeH\x00\x12/\n\x08\x63sv_scan\x18\x02 \x01(\x0b\x32\x1b.datafusion.CsvScanExecNodeH\x00\x12*\n\x05\x65mpty\x18\x03 \x01(\x0b\x32\x19.datafusion.EmptyExecNodeH\x00\x12\x34\n\nprojection\x18\x04 \x01(\x0b\x32\x1e.datafusion.ProjectionExecNodeH\x00\x12\x37\n\x0cglobal_limit\x18\x06 \x01(\x0b\x32\x1f.datafusion.GlobalLimitExecNodeH\x00\x12\x35\n\x0blocal_limit\x18\x07 \x01(\x0b\x32\x1e.datafusion.LocalLimitExecNodeH\x00\x12\x32\n\taggregate\x18\x08 \x01(\x0b\x32\x1d.datafusion.AggregateExecNodeH\x00\x12\x31\n\thash_join\x18\t \x01(\x0b\x32\x1c.datafusion.HashJoinExecNodeH\x00\x12(\n\x04sort\x18\n \x01(\x0b\x32\x18.datafusion.SortExecNodeH\x00\x12?\n\x10\x63oalesce_batches\x18\x0b \x01(\x0b\x32#.datafusion.CoalesceBatchesExecNodeH\x00\x12,\n\x06\x66ilter\x18\x0c \x01(\x0b\x32\x1a.datafusion.FilterExecNodeH\x00\x12\x37\n\x05merge\x18\r \x01(\x0b\x32&.datafusion.CoalescePartitionsExecNodeH\x00\x12\x36\n\x0brepartition\x18\x0e \x01(\x0b\x32\x1f.datafusion.RepartitionExecNodeH\x00\x12/\n\x06window\x18\x0f \x01(\x0b\x32\x1d.datafusion.WindowAggExecNodeH\x00\x12\x33\n\ncross_join\x18\x10 \x01(\x0b\x32\x1d.datafusion.CrossJoinExecNodeH\x00\x12\x31\n\tavro_scan\x18\x11 \x01(\x0b\x32\x1c.datafusion.AvroScanExecNodeH\x00\x12\x36\n\textension\x18\x12 \x01(\x0b\x32!.datafusion.PhysicalExtensionNodeH\x00\x12*\n\x05union\x18\x13 \x01(\x0b\x32\x19.datafusion.UnionExecNodeH\x00\x12.\n\x07\x65xplain\x18\x14 \x01(\x0b\x32\x1b.datafusion.ExplainExecNodeH\x00\x12H\n\x15sort_preserving_merge\x18\x15 \x01(\x0b\x32\'.datafusion.SortPreservingMergeExecNodeH\x00\x12>\n\x10nested_loop_join\x18\x16 \x01(\x0b\x32\".datafusion.NestedLoopJoinExecNodeH\x00\x42\x12\n\x10PhysicalPlanType\"S\n\x15PhysicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12,\n\x06inputs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"\xcb\x08\n\x10PhysicalExprNode\x12,\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x1a.datafusion.PhysicalColumnH\x00\x12*\n\x07literal\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x39\n\x0b\x62inary_expr\x18\x03 \x01(\x0b\x32\".datafusion.PhysicalBinaryExprNodeH\x00\x12?\n\x0e\x61ggregate_expr\x18\x04 \x01(\x0b\x32%.datafusion.PhysicalAggregateExprNodeH\x00\x12\x32\n\x0cis_null_expr\x18\x05 \x01(\x0b\x32\x1a.datafusion.PhysicalIsNullH\x00\x12\x39\n\x10is_not_null_expr\x18\x06 \x01(\x0b\x32\x1d.datafusion.PhysicalIsNotNullH\x00\x12+\n\x08not_expr\x18\x07 \x01(\x0b\x32\x17.datafusion.PhysicalNotH\x00\x12-\n\x05\x63\x61se_\x18\x08 \x01(\x0b\x32\x1c.datafusion.PhysicalCaseNodeH\x00\x12,\n\x04\x63\x61st\x18\t \x01(\x0b\x32\x1c.datafusion.PhysicalCastNodeH\x00\x12\x30\n\x04sort\x18\n \x01(\x0b\x32 .datafusion.PhysicalSortExprNodeH\x00\x12\x34\n\x08negative\x18\x0b \x01(\x0b\x32 .datafusion.PhysicalNegativeNodeH\x00\x12\x31\n\x07in_list\x18\x0c \x01(\x0b\x32\x1e.datafusion.PhysicalInListNodeH\x00\x12\x41\n\x0fscalar_function\x18\r \x01(\x0b\x32&.datafusion.PhysicalScalarFunctionNodeH\x00\x12\x33\n\x08try_cast\x18\x0e \x01(\x0b\x32\x1f.datafusion.PhysicalTryCastNodeH\x00\x12\x39\n\x0bwindow_expr\x18\x0f \x01(\x0b\x32\".datafusion.PhysicalWindowExprNodeH\x00\x12\x37\n\nscalar_udf\x18\x10 \x01(\x0b\x32!.datafusion.PhysicalScalarUdfNodeH\x00\x12O\n\x17\x64\x61te_time_interval_expr\x18\x11 \x01(\x0b\x32,.datafusion.PhysicalDateTimeIntervalExprNodeH\x00\x12\x35\n\tlike_expr\x18\x12 \x01(\x0b\x32 .datafusion.PhysicalLikeExprNodeH\x00\x12M\n\x16get_indexed_field_expr\x18\x13 \x01(\x0b\x32+.datafusion.PhysicalGetIndexedFieldExprNodeH\x00\x42\n\n\x08\x45xprType\"}\n\x15PhysicalScalarUdfNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12*\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"\x84\x02\n\x19PhysicalAggregateExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12$\n\x1auser_defined_aggr_function\x18\x04 \x01(\tH\x00\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x36\n\x0cordering_req\x18\x05 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x42\x13\n\x11\x41ggregateFunction\"\xcf\x01\n\x16PhysicalWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12*\n\x04\x65xpr\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNodeB\x11\n\x0fwindow_function\"<\n\x0ePhysicalIsNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"?\n\x11PhysicalIsNotNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"9\n\x0bPhysicalNot\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"N\n\x11PhysicalAliasNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"v\n\x16PhysicalBinaryExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x80\x01\n PhysicalDateTimeIntervalExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x9c\x01\n\x14PhysicalLikeExprNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12\x18\n\x10\x63\x61se_insensitive\x18\x02 \x01(\x08\x12*\n\x04\x65xpr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12-\n\x07pattern\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"d\n\x14PhysicalSortExprNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"t\n\x10PhysicalWhenThen\x12/\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"}\n\x12PhysicalInListNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x04list\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"\xa5\x01\n\x10PhysicalCaseNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x34\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalWhenThen\x12/\n\telse_expr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\xab\x01\n\x1aPhysicalScalarFunctionNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\'\n\x03\x66un\x18\x02 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12*\n\x04\x61rgs\x18\x03 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"l\n\x13PhysicalTryCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"i\n\x10PhysicalCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"B\n\x14PhysicalNegativeNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"i\n\x0e\x46ilterExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"7\n\tFileGroup\x12*\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x1b.datafusion.PartitionedFile\"\x1a\n\tScanLimit\x12\r\n\x05limit\x18\x01 \x01(\r\"d\n\x1ePhysicalSortExprNodeCollection\x12\x42\n\x18physical_sort_expr_nodes\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\xcb\x02\n\x10\x46ileScanExecConf\x12*\n\x0b\x66ile_groups\x18\x01 \x03(\x0b\x32\x15.datafusion.FileGroup\x12\"\n\x06schema\x18\x02 \x01(\x0b\x32\x12.datafusion.Schema\x12\x12\n\nprojection\x18\x04 \x03(\r\x12$\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.ScanLimit\x12*\n\nstatistics\x18\x06 \x01(\x0b\x32\x16.datafusion.Statistics\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x18\n\x10object_store_url\x18\x08 \x01(\t\x12\x43\n\x0foutput_ordering\x18\t \x03(\x0b\x32*.datafusion.PhysicalSortExprNodeCollectionJ\x04\x08\n\x10\x0b\"}\n\x13ParquetScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12/\n\tpredicate\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNodeJ\x04\x08\x02\x10\x03\"i\n\x0f\x43svScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12\x12\n\nhas_header\x18\x02 \x01(\x08\x12\x11\n\tdelimiter\x18\x03 \x01(\t\"C\n\x10\x41vroScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\"\xa9\x02\n\x10HashJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x1e\n\x02on\x18\x03 \x03(\x0b\x32\x12.datafusion.JoinOn\x12\'\n\tjoin_type\x18\x04 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x31\n\x0epartition_mode\x18\x06 \x01(\x0e\x32\x19.datafusion.PartitionMode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12&\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x16.datafusion.JoinFilter\"=\n\rUnionExecNode\x12,\n\x06inputs\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"~\n\x0f\x45xplainExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\x12\x36\n\x11stringified_plans\x18\x02 \x03(\x0b\x32\x1b.datafusion.StringifiedPlan\x12\x0f\n\x07verbose\x18\x03 \x01(\x08\"l\n\x11\x43rossJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"-\n\x0ePhysicalColumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\r\"]\n\x06JoinOn\x12(\n\x04left\x18\x01 \x01(\x0b\x32\x1a.datafusion.PhysicalColumn\x12)\n\x05right\x18\x02 \x01(\x0b\x32\x1a.datafusion.PhysicalColumn\"L\n\rEmptyExecNode\x12\x17\n\x0fproduce_one_row\x18\x01 \x01(\x08\x12\"\n\x06schema\x18\x02 \x01(\x0b\x32\x12.datafusion.Schema\"\x80\x01\n\x12ProjectionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x11\n\texpr_name\x18\x03 \x03(\t\"\xb7\x01\n\x11WindowAggExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x31\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x18\n\x10window_expr_name\x18\x03 \x03(\t\x12(\n\x0cinput_schema\x18\x04 \x01(\x0b\x32\x12.datafusion.Schema\"9\n\x0bMaybeFilter\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"M\n\x16MaybePhysicalSortExprs\x12\x33\n\tsort_expr\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\xd1\x03\n\x11\x41ggregateExecNode\x12\x30\n\ngroup_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\taggr_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x04mode\x18\x03 \x01(\x0e\x32\x19.datafusion.AggregateMode\x12+\n\x05input\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x17\n\x0fgroup_expr_name\x18\x05 \x03(\t\x12\x16\n\x0e\x61ggr_expr_name\x18\x06 \x03(\t\x12(\n\x0cinput_schema\x18\x07 \x01(\x0b\x32\x12.datafusion.Schema\x12/\n\tnull_expr\x18\x08 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0e\n\x06groups\x18\t \x03(\x08\x12,\n\x0b\x66ilter_expr\x18\n \x03(\x0b\x32\x17.datafusion.MaybeFilter\x12\x39\n\rorder_by_expr\x18\x0b \x03(\x0b\x32\".datafusion.MaybePhysicalSortExprs\"_\n\x13GlobalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\r\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"P\n\x12LocalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\r\n\x05\x66\x65tch\x18\x02 \x01(\r\"\x95\x01\n\x0cSortExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\x12\x1d\n\x15preserve_partitioning\x18\x04 \x01(\x08\"\x85\x01\n\x1bSortPreservingMergeExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\xc2\x01\n\x16NestedLoopJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12&\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x16.datafusion.JoinFilter\"a\n\x17\x43oalesceBatchesExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x19\n\x11target_batch_size\x18\x02 \x01(\r\"I\n\x1a\x43oalescePartitionsExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"c\n\x17PhysicalHashRepartition\x12/\n\thash_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\"\xb5\x01\n\x13RepartitionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12\x33\n\x04hash\x18\x03 \x01(\x0b\x32#.datafusion.PhysicalHashRepartitionH\x00\x12\x11\n\x07unknown\x18\x04 \x01(\x04H\x00\x42\x12\n\x10partition_method\"\x93\x01\n\nJoinFilter\x12\x30\n\nexpression\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\x0e\x63olumn_indices\x18\x02 \x03(\x0b\x32\x17.datafusion.ColumnIndex\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\"@\n\x0b\x43olumnIndex\x12\r\n\x05index\x18\x01 \x01(\r\x12\"\n\x04side\x18\x02 \x01(\x0e\x32\x14.datafusion.JoinSide\"\xa0\x01\n\x0fPartitionedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\x12\x18\n\x10last_modified_ns\x18\x03 \x01(\x04\x12\x31\n\x10partition_values\x18\x04 \x03(\x0b\x32\x17.datafusion.ScalarValue\x12$\n\x05range\x18\x05 \x01(\x0b\x32\x15.datafusion.FileRange\"\'\n\tFileRange\x12\r\n\x05start\x18\x01 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x03\"y\n\x0ePartitionStats\x12\x10\n\x08num_rows\x18\x01 \x01(\x03\x12\x13\n\x0bnum_batches\x18\x02 \x01(\x03\x12\x11\n\tnum_bytes\x18\x03 \x01(\x03\x12-\n\x0c\x63olumn_stats\x18\x04 \x03(\x0b\x32\x17.datafusion.ColumnStats\"x\n\nStatistics\x12\x10\n\x08num_rows\x18\x01 \x01(\x03\x12\x17\n\x0ftotal_byte_size\x18\x02 \x01(\x03\x12-\n\x0c\x63olumn_stats\x18\x03 \x03(\x0b\x32\x17.datafusion.ColumnStats\x12\x10\n\x08is_exact\x18\x04 \x01(\x08\"\x91\x01\n\x0b\x43olumnStats\x12*\n\tmin_value\x18\x01 \x01(\x0b\x32\x17.datafusion.ScalarValue\x12*\n\tmax_value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\x12\x12\n\nnull_count\x18\x03 \x01(\r\x12\x16\n\x0e\x64istinct_count\x18\x04 \x01(\r\"r\n\x1fPhysicalGetIndexedFieldExprNode\x12)\n\x03\x61rg\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12$\n\x03key\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue*n\n\x08JoinType\x12\t\n\x05INNER\x10\x00\x12\x08\n\x04LEFT\x10\x01\x12\t\n\x05RIGHT\x10\x02\x12\x08\n\x04\x46ULL\x10\x03\x12\x0c\n\x08LEFTSEMI\x10\x04\x12\x0c\n\x08LEFTANTI\x10\x05\x12\r\n\tRIGHTSEMI\x10\x06\x12\r\n\tRIGHTANTI\x10\x07*#\n\x0eJoinConstraint\x12\x06\n\x02ON\x10\x00\x12\t\n\x05USING\x10\x01*\xe6\n\n\x0eScalarFunction\x12\x07\n\x03\x41\x62s\x10\x00\x12\x08\n\x04\x41\x63os\x10\x01\x12\x08\n\x04\x41sin\x10\x02\x12\x08\n\x04\x41tan\x10\x03\x12\t\n\x05\x41scii\x10\x04\x12\x08\n\x04\x43\x65il\x10\x05\x12\x07\n\x03\x43os\x10\x06\x12\n\n\x06\x44igest\x10\x07\x12\x07\n\x03\x45xp\x10\x08\x12\t\n\x05\x46loor\x10\t\x12\x06\n\x02Ln\x10\n\x12\x07\n\x03Log\x10\x0b\x12\t\n\x05Log10\x10\x0c\x12\x08\n\x04Log2\x10\r\x12\t\n\x05Round\x10\x0e\x12\n\n\x06Signum\x10\x0f\x12\x07\n\x03Sin\x10\x10\x12\x08\n\x04Sqrt\x10\x11\x12\x07\n\x03Tan\x10\x12\x12\t\n\x05Trunc\x10\x13\x12\t\n\x05\x41rray\x10\x14\x12\x0f\n\x0bRegexpMatch\x10\x15\x12\r\n\tBitLength\x10\x16\x12\t\n\x05\x42trim\x10\x17\x12\x13\n\x0f\x43haracterLength\x10\x18\x12\x07\n\x03\x43hr\x10\x19\x12\n\n\x06\x43oncat\x10\x1a\x12\x17\n\x13\x43oncatWithSeparator\x10\x1b\x12\x0c\n\x08\x44\x61tePart\x10\x1c\x12\r\n\tDateTrunc\x10\x1d\x12\x0b\n\x07InitCap\x10\x1e\x12\x08\n\x04Left\x10\x1f\x12\x08\n\x04Lpad\x10 \x12\t\n\x05Lower\x10!\x12\t\n\x05Ltrim\x10\"\x12\x07\n\x03MD5\x10#\x12\n\n\x06NullIf\x10$\x12\x0f\n\x0bOctetLength\x10%\x12\n\n\x06Random\x10&\x12\x11\n\rRegexpReplace\x10\'\x12\n\n\x06Repeat\x10(\x12\x0b\n\x07Replace\x10)\x12\x0b\n\x07Reverse\x10*\x12\t\n\x05Right\x10+\x12\x08\n\x04Rpad\x10,\x12\t\n\x05Rtrim\x10-\x12\n\n\x06SHA224\x10.\x12\n\n\x06SHA256\x10/\x12\n\n\x06SHA384\x10\x30\x12\n\n\x06SHA512\x10\x31\x12\r\n\tSplitPart\x10\x32\x12\x0e\n\nStartsWith\x10\x33\x12\n\n\x06Strpos\x10\x34\x12\n\n\x06Substr\x10\x35\x12\t\n\x05ToHex\x10\x36\x12\x0f\n\x0bToTimestamp\x10\x37\x12\x15\n\x11ToTimestampMillis\x10\x38\x12\x15\n\x11ToTimestampMicros\x10\x39\x12\x16\n\x12ToTimestampSeconds\x10:\x12\x07\n\x03Now\x10;\x12\r\n\tTranslate\x10<\x12\x08\n\x04Trim\x10=\x12\t\n\x05Upper\x10>\x12\x0c\n\x08\x43oalesce\x10?\x12\t\n\x05Power\x10@\x12\r\n\tStructFun\x10\x41\x12\x10\n\x0c\x46romUnixtime\x10\x42\x12\t\n\x05\x41tan2\x10\x43\x12\x0b\n\x07\x44\x61teBin\x10\x44\x12\x0f\n\x0b\x41rrowTypeof\x10\x45\x12\x0f\n\x0b\x43urrentDate\x10\x46\x12\x0f\n\x0b\x43urrentTime\x10G\x12\x08\n\x04Uuid\x10H\x12\x08\n\x04\x43\x62rt\x10I\x12\t\n\x05\x41\x63osh\x10J\x12\t\n\x05\x41sinh\x10K\x12\t\n\x05\x41tanh\x10L\x12\x08\n\x04Sinh\x10M\x12\x08\n\x04\x43osh\x10N\x12\x08\n\x04Tanh\x10O\x12\x06\n\x02Pi\x10P\x12\x0b\n\x07\x44\x65grees\x10Q\x12\x0b\n\x07Radians\x10R\x12\r\n\tFactorial\x10S\x12\x07\n\x03Lcm\x10T\x12\x07\n\x03Gcd\x10U\x12\x0f\n\x0b\x41rrayAppend\x10V\x12\x0f\n\x0b\x41rrayConcat\x10W\x12\r\n\tArrayDims\x10X\x12\r\n\tArrayFill\x10Y\x12\x0f\n\x0b\x41rrayLength\x10Z\x12\x0e\n\nArrayNdims\x10[\x12\x11\n\rArrayPosition\x10\\\x12\x12\n\x0e\x41rrayPositions\x10]\x12\x10\n\x0c\x41rrayPrepend\x10^\x12\x0f\n\x0b\x41rrayRemove\x10_\x12\x10\n\x0c\x41rrayReplace\x10`\x12\x11\n\rArrayToString\x10\x61\x12\x0f\n\x0b\x43\x61rdinality\x10\x62\x12\r\n\tTrimArray\x10\x63\x12\x11\n\rArrayContains\x10\x64\x12\n\n\x06\x45ncode\x10\x65\x12\n\n\x06\x44\x65\x63ode\x10\x66\x12\x07\n\x03\x43ot\x10g*\xb2\x03\n\x11\x41ggregateFunction\x12\x07\n\x03MIN\x10\x00\x12\x07\n\x03MAX\x10\x01\x12\x07\n\x03SUM\x10\x02\x12\x07\n\x03\x41VG\x10\x03\x12\t\n\x05\x43OUNT\x10\x04\x12\x13\n\x0f\x41PPROX_DISTINCT\x10\x05\x12\r\n\tARRAY_AGG\x10\x06\x12\x0c\n\x08VARIANCE\x10\x07\x12\x10\n\x0cVARIANCE_POP\x10\x08\x12\x0e\n\nCOVARIANCE\x10\t\x12\x12\n\x0e\x43OVARIANCE_POP\x10\n\x12\n\n\x06STDDEV\x10\x0b\x12\x0e\n\nSTDDEV_POP\x10\x0c\x12\x0f\n\x0b\x43ORRELATION\x10\r\x12\x1a\n\x16\x41PPROX_PERCENTILE_CONT\x10\x0e\x12\x11\n\rAPPROX_MEDIAN\x10\x0f\x12&\n\"APPROX_PERCENTILE_CONT_WITH_WEIGHT\x10\x10\x12\x0c\n\x08GROUPING\x10\x11\x12\n\n\x06MEDIAN\x10\x12\x12\x0b\n\x07\x42IT_AND\x10\x13\x12\n\n\x06\x42IT_OR\x10\x14\x12\x0b\n\x07\x42IT_XOR\x10\x15\x12\x0c\n\x08\x42OOL_AND\x10\x16\x12\x0b\n\x07\x42OOL_OR\x10\x17\x12\x13\n\x0f\x46IRST_VALUE_AGG\x10\x18\x12\x12\n\x0eLAST_VALUE_AGG\x10\x19*\xb0\x01\n\x15\x42uiltInWindowFunction\x12\x0e\n\nROW_NUMBER\x10\x00\x12\x08\n\x04RANK\x10\x01\x12\x0e\n\nDENSE_RANK\x10\x02\x12\x10\n\x0cPERCENT_RANK\x10\x03\x12\r\n\tCUME_DIST\x10\x04\x12\t\n\x05NTILE\x10\x05\x12\x07\n\x03LAG\x10\x06\x12\x08\n\x04LEAD\x10\x07\x12\x0f\n\x0b\x46IRST_VALUE\x10\x08\x12\x0e\n\nLAST_VALUE\x10\t\x12\r\n\tNTH_VALUE\x10\n*3\n\x10WindowFrameUnits\x12\x08\n\x04ROWS\x10\x00\x12\t\n\x05RANGE\x10\x01\x12\n\n\x06GROUPS\x10\x02*E\n\x14WindowFrameBoundType\x12\x0f\n\x0b\x43URRENT_ROW\x10\x00\x12\r\n\tPRECEDING\x10\x01\x12\r\n\tFOLLOWING\x10\x02*(\n\x08\x44\x61teUnit\x12\x07\n\x03\x44\x61y\x10\x00\x12\x13\n\x0f\x44\x61teMillisecond\x10\x01*H\n\x08TimeUnit\x12\n\n\x06Second\x10\x00\x12\x0f\n\x0bMillisecond\x10\x01\x12\x0f\n\x0bMicrosecond\x10\x02\x12\x0e\n\nNanosecond\x10\x03*<\n\x0cIntervalUnit\x12\r\n\tYearMonth\x10\x00\x12\x0b\n\x07\x44\x61yTime\x10\x01\x12\x10\n\x0cMonthDayNano\x10\x02*\"\n\tUnionMode\x12\n\n\x06sparse\x10\x00\x12\t\n\x05\x64\x65nse\x10\x01*<\n\rPartitionMode\x12\x10\n\x0c\x43OLLECT_LEFT\x10\x00\x12\x0f\n\x0bPARTITIONED\x10\x01\x12\x08\n\x04\x41UTO\x10\x02*b\n\rAggregateMode\x12\x0b\n\x07PARTIAL\x10\x00\x12\t\n\x05\x46INAL\x10\x01\x12\x15\n\x11\x46INAL_PARTITIONED\x10\x02\x12\n\n\x06SINGLE\x10\x03\x12\x16\n\x12SINGLE_PARTITIONED\x10\x04*)\n\x08JoinSide\x12\r\n\tLEFT_SIDE\x10\x00\x12\x0e\n\nRIGHT_SIDE\x10\x01\x42\x39\n$org.apache.arrow.datafusion.protobufB\x0f\x44\x61tafusionProtoP\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16proto/datafusion.proto\x12\ndatafusion\"\"\n\x0e\x43olumnRelation\x12\x10\n\x08relation\x18\x01 \x01(\t\"D\n\x06\x43olumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\x08relation\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"Z\n\x07\x44\x66\x46ield\x12 \n\x05\x66ield\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12-\n\tqualifier\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"\x97\x01\n\x08\x44\x66Schema\x12$\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x13.datafusion.DfField\x12\x34\n\x08metadata\x18\x02 \x03(\x0b\x32\".datafusion.DfSchema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x91\x0b\n\x0fLogicalPlanNode\x12\x38\n\x0clisting_scan\x18\x01 \x01(\x0b\x32 .datafusion.ListingTableScanNodeH\x00\x12\x30\n\nprojection\x18\x03 \x01(\x0b\x32\x1a.datafusion.ProjectionNodeH\x00\x12.\n\tselection\x18\x04 \x01(\x0b\x32\x19.datafusion.SelectionNodeH\x00\x12&\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.LimitNodeH\x00\x12.\n\taggregate\x18\x06 \x01(\x0b\x32\x19.datafusion.AggregateNodeH\x00\x12$\n\x04join\x18\x07 \x01(\x0b\x32\x14.datafusion.JoinNodeH\x00\x12$\n\x04sort\x18\x08 \x01(\x0b\x32\x14.datafusion.SortNodeH\x00\x12\x32\n\x0brepartition\x18\t \x01(\x0b\x32\x1b.datafusion.RepartitionNodeH\x00\x12\x37\n\x0e\x65mpty_relation\x18\n \x01(\x0b\x32\x1d.datafusion.EmptyRelationNodeH\x00\x12\x44\n\x15\x63reate_external_table\x18\x0b \x01(\x0b\x32#.datafusion.CreateExternalTableNodeH\x00\x12*\n\x07\x65xplain\x18\x0c \x01(\x0b\x32\x17.datafusion.ExplainNodeH\x00\x12(\n\x06window\x18\r \x01(\x0b\x32\x16.datafusion.WindowNodeH\x00\x12*\n\x07\x61nalyze\x18\x0e \x01(\x0b\x32\x17.datafusion.AnalyzeNodeH\x00\x12/\n\ncross_join\x18\x0f \x01(\x0b\x32\x19.datafusion.CrossJoinNodeH\x00\x12(\n\x06values\x18\x10 \x01(\x0b\x32\x16.datafusion.ValuesNodeH\x00\x12\x35\n\textension\x18\x11 \x01(\x0b\x32 .datafusion.LogicalExtensionNodeH\x00\x12\x44\n\x15\x63reate_catalog_schema\x18\x12 \x01(\x0b\x32#.datafusion.CreateCatalogSchemaNodeH\x00\x12&\n\x05union\x18\x13 \x01(\x0b\x32\x15.datafusion.UnionNodeH\x00\x12\x37\n\x0e\x63reate_catalog\x18\x14 \x01(\x0b\x32\x1d.datafusion.CreateCatalogNodeH\x00\x12\x37\n\x0esubquery_alias\x18\x15 \x01(\x0b\x32\x1d.datafusion.SubqueryAliasNodeH\x00\x12\x31\n\x0b\x63reate_view\x18\x16 \x01(\x0b\x32\x1a.datafusion.CreateViewNodeH\x00\x12,\n\x08\x64istinct\x18\x17 \x01(\x0b\x32\x18.datafusion.DistinctNodeH\x00\x12\x32\n\tview_scan\x18\x18 \x01(\x0b\x32\x1d.datafusion.ViewTableScanNodeH\x00\x12\x36\n\x0b\x63ustom_scan\x18\x19 \x01(\x0b\x32\x1f.datafusion.CustomTableScanNodeH\x00\x12*\n\x07prepare\x18\x1a \x01(\x0b\x32\x17.datafusion.PrepareNodeH\x00\x12-\n\tdrop_view\x18\x1b \x01(\x0b\x32\x18.datafusion.DropViewNodeH\x00\x12\x31\n\x0b\x64istinct_on\x18\x1c \x01(\x0b\x32\x1a.datafusion.DistinctOnNodeH\x00\x12)\n\x07\x63opy_to\x18\x1d \x01(\x0b\x32\x16.datafusion.CopyToNodeH\x00\x42\x11\n\x0fLogicalPlanType\"Q\n\x14LogicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"$\n\x11ProjectionColumns\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\"f\n\tCsvFormat\x12\x12\n\nhas_header\x18\x01 \x01(\x08\x12\x11\n\tdelimiter\x18\x02 \x01(\t\x12\r\n\x05quote\x18\x03 \x01(\t\x12\x10\n\x06\x65scape\x18\x04 \x01(\tH\x00\x42\x11\n\x0foptional_escape\"\x15\n\rParquetFormatJ\x04\x08\x01\x10\x02\"\x0c\n\nAvroFormat\"T\n\x19LogicalExprNodeCollection\x12\x37\n\x12logical_expr_nodes\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x9a\x04\n\x14ListingTableScanNode\x12\x33\n\ntable_name\x18\x0e \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\r\n\x05paths\x18\x02 \x03(\t\x12\x16\n\x0e\x66ile_extension\x18\x03 \x01(\t\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x05 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x14\n\x0c\x63ollect_stat\x18\x08 \x01(\x08\x12\x19\n\x11target_partitions\x18\t \x01(\r\x12$\n\x03\x63sv\x18\n \x01(\x0b\x32\x15.datafusion.CsvFormatH\x00\x12,\n\x07parquet\x18\x0b \x01(\x0b\x32\x19.datafusion.ParquetFormatH\x00\x12&\n\x04\x61vro\x18\x0c \x01(\x0b\x32\x16.datafusion.AvroFormatH\x00\x12>\n\x0f\x66ile_sort_order\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollectionB\x10\n\x0e\x46ileFormatTypeJ\x04\x08\x01\x10\x02\"\xe5\x01\n\x11ViewTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\x12\n\ndefinition\x18\x05 \x01(\tJ\x04\x08\x01\x10\x02\"\xf0\x01\n\x13\x43ustomTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x31\n\nprojection\x18\x02 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x19\n\x11\x63ustom_table_data\x18\x05 \x01(\x0cJ\x04\x08\x01\x10\x02\"\x8a\x01\n\x0eProjectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x05\x61lias\x18\x03 \x01(\tH\x00\x42\x10\n\x0eoptional_alias\"f\n\rSelectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"p\n\x08SortNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\x95\x01\n\x0fRepartitionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12+\n\x04hash\x18\x03 \x01(\x0b\x32\x1b.datafusion.HashRepartitionH\x00\x42\x12\n\x10partition_method\"Z\n\x0fHashRepartition\x12.\n\thash_expr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\",\n\x11\x45mptyRelationNode\x12\x17\n\x0fproduce_one_row\x18\x01 \x01(\x08\"\'\n\x14PrimaryKeyConstraint\x12\x0f\n\x07indices\x18\x01 \x03(\x04\"#\n\x10UniqueConstraint\x12\x0f\n\x07indices\x18\x01 \x03(\x04\"\x88\x01\n\nConstraint\x12\x37\n\x0bprimary_key\x18\x01 \x01(\x0b\x32 .datafusion.PrimaryKeyConstraintH\x00\x12.\n\x06unique\x18\x02 \x01(\x0b\x32\x1c.datafusion.UniqueConstraintH\x00\x42\x11\n\x0f\x63onstraint_mode\":\n\x0b\x43onstraints\x12+\n\x0b\x63onstraints\x18\x01 \x03(\x0b\x32\x16.datafusion.Constraint\"\xe8\x05\n\x17\x43reateExternalTableNode\x12-\n\x04name\x18\x0c \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x10\n\x08location\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x12\n\nhas_header\x18\x04 \x01(\x08\x12$\n\x06schema\x18\x05 \x01(\x0b\x32\x14.datafusion.DfSchema\x12\x1c\n\x14table_partition_cols\x18\x06 \x03(\t\x12\x15\n\rif_not_exists\x18\x07 \x01(\x08\x12\x11\n\tdelimiter\x18\x08 \x01(\t\x12\x12\n\ndefinition\x18\t \x01(\t\x12\x41\n\x15\x66ile_compression_type\x18\x11 \x01(\x0e\x32\".datafusion.CompressionTypeVariant\x12:\n\x0border_exprs\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollection\x12\x11\n\tunbounded\x18\x0e \x01(\x08\x12\x41\n\x07options\x18\x0b \x03(\x0b\x32\x30.datafusion.CreateExternalTableNode.OptionsEntry\x12,\n\x0b\x63onstraints\x18\x0f \x01(\x0b\x32\x17.datafusion.Constraints\x12P\n\x0f\x63olumn_defaults\x18\x10 \x03(\x0b\x32\x37.datafusion.CreateExternalTableNode.ColumnDefaultsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aR\n\x13\x43olumnDefaultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode:\x02\x38\x01J\x04\x08\x01\x10\x02J\x04\x08\n\x10\x0b\"r\n\x0bPrepareNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\ndata_types\x18\x02 \x03(\x0b\x32\x15.datafusion.ArrowType\x12*\n\x05input\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"k\n\x17\x43reateCatalogSchemaNode\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"f\n\x11\x43reateCatalogNode\x12\x14\n\x0c\x63\x61talog_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"v\n\x0c\x44ropViewNode\x12-\n\x04name\x18\x01 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x11\n\tif_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"\x99\x01\n\x0e\x43reateViewNode\x12-\n\x04name\x18\x05 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x12\n\nor_replace\x18\x03 \x01(\x08\x12\x12\n\ndefinition\x18\x04 \x01(\tJ\x04\x08\x01\x10\x02\"N\n\nValuesNode\x12\x0e\n\x06n_cols\x18\x01 \x01(\x04\x12\x30\n\x0bvalues_list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"J\n\x0b\x41nalyzeNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"J\n\x0b\x45xplainNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"\x9c\x01\n\rAggregateNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12/\n\ngroup_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\taggr_expr\x18\x03 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\nWindowNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x30\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xef\x02\n\x08JoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x33\n\x0fjoin_constraint\x18\x04 \x01(\x0e\x32\x1a.datafusion.JoinConstraint\x12\x32\n\rleft_join_key\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x33\n\x0eright_join_key\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12+\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\":\n\x0c\x44istinctNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"\xcc\x01\n\x0e\x44istinctOnNode\x12,\n\x07on_expr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x30\n\x0bselect_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\tsort_expr\x18\x03 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12*\n\x05input\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"\xda\x01\n\nCopyToNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x12\n\noutput_url\x18\x02 \x01(\t\x12-\n\x0bsql_options\x18\x04 \x01(\x0b\x32\x16.datafusion.SQLOptionsH\x00\x12;\n\x0ewriter_options\x18\x05 \x01(\x0b\x32!.datafusion.FileTypeWriterOptionsH\x00\x12\x11\n\tfile_type\x18\x06 \x01(\tB\r\n\x0b\x43opyOptions\"3\n\nSQLOptions\x12%\n\x06option\x18\x01 \x03(\x0b\x32\x15.datafusion.SQLOption\"\'\n\tSQLOption\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"8\n\tUnionNode\x12+\n\x06inputs\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"f\n\rCrossJoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"T\n\tLimitNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\x03\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\">\n\x11SelectionExecNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"u\n\x11SubqueryAliasNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12.\n\x05\x61lias\x18\x03 \x01(\x0b\x32\x1f.datafusion.OwnedTableReferenceJ\x04\x08\x02\x10\x03\"\xe9\x0c\n\x0fLogicalExprNode\x12$\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x12.datafusion.ColumnH\x00\x12&\n\x05\x61lias\x18\x02 \x01(\x0b\x32\x15.datafusion.AliasNodeH\x00\x12*\n\x07literal\x18\x03 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x31\n\x0b\x62inary_expr\x18\x04 \x01(\x0b\x32\x1a.datafusion.BinaryExprNodeH\x00\x12\x37\n\x0e\x61ggregate_expr\x18\x05 \x01(\x0b\x32\x1d.datafusion.AggregateExprNodeH\x00\x12*\n\x0cis_null_expr\x18\x06 \x01(\x0b\x32\x12.datafusion.IsNullH\x00\x12\x31\n\x10is_not_null_expr\x18\x07 \x01(\x0b\x32\x15.datafusion.IsNotNullH\x00\x12#\n\x08not_expr\x18\x08 \x01(\x0b\x32\x0f.datafusion.NotH\x00\x12*\n\x07\x62\x65tween\x18\t \x01(\x0b\x32\x17.datafusion.BetweenNodeH\x00\x12%\n\x05\x63\x61se_\x18\n \x01(\x0b\x32\x14.datafusion.CaseNodeH\x00\x12$\n\x04\x63\x61st\x18\x0b \x01(\x0b\x32\x14.datafusion.CastNodeH\x00\x12(\n\x04sort\x18\x0c \x01(\x0b\x32\x18.datafusion.SortExprNodeH\x00\x12,\n\x08negative\x18\r \x01(\x0b\x32\x18.datafusion.NegativeNodeH\x00\x12)\n\x07in_list\x18\x0e \x01(\x0b\x32\x16.datafusion.InListNodeH\x00\x12(\n\x08wildcard\x18\x0f \x01(\x0b\x32\x14.datafusion.WildcardH\x00\x12\x39\n\x0fscalar_function\x18\x10 \x01(\x0b\x32\x1e.datafusion.ScalarFunctionNodeH\x00\x12+\n\x08try_cast\x18\x11 \x01(\x0b\x32\x17.datafusion.TryCastNodeH\x00\x12\x31\n\x0bwindow_expr\x18\x12 \x01(\x0b\x32\x1a.datafusion.WindowExprNodeH\x00\x12>\n\x12\x61ggregate_udf_expr\x18\x13 \x01(\x0b\x32 .datafusion.AggregateUDFExprNodeH\x00\x12\x38\n\x0fscalar_udf_expr\x18\x14 \x01(\x0b\x32\x1d.datafusion.ScalarUDFExprNodeH\x00\x12\x38\n\x11get_indexed_field\x18\x15 \x01(\x0b\x32\x1b.datafusion.GetIndexedFieldH\x00\x12\x33\n\x0cgrouping_set\x18\x16 \x01(\x0b\x32\x1b.datafusion.GroupingSetNodeH\x00\x12$\n\x04\x63ube\x18\x17 \x01(\x0b\x32\x14.datafusion.CubeNodeH\x00\x12(\n\x06rollup\x18\x18 \x01(\x0b\x32\x16.datafusion.RollupNodeH\x00\x12%\n\x07is_true\x18\x19 \x01(\x0b\x32\x12.datafusion.IsTrueH\x00\x12\'\n\x08is_false\x18\x1a \x01(\x0b\x32\x13.datafusion.IsFalseH\x00\x12+\n\nis_unknown\x18\x1b \x01(\x0b\x32\x15.datafusion.IsUnknownH\x00\x12,\n\x0bis_not_true\x18\x1c \x01(\x0b\x32\x15.datafusion.IsNotTrueH\x00\x12.\n\x0cis_not_false\x18\x1d \x01(\x0b\x32\x16.datafusion.IsNotFalseH\x00\x12\x32\n\x0eis_not_unknown\x18\x1e \x01(\x0b\x32\x18.datafusion.IsNotUnknownH\x00\x12$\n\x04like\x18\x1f \x01(\x0b\x32\x14.datafusion.LikeNodeH\x00\x12&\n\x05ilike\x18 \x01(\x0b\x32\x15.datafusion.ILikeNodeH\x00\x12/\n\nsimilar_to\x18! \x01(\x0b\x32\x19.datafusion.SimilarToNodeH\x00\x12\x32\n\x0bplaceholder\x18\" \x01(\x0b\x32\x1b.datafusion.PlaceholderNodeH\x00\x12$\n\x06unnest\x18# \x01(\x0b\x32\x12.datafusion.UnnestH\x00\x42\n\n\x08\x45xprType\"\x1d\n\x08Wildcard\x12\x11\n\tqualifier\x18\x01 \x01(\t\"G\n\x0fPlaceholderNode\x12\n\n\x02id\x18\x01 \x01(\t\x12(\n\tdata_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"<\n\x0fLogicalExprList\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"<\n\x0fGroupingSetNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprList\"5\n\x08\x43ubeNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nRollupNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"9\n\x10NamedStructField\x12%\n\x04name\x18\x01 \x01(\x0b\x32\x17.datafusion.ScalarValue\"5\n\tListIndex\x12(\n\x03key\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x8f\x01\n\tListRange\x12*\n\x05start\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04stop\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12+\n\x06stride\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xdb\x01\n\x0fGetIndexedField\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12:\n\x12named_struct_field\x18\x02 \x01(\x0b\x32\x1c.datafusion.NamedStructFieldH\x00\x12+\n\nlist_index\x18\x03 \x01(\x0b\x32\x15.datafusion.ListIndexH\x00\x12+\n\nlist_range\x18\x04 \x01(\x0b\x32\x15.datafusion.ListRangeH\x00\x42\x07\n\x05\x66ield\"3\n\x06IsNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"3\n\x06IsTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"4\n\x07IsFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nIsNotFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"9\n\x0cIsNotUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"0\n\x03Not\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"x\n\tAliasNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\x12\x31\n\x08relation\x18\x03 \x03(\x0b\x32\x1f.datafusion.OwnedTableReference\"K\n\x0e\x42inaryExprNode\x12-\n\x08operands\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"9\n\x0cNegativeNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"4\n\x06Unnest\x12*\n\x05\x65xprs\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"s\n\nInListNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"h\n\x12ScalarFunctionNode\x12\'\n\x03\x66un\x18\x01 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xe2\x01\n\x11\x41ggregateExprNode\x12\x34\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunction\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x12+\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xaf\x01\n\x14\x41ggregateUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12+\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"P\n\x11ScalarUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xf7\x02\n\x0eWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12\x0e\n\x04udaf\x18\x03 \x01(\tH\x00\x12\x0e\n\x04udwf\x18\t \x01(\tH\x00\x12)\n\x04\x65xpr\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x31\n\x0cpartition_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x0cwindow_frame\x18\x08 \x01(\x0b\x32\x17.datafusion.WindowFrameB\x11\n\x0fwindow_function\"\x9e\x01\n\x0b\x42\x65tweenNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x02 \x01(\x08\x12(\n\x03low\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04high\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x89\x01\n\x08LikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8a\x01\n\tILikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8e\x01\n\rSimilarToNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x93\x01\n\x08\x43\x61seNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x14.datafusion.WhenThen\x12.\n\telse_expr\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\x08WhenThen\x12.\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"`\n\x08\x43\x61stNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"c\n\x0bTryCastNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"[\n\x0cSortExprNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"\xb6\x01\n\x0bWindowFrame\x12\x38\n\x12window_frame_units\x18\x01 \x01(\x0e\x32\x1c.datafusion.WindowFrameUnits\x12\x31\n\x0bstart_bound\x18\x02 \x01(\x0b\x32\x1c.datafusion.WindowFrameBound\x12-\n\x05\x62ound\x18\x03 \x01(\x0b\x32\x1c.datafusion.WindowFrameBoundH\x00\x42\x0b\n\tend_bound\"\x83\x01\n\x10WindowFrameBound\x12\x41\n\x17window_frame_bound_type\x18\x01 \x01(\x0e\x32 .datafusion.WindowFrameBoundType\x12,\n\x0b\x62ound_value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"\x91\x01\n\x06Schema\x12\"\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12\x32\n\x08metadata\x18\x02 \x03(\x0b\x32 .datafusion.Schema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x82\x02\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\x12\x10\n\x08nullable\x18\x03 \x01(\x08\x12#\n\x08\x63hildren\x18\x04 \x03(\x0b\x32\x11.datafusion.Field\x12\x31\n\x08metadata\x18\x05 \x03(\x0b\x32\x1f.datafusion.Field.MetadataEntry\x12\x0f\n\x07\x64ict_id\x18\x06 \x01(\x03\x12\x14\n\x0c\x64ict_ordered\x18\x07 \x01(\x08\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"!\n\x0f\x46ixedSizeBinary\x12\x0e\n\x06length\x18\x01 \x01(\x05\"F\n\tTimestamp\x12\'\n\ttime_unit\x18\x01 \x01(\x0e\x32\x14.datafusion.TimeUnit\x12\x10\n\x08timezone\x18\x02 \x01(\t\"7\n\x07\x44\x65\x63imal\x12\x11\n\tprecision\x18\x03 \x01(\r\x12\r\n\x05scale\x18\x04 \x01(\x05J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03\"-\n\x04List\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\"I\n\rFixedSizeList\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x11\n\tlist_size\x18\x02 \x01(\x05\"V\n\nDictionary\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"4\n\x06Struct\x12*\n\x0fsub_field_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\"A\n\x03Map\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x13\n\x0bkeys_sorted\x18\x02 \x01(\x08\"l\n\x05Union\x12&\n\x0bunion_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12)\n\nunion_mode\x18\x02 \x01(\x0e\x32\x15.datafusion.UnionMode\x12\x10\n\x08type_ids\x18\x03 \x03(\x05\"`\n\x11ScalarNestedValue\x12\x13\n\x0bipc_message\x18\x01 \x01(\x0c\x12\x12\n\narrow_data\x18\x02 \x01(\x0c\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\"_\n\x11ScalarTime32Value\x12\x1d\n\x13time32_second_value\x18\x01 \x01(\x05H\x00\x12\"\n\x18time32_millisecond_value\x18\x02 \x01(\x05H\x00\x42\x07\n\x05value\"c\n\x11ScalarTime64Value\x12\"\n\x18time64_microsecond_value\x18\x01 \x01(\x03H\x00\x12!\n\x17time64_nanosecond_value\x18\x02 \x01(\x03H\x00\x42\x07\n\x05value\"\xb3\x01\n\x14ScalarTimestampValue\x12 \n\x16time_microsecond_value\x18\x01 \x01(\x03H\x00\x12\x1f\n\x15time_nanosecond_value\x18\x02 \x01(\x03H\x00\x12\x1b\n\x11time_second_value\x18\x03 \x01(\x03H\x00\x12 \n\x16time_millisecond_value\x18\x04 \x01(\x03H\x00\x12\x10\n\x08timezone\x18\x05 \x01(\tB\x07\n\x05value\"j\n\x15ScalarDictionaryValue\x12)\n\nindex_type\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"H\n\x19IntervalMonthDayNanoValue\x12\x0e\n\x06months\x18\x01 \x01(\x05\x12\x0c\n\x04\x64\x61ys\x18\x02 \x01(\x05\x12\r\n\x05nanos\x18\x03 \x01(\x03\"7\n\x15ScalarFixedSizeBinary\x12\x0e\n\x06values\x18\x01 \x01(\x0c\x12\x0e\n\x06length\x18\x02 \x01(\x05\"\x86\x0b\n\x0bScalarValue\x12+\n\nnull_value\x18! \x01(\x0b\x32\x15.datafusion.ArrowTypeH\x00\x12\x14\n\nbool_value\x18\x01 \x01(\x08H\x00\x12\x14\n\nutf8_value\x18\x02 \x01(\tH\x00\x12\x1a\n\x10large_utf8_value\x18\x03 \x01(\tH\x00\x12\x14\n\nint8_value\x18\x04 \x01(\x05H\x00\x12\x15\n\x0bint16_value\x18\x05 \x01(\x05H\x00\x12\x15\n\x0bint32_value\x18\x06 \x01(\x05H\x00\x12\x15\n\x0bint64_value\x18\x07 \x01(\x03H\x00\x12\x15\n\x0buint8_value\x18\x08 \x01(\rH\x00\x12\x16\n\x0cuint16_value\x18\t \x01(\rH\x00\x12\x16\n\x0cuint32_value\x18\n \x01(\rH\x00\x12\x16\n\x0cuint64_value\x18\x0b \x01(\x04H\x00\x12\x17\n\rfloat32_value\x18\x0c \x01(\x02H\x00\x12\x17\n\rfloat64_value\x18\r \x01(\x01H\x00\x12\x17\n\rdate_32_value\x18\x0e \x01(\x05H\x00\x12\x35\n\x0ctime32_value\x18\x0f \x01(\x0b\x32\x1d.datafusion.ScalarTime32ValueH\x00\x12\x39\n\x10large_list_value\x18\x10 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12\x33\n\nlist_value\x18\x11 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12>\n\x15\x66ixed_size_list_value\x18\x12 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12\x35\n\x0cstruct_value\x18 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12\x32\n\x10\x64\x65\x63imal128_value\x18\x14 \x01(\x0b\x32\x16.datafusion.Decimal128H\x00\x12\x32\n\x10\x64\x65\x63imal256_value\x18\' \x01(\x0b\x32\x16.datafusion.Decimal256H\x00\x12\x17\n\rdate_64_value\x18\x15 \x01(\x03H\x00\x12\"\n\x18interval_yearmonth_value\x18\x18 \x01(\x05H\x00\x12 \n\x16interval_daytime_value\x18\x19 \x01(\x03H\x00\x12\x1f\n\x15\x64uration_second_value\x18# \x01(\x03H\x00\x12$\n\x1a\x64uration_millisecond_value\x18$ \x01(\x03H\x00\x12$\n\x1a\x64uration_microsecond_value\x18% \x01(\x03H\x00\x12#\n\x19\x64uration_nanosecond_value\x18& \x01(\x03H\x00\x12;\n\x0ftimestamp_value\x18\x1a \x01(\x0b\x32 .datafusion.ScalarTimestampValueH\x00\x12=\n\x10\x64ictionary_value\x18\x1b \x01(\x0b\x32!.datafusion.ScalarDictionaryValueH\x00\x12\x16\n\x0c\x62inary_value\x18\x1c \x01(\x0cH\x00\x12\x1c\n\x12large_binary_value\x18\x1d \x01(\x0cH\x00\x12\x35\n\x0ctime64_value\x18\x1e \x01(\x0b\x32\x1d.datafusion.ScalarTime64ValueH\x00\x12H\n\x17interval_month_day_nano\x18\x1f \x01(\x0b\x32%.datafusion.IntervalMonthDayNanoValueH\x00\x12\x44\n\x17\x66ixed_size_binary_value\x18\" \x01(\x0b\x32!.datafusion.ScalarFixedSizeBinaryH\x00\x42\x07\n\x05valueJ\x04\x08\x13\x10\x14\"1\n\nDecimal128\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\t\n\x01p\x18\x02 \x01(\x03\x12\t\n\x01s\x18\x03 \x01(\x03\"1\n\nDecimal256\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\t\n\x01p\x18\x02 \x01(\x03\x12\t\n\x01s\x18\x03 \x01(\x03\"\x94\x0b\n\tArrowType\x12(\n\x04NONE\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04\x42OOL\x18\x02 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05UINT8\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04INT8\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT16\x18\x05 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT16\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT32\x18\x07 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT32\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT64\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT64\x18\n \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT16\x18\x0b \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT32\x18\x0c \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT64\x18\r \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04UTF8\x18\x0e \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12.\n\nLARGE_UTF8\x18 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x42INARY\x18\x0f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x1b\n\x11\x46IXED_SIZE_BINARY\x18\x10 \x01(\x05H\x00\x12\x30\n\x0cLARGE_BINARY\x18\x1f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE32\x18\x11 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE64\x18\x12 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x08\x44URATION\x18\x13 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12*\n\tTIMESTAMP\x18\x14 \x01(\x0b\x32\x15.datafusion.TimestampH\x00\x12&\n\x06TIME32\x18\x15 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12&\n\x06TIME64\x18\x16 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12,\n\x08INTERVAL\x18\x17 \x01(\x0e\x32\x18.datafusion.IntervalUnitH\x00\x12&\n\x07\x44\x45\x43IMAL\x18\x18 \x01(\x0b\x32\x13.datafusion.DecimalH\x00\x12 \n\x04LIST\x18\x19 \x01(\x0b\x32\x10.datafusion.ListH\x00\x12&\n\nLARGE_LIST\x18\x1a \x01(\x0b\x32\x10.datafusion.ListH\x00\x12\x34\n\x0f\x46IXED_SIZE_LIST\x18\x1b \x01(\x0b\x32\x19.datafusion.FixedSizeListH\x00\x12$\n\x06STRUCT\x18\x1c \x01(\x0b\x32\x12.datafusion.StructH\x00\x12\"\n\x05UNION\x18\x1d \x01(\x0b\x32\x11.datafusion.UnionH\x00\x12,\n\nDICTIONARY\x18\x1e \x01(\x0b\x32\x16.datafusion.DictionaryH\x00\x12\x1e\n\x03MAP\x18! \x01(\x0b\x32\x0f.datafusion.MapH\x00\x42\x11\n\x0f\x61rrow_type_enum\"\x0e\n\x0c\x45mptyMessage\"0\n\x17\x41nalyzedLogicalPlanType\x12\x15\n\ranalyzer_name\x18\x01 \x01(\t\"2\n\x18OptimizedLogicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"3\n\x19OptimizedPhysicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"\x8c\x05\n\x08PlanType\x12\x36\n\x12InitialLogicalPlan\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x42\n\x13\x41nalyzedLogicalPlan\x18\x07 \x01(\x0b\x32#.datafusion.AnalyzedLogicalPlanTypeH\x00\x12<\n\x18\x46inalAnalyzedLogicalPlan\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x44\n\x14OptimizedLogicalPlan\x18\x02 \x01(\x0b\x32$.datafusion.OptimizedLogicalPlanTypeH\x00\x12\x34\n\x10\x46inalLogicalPlan\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x37\n\x13InitialPhysicalPlan\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12@\n\x1cInitialPhysicalPlanWithStats\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x46\n\x15OptimizedPhysicalPlan\x18\x05 \x01(\x0b\x32%.datafusion.OptimizedPhysicalPlanTypeH\x00\x12\x35\n\x11\x46inalPhysicalPlan\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12>\n\x1a\x46inalPhysicalPlanWithStats\x18\n \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x42\x10\n\x0eplan_type_enum\"H\n\x0fStringifiedPlan\x12\'\n\tplan_type\x18\x01 \x01(\x0b\x32\x14.datafusion.PlanType\x12\x0c\n\x04plan\x18\x02 \x01(\t\"#\n\x12\x42\x61reTableReference\x12\r\n\x05table\x18\x01 \x01(\t\"6\n\x15PartialTableReference\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\r\n\x05table\x18\x02 \x01(\t\"D\n\x12\x46ullTableReference\x12\x0f\n\x07\x63\x61talog\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\xc3\x01\n\x13OwnedTableReference\x12.\n\x04\x62\x61re\x18\x01 \x01(\x0b\x32\x1e.datafusion.BareTableReferenceH\x00\x12\x34\n\x07partial\x18\x02 \x01(\x0b\x32!.datafusion.PartialTableReferenceH\x00\x12.\n\x04\x66ull\x18\x03 \x01(\x0b\x32\x1e.datafusion.FullTableReferenceH\x00\x42\x16\n\x14table_reference_enum\"\x92\x0c\n\x10PhysicalPlanNode\x12\x37\n\x0cparquet_scan\x18\x01 \x01(\x0b\x32\x1f.datafusion.ParquetScanExecNodeH\x00\x12/\n\x08\x63sv_scan\x18\x02 \x01(\x0b\x32\x1b.datafusion.CsvScanExecNodeH\x00\x12*\n\x05\x65mpty\x18\x03 \x01(\x0b\x32\x19.datafusion.EmptyExecNodeH\x00\x12\x34\n\nprojection\x18\x04 \x01(\x0b\x32\x1e.datafusion.ProjectionExecNodeH\x00\x12\x37\n\x0cglobal_limit\x18\x06 \x01(\x0b\x32\x1f.datafusion.GlobalLimitExecNodeH\x00\x12\x35\n\x0blocal_limit\x18\x07 \x01(\x0b\x32\x1e.datafusion.LocalLimitExecNodeH\x00\x12\x32\n\taggregate\x18\x08 \x01(\x0b\x32\x1d.datafusion.AggregateExecNodeH\x00\x12\x31\n\thash_join\x18\t \x01(\x0b\x32\x1c.datafusion.HashJoinExecNodeH\x00\x12(\n\x04sort\x18\n \x01(\x0b\x32\x18.datafusion.SortExecNodeH\x00\x12?\n\x10\x63oalesce_batches\x18\x0b \x01(\x0b\x32#.datafusion.CoalesceBatchesExecNodeH\x00\x12,\n\x06\x66ilter\x18\x0c \x01(\x0b\x32\x1a.datafusion.FilterExecNodeH\x00\x12\x37\n\x05merge\x18\r \x01(\x0b\x32&.datafusion.CoalescePartitionsExecNodeH\x00\x12\x36\n\x0brepartition\x18\x0e \x01(\x0b\x32\x1f.datafusion.RepartitionExecNodeH\x00\x12/\n\x06window\x18\x0f \x01(\x0b\x32\x1d.datafusion.WindowAggExecNodeH\x00\x12\x33\n\ncross_join\x18\x10 \x01(\x0b\x32\x1d.datafusion.CrossJoinExecNodeH\x00\x12\x31\n\tavro_scan\x18\x11 \x01(\x0b\x32\x1c.datafusion.AvroScanExecNodeH\x00\x12\x36\n\textension\x18\x12 \x01(\x0b\x32!.datafusion.PhysicalExtensionNodeH\x00\x12*\n\x05union\x18\x13 \x01(\x0b\x32\x19.datafusion.UnionExecNodeH\x00\x12.\n\x07\x65xplain\x18\x14 \x01(\x0b\x32\x1b.datafusion.ExplainExecNodeH\x00\x12H\n\x15sort_preserving_merge\x18\x15 \x01(\x0b\x32\'.datafusion.SortPreservingMergeExecNodeH\x00\x12>\n\x10nested_loop_join\x18\x16 \x01(\x0b\x32\".datafusion.NestedLoopJoinExecNodeH\x00\x12.\n\x07\x61nalyze\x18\x17 \x01(\x0b\x32\x1b.datafusion.AnalyzeExecNodeH\x00\x12\x31\n\tjson_sink\x18\x18 \x01(\x0b\x32\x1c.datafusion.JsonSinkExecNodeH\x00\x12\x44\n\x13symmetric_hash_join\x18\x19 \x01(\x0b\x32%.datafusion.SymmetricHashJoinExecNodeH\x00\x12\x34\n\ninterleave\x18\x1a \x01(\x0b\x32\x1e.datafusion.InterleaveExecNodeH\x00\x12=\n\x0fplaceholder_row\x18\x1b \x01(\x0b\x32\".datafusion.PlaceholderRowExecNodeH\x00\x12/\n\x08\x63sv_sink\x18\x1c \x01(\x0b\x32\x1b.datafusion.CsvSinkExecNodeH\x00\x12\x37\n\x0cparquet_sink\x18\x1d \x01(\x0b\x32\x1f.datafusion.ParquetSinkExecNodeH\x00\x42\x12\n\x10PhysicalPlanType\"J\n\x0fPartitionColumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"\x85\x02\n\x15\x46ileTypeWriterOptions\x12\x35\n\x0cjson_options\x18\x01 \x01(\x0b\x32\x1d.datafusion.JsonWriterOptionsH\x00\x12;\n\x0fparquet_options\x18\x02 \x01(\x0b\x32 .datafusion.ParquetWriterOptionsH\x00\x12\x33\n\x0b\x63sv_options\x18\x03 \x01(\x0b\x32\x1c.datafusion.CsvWriterOptionsH\x00\x12\x37\n\rarrow_options\x18\x04 \x01(\x0b\x32\x1e.datafusion.ArrowWriterOptionsH\x00\x42\n\n\x08\x46ileType\"L\n\x11JsonWriterOptions\x12\x37\n\x0b\x63ompression\x18\x01 \x01(\x0e\x32\".datafusion.CompressionTypeVariant\"O\n\x14ParquetWriterOptions\x12\x37\n\x11writer_properties\x18\x01 \x01(\x0b\x32\x1c.datafusion.WriterProperties\"\xe3\x01\n\x10\x43svWriterOptions\x12\x37\n\x0b\x63ompression\x18\x01 \x01(\x0e\x32\".datafusion.CompressionTypeVariant\x12\x11\n\tdelimiter\x18\x02 \x01(\t\x12\x12\n\nhas_header\x18\x03 \x01(\x08\x12\x13\n\x0b\x64\x61te_format\x18\x04 \x01(\t\x12\x17\n\x0f\x64\x61tetime_format\x18\x05 \x01(\t\x12\x18\n\x10timestamp_format\x18\x06 \x01(\t\x12\x13\n\x0btime_format\x18\x07 \x01(\t\x12\x12\n\nnull_value\x18\x08 \x01(\t\"\x14\n\x12\x41rrowWriterOptions\"\xd9\x01\n\x10WriterProperties\x12\x1c\n\x14\x64\x61ta_page_size_limit\x18\x01 \x01(\x04\x12\"\n\x1a\x64ictionary_page_size_limit\x18\x02 \x01(\x04\x12!\n\x19\x64\x61ta_page_row_count_limit\x18\x03 \x01(\x04\x12\x18\n\x10write_batch_size\x18\x04 \x01(\x04\x12\x1a\n\x12max_row_group_size\x18\x05 \x01(\x04\x12\x16\n\x0ewriter_version\x18\x06 \x01(\t\x12\x12\n\ncreated_by\x18\x07 \x01(\t\"\xb5\x02\n\x0e\x46ileSinkConfig\x12\x18\n\x10object_store_url\x18\x01 \x01(\t\x12\x30\n\x0b\x66ile_groups\x18\x02 \x03(\x0b\x32\x1b.datafusion.PartitionedFile\x12\x13\n\x0btable_paths\x18\x03 \x03(\t\x12)\n\routput_schema\x18\x04 \x01(\x0b\x32\x12.datafusion.Schema\x12\x39\n\x14table_partition_cols\x18\x05 \x03(\x0b\x32\x1b.datafusion.PartitionColumn\x12\x11\n\toverwrite\x18\x08 \x01(\x08\x12\x43\n\x18\x66ile_type_writer_options\x18\t \x01(\x0b\x32!.datafusion.FileTypeWriterOptionsJ\x04\x08\x06\x10\x07\"6\n\x08JsonSink\x12*\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x1a.datafusion.FileSinkConfig\"\xcc\x01\n\x10JsonSinkExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\"\n\x04sink\x18\x02 \x01(\x0b\x32\x14.datafusion.JsonSink\x12\'\n\x0bsink_schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12>\n\nsort_order\x18\x04 \x01(\x0b\x32*.datafusion.PhysicalSortExprNodeCollection\"5\n\x07\x43svSink\x12*\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x1a.datafusion.FileSinkConfig\"\xca\x01\n\x0f\x43svSinkExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12!\n\x04sink\x18\x02 \x01(\x0b\x32\x13.datafusion.CsvSink\x12\'\n\x0bsink_schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12>\n\nsort_order\x18\x04 \x01(\x0b\x32*.datafusion.PhysicalSortExprNodeCollection\"9\n\x0bParquetSink\x12*\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x1a.datafusion.FileSinkConfig\"\xd2\x01\n\x13ParquetSinkExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12%\n\x04sink\x18\x02 \x01(\x0b\x32\x17.datafusion.ParquetSink\x12\'\n\x0bsink_schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12>\n\nsort_order\x18\x04 \x01(\x0b\x32*.datafusion.PhysicalSortExprNodeCollection\"S\n\x15PhysicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12,\n\x06inputs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"\x80\x08\n\x10PhysicalExprNode\x12,\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x1a.datafusion.PhysicalColumnH\x00\x12*\n\x07literal\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x39\n\x0b\x62inary_expr\x18\x03 \x01(\x0b\x32\".datafusion.PhysicalBinaryExprNodeH\x00\x12?\n\x0e\x61ggregate_expr\x18\x04 \x01(\x0b\x32%.datafusion.PhysicalAggregateExprNodeH\x00\x12\x32\n\x0cis_null_expr\x18\x05 \x01(\x0b\x32\x1a.datafusion.PhysicalIsNullH\x00\x12\x39\n\x10is_not_null_expr\x18\x06 \x01(\x0b\x32\x1d.datafusion.PhysicalIsNotNullH\x00\x12+\n\x08not_expr\x18\x07 \x01(\x0b\x32\x17.datafusion.PhysicalNotH\x00\x12-\n\x05\x63\x61se_\x18\x08 \x01(\x0b\x32\x1c.datafusion.PhysicalCaseNodeH\x00\x12,\n\x04\x63\x61st\x18\t \x01(\x0b\x32\x1c.datafusion.PhysicalCastNodeH\x00\x12\x30\n\x04sort\x18\n \x01(\x0b\x32 .datafusion.PhysicalSortExprNodeH\x00\x12\x34\n\x08negative\x18\x0b \x01(\x0b\x32 .datafusion.PhysicalNegativeNodeH\x00\x12\x31\n\x07in_list\x18\x0c \x01(\x0b\x32\x1e.datafusion.PhysicalInListNodeH\x00\x12\x41\n\x0fscalar_function\x18\r \x01(\x0b\x32&.datafusion.PhysicalScalarFunctionNodeH\x00\x12\x33\n\x08try_cast\x18\x0e \x01(\x0b\x32\x1f.datafusion.PhysicalTryCastNodeH\x00\x12\x39\n\x0bwindow_expr\x18\x0f \x01(\x0b\x32\".datafusion.PhysicalWindowExprNodeH\x00\x12\x37\n\nscalar_udf\x18\x10 \x01(\x0b\x32!.datafusion.PhysicalScalarUdfNodeH\x00\x12\x35\n\tlike_expr\x18\x12 \x01(\x0b\x32 .datafusion.PhysicalLikeExprNodeH\x00\x12M\n\x16get_indexed_field_expr\x18\x13 \x01(\x0b\x32+.datafusion.PhysicalGetIndexedFieldExprNodeH\x00\x42\n\n\x08\x45xprTypeJ\x04\x08\x11\x10\x12\"}\n\x15PhysicalScalarUdfNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12*\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"\x84\x02\n\x19PhysicalAggregateExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12$\n\x1auser_defined_aggr_function\x18\x04 \x01(\tH\x00\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x36\n\x0cordering_req\x18\x05 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x42\x13\n\x11\x41ggregateFunction\"\xf4\x02\n\x16PhysicalWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12*\n\x04\x61rgs\x18\x04 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x32\n\x0cpartition_by\x18\x05 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x32\n\x08order_by\x18\x06 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12-\n\x0cwindow_frame\x18\x07 \x01(\x0b\x32\x17.datafusion.WindowFrame\x12\x0c\n\x04name\x18\x08 \x01(\tB\x11\n\x0fwindow_function\"<\n\x0ePhysicalIsNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"?\n\x11PhysicalIsNotNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"9\n\x0bPhysicalNot\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"N\n\x11PhysicalAliasNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"v\n\x16PhysicalBinaryExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x80\x01\n PhysicalDateTimeIntervalExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x9c\x01\n\x14PhysicalLikeExprNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12\x18\n\x10\x63\x61se_insensitive\x18\x02 \x01(\x08\x12*\n\x04\x65xpr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12-\n\x07pattern\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"d\n\x14PhysicalSortExprNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"t\n\x10PhysicalWhenThen\x12/\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"}\n\x12PhysicalInListNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x04list\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"\xa5\x01\n\x10PhysicalCaseNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x34\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalWhenThen\x12/\n\telse_expr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\xab\x01\n\x1aPhysicalScalarFunctionNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\'\n\x03\x66un\x18\x02 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12*\n\x04\x61rgs\x18\x03 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"l\n\x13PhysicalTryCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"i\n\x10PhysicalCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"B\n\x14PhysicalNegativeNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\x8d\x01\n\x0e\x46ilterExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\"\n\x1a\x64\x65\x66\x61ult_filter_selectivity\x18\x03 \x01(\r\"7\n\tFileGroup\x12*\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x1b.datafusion.PartitionedFile\"\x1a\n\tScanLimit\x12\r\n\x05limit\x18\x01 \x01(\r\"d\n\x1ePhysicalSortExprNodeCollection\x12\x42\n\x18physical_sort_expr_nodes\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\xcb\x02\n\x10\x46ileScanExecConf\x12*\n\x0b\x66ile_groups\x18\x01 \x03(\x0b\x32\x15.datafusion.FileGroup\x12\"\n\x06schema\x18\x02 \x01(\x0b\x32\x12.datafusion.Schema\x12\x12\n\nprojection\x18\x04 \x03(\r\x12$\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.ScanLimit\x12*\n\nstatistics\x18\x06 \x01(\x0b\x32\x16.datafusion.Statistics\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x18\n\x10object_store_url\x18\x08 \x01(\t\x12\x43\n\x0foutput_ordering\x18\t \x03(\x0b\x32*.datafusion.PhysicalSortExprNodeCollectionJ\x04\x08\n\x10\x0b\"}\n\x13ParquetScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12/\n\tpredicate\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNodeJ\x04\x08\x02\x10\x03\"\x9d\x01\n\x0f\x43svScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12\x12\n\nhas_header\x18\x02 \x01(\x08\x12\x11\n\tdelimiter\x18\x03 \x01(\t\x12\r\n\x05quote\x18\x04 \x01(\t\x12\x10\n\x06\x65scape\x18\x05 \x01(\tH\x00\x42\x11\n\x0foptional_escape\"C\n\x10\x41vroScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\"\xa9\x02\n\x10HashJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x1e\n\x02on\x18\x03 \x03(\x0b\x32\x12.datafusion.JoinOn\x12\'\n\tjoin_type\x18\x04 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x31\n\x0epartition_mode\x18\x06 \x01(\x0e\x32\x19.datafusion.PartitionMode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12&\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x16.datafusion.JoinFilter\"\xaf\x03\n\x19SymmetricHashJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x1e\n\x02on\x18\x03 \x03(\x0b\x32\x12.datafusion.JoinOn\x12\'\n\tjoin_type\x18\x04 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x37\n\x0epartition_mode\x18\x06 \x01(\x0e\x32\x1f.datafusion.StreamPartitionMode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12&\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x16.datafusion.JoinFilter\x12\x39\n\x0fleft_sort_exprs\x18\t \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12:\n\x10right_sort_exprs\x18\n \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"B\n\x12InterleaveExecNode\x12,\n\x06inputs\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"=\n\rUnionExecNode\x12,\n\x06inputs\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"~\n\x0f\x45xplainExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\x12\x36\n\x11stringified_plans\x18\x02 \x03(\x0b\x32\x1b.datafusion.StringifiedPlan\x12\x0f\n\x07verbose\x18\x03 \x01(\x08\"\x8c\x01\n\x0f\x41nalyzeExecNode\x12\x0f\n\x07verbose\x18\x01 \x01(\x08\x12\x17\n\x0fshow_statistics\x18\x02 \x01(\x08\x12+\n\x05input\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\"\n\x06schema\x18\x04 \x01(\x0b\x32\x12.datafusion.Schema\"l\n\x11\x43rossJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"-\n\x0ePhysicalColumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\r\"a\n\x06JoinOn\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"3\n\rEmptyExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\"<\n\x16PlaceholderRowExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\"\x80\x01\n\x12ProjectionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x11\n\texpr_name\x18\x03 \x03(\t\"0\n\x1dPartiallySortedInputOrderMode\x12\x0f\n\x07\x63olumns\x18\x06 \x03(\x04\"\xe2\x02\n\x11WindowAggExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x37\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\".datafusion.PhysicalWindowExprNode\x12\x34\n\x0epartition_keys\x18\x05 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x06linear\x18\x07 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x45\n\x10partially_sorted\x18\x08 \x01(\x0b\x32).datafusion.PartiallySortedInputOrderModeH\x00\x12*\n\x06sorted\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x42\x12\n\x10input_order_mode\"9\n\x0bMaybeFilter\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"M\n\x16MaybePhysicalSortExprs\x12\x33\n\tsort_expr\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\x96\x03\n\x11\x41ggregateExecNode\x12\x30\n\ngroup_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\taggr_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x04mode\x18\x03 \x01(\x0e\x32\x19.datafusion.AggregateMode\x12+\n\x05input\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x17\n\x0fgroup_expr_name\x18\x05 \x03(\t\x12\x16\n\x0e\x61ggr_expr_name\x18\x06 \x03(\t\x12(\n\x0cinput_schema\x18\x07 \x01(\x0b\x32\x12.datafusion.Schema\x12/\n\tnull_expr\x18\x08 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0e\n\x06groups\x18\t \x03(\x08\x12,\n\x0b\x66ilter_expr\x18\n \x03(\x0b\x32\x17.datafusion.MaybeFilter\"_\n\x13GlobalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\r\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"P\n\x12LocalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\r\n\x05\x66\x65tch\x18\x02 \x01(\r\"\x95\x01\n\x0cSortExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\x12\x1d\n\x15preserve_partitioning\x18\x04 \x01(\x08\"\x85\x01\n\x1bSortPreservingMergeExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\xc2\x01\n\x16NestedLoopJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12&\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x16.datafusion.JoinFilter\"a\n\x17\x43oalesceBatchesExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x19\n\x11target_batch_size\x18\x02 \x01(\r\"I\n\x1a\x43oalescePartitionsExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"c\n\x17PhysicalHashRepartition\x12/\n\thash_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\"\xb5\x01\n\x13RepartitionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12\x33\n\x04hash\x18\x03 \x01(\x0b\x32#.datafusion.PhysicalHashRepartitionH\x00\x12\x11\n\x07unknown\x18\x04 \x01(\x04H\x00\x42\x12\n\x10partition_method\"\x93\x01\n\nJoinFilter\x12\x30\n\nexpression\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\x0e\x63olumn_indices\x18\x02 \x03(\x0b\x32\x17.datafusion.ColumnIndex\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\"@\n\x0b\x43olumnIndex\x12\r\n\x05index\x18\x01 \x01(\r\x12\"\n\x04side\x18\x02 \x01(\x0e\x32\x14.datafusion.JoinSide\"\xa0\x01\n\x0fPartitionedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\x12\x18\n\x10last_modified_ns\x18\x03 \x01(\x04\x12\x31\n\x10partition_values\x18\x04 \x03(\x0b\x32\x17.datafusion.ScalarValue\x12$\n\x05range\x18\x05 \x01(\x0b\x32\x15.datafusion.FileRange\"\'\n\tFileRange\x12\r\n\x05start\x18\x01 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x03\"y\n\x0ePartitionStats\x12\x10\n\x08num_rows\x18\x01 \x01(\x03\x12\x13\n\x0bnum_batches\x18\x02 \x01(\x03\x12\x11\n\tnum_bytes\x18\x03 \x01(\x03\x12-\n\x0c\x63olumn_stats\x18\x04 \x03(\x0b\x32\x17.datafusion.ColumnStats\"d\n\tPrecision\x12\x31\n\x0eprecision_info\x18\x01 \x01(\x0e\x32\x19.datafusion.PrecisionInfo\x12$\n\x03val\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"\x94\x01\n\nStatistics\x12\'\n\x08num_rows\x18\x01 \x01(\x0b\x32\x15.datafusion.Precision\x12.\n\x0ftotal_byte_size\x18\x02 \x01(\x0b\x32\x15.datafusion.Precision\x12-\n\x0c\x63olumn_stats\x18\x03 \x03(\x0b\x32\x17.datafusion.ColumnStats\"\xbb\x01\n\x0b\x43olumnStats\x12(\n\tmin_value\x18\x01 \x01(\x0b\x32\x15.datafusion.Precision\x12(\n\tmax_value\x18\x02 \x01(\x0b\x32\x15.datafusion.Precision\x12)\n\nnull_count\x18\x03 \x01(\x0b\x32\x15.datafusion.Precision\x12-\n\x0e\x64istinct_count\x18\x04 \x01(\x0b\x32\x15.datafusion.Precision\"=\n\x14NamedStructFieldExpr\x12%\n\x04name\x18\x01 \x01(\x0b\x32\x17.datafusion.ScalarValue\":\n\rListIndexExpr\x12)\n\x03key\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\x96\x01\n\rListRangeExpr\x12+\n\x05start\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x04stop\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12,\n\x06stride\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\x86\x02\n\x1fPhysicalGetIndexedFieldExprNode\x12)\n\x03\x61rg\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x43\n\x17named_struct_field_expr\x18\x02 \x01(\x0b\x32 .datafusion.NamedStructFieldExprH\x00\x12\x34\n\x0flist_index_expr\x18\x03 \x01(\x0b\x32\x19.datafusion.ListIndexExprH\x00\x12\x34\n\x0flist_range_expr\x18\x04 \x01(\x0b\x32\x19.datafusion.ListRangeExprH\x00\x42\x07\n\x05\x66ield*n\n\x08JoinType\x12\t\n\x05INNER\x10\x00\x12\x08\n\x04LEFT\x10\x01\x12\t\n\x05RIGHT\x10\x02\x12\x08\n\x04\x46ULL\x10\x03\x12\x0c\n\x08LEFTSEMI\x10\x04\x12\x0c\n\x08LEFTANTI\x10\x05\x12\r\n\tRIGHTSEMI\x10\x06\x12\r\n\tRIGHTANTI\x10\x07*#\n\x0eJoinConstraint\x12\x06\n\x02ON\x10\x00\x12\t\n\x05USING\x10\x01*\xc0\x0e\n\x0eScalarFunction\x12\x07\n\x03\x41\x62s\x10\x00\x12\x08\n\x04\x41\x63os\x10\x01\x12\x08\n\x04\x41sin\x10\x02\x12\x08\n\x04\x41tan\x10\x03\x12\t\n\x05\x41scii\x10\x04\x12\x08\n\x04\x43\x65il\x10\x05\x12\x07\n\x03\x43os\x10\x06\x12\n\n\x06\x44igest\x10\x07\x12\x07\n\x03\x45xp\x10\x08\x12\t\n\x05\x46loor\x10\t\x12\x06\n\x02Ln\x10\n\x12\x07\n\x03Log\x10\x0b\x12\t\n\x05Log10\x10\x0c\x12\x08\n\x04Log2\x10\r\x12\t\n\x05Round\x10\x0e\x12\n\n\x06Signum\x10\x0f\x12\x07\n\x03Sin\x10\x10\x12\x08\n\x04Sqrt\x10\x11\x12\x07\n\x03Tan\x10\x12\x12\t\n\x05Trunc\x10\x13\x12\t\n\x05\x41rray\x10\x14\x12\x0f\n\x0bRegexpMatch\x10\x15\x12\r\n\tBitLength\x10\x16\x12\t\n\x05\x42trim\x10\x17\x12\x13\n\x0f\x43haracterLength\x10\x18\x12\x07\n\x03\x43hr\x10\x19\x12\n\n\x06\x43oncat\x10\x1a\x12\x17\n\x13\x43oncatWithSeparator\x10\x1b\x12\x0c\n\x08\x44\x61tePart\x10\x1c\x12\r\n\tDateTrunc\x10\x1d\x12\x0b\n\x07InitCap\x10\x1e\x12\x08\n\x04Left\x10\x1f\x12\x08\n\x04Lpad\x10 \x12\t\n\x05Lower\x10!\x12\t\n\x05Ltrim\x10\"\x12\x07\n\x03MD5\x10#\x12\x0f\n\x0bOctetLength\x10%\x12\n\n\x06Random\x10&\x12\x11\n\rRegexpReplace\x10\'\x12\n\n\x06Repeat\x10(\x12\x0b\n\x07Replace\x10)\x12\x0b\n\x07Reverse\x10*\x12\t\n\x05Right\x10+\x12\x08\n\x04Rpad\x10,\x12\t\n\x05Rtrim\x10-\x12\n\n\x06SHA224\x10.\x12\n\n\x06SHA256\x10/\x12\n\n\x06SHA384\x10\x30\x12\n\n\x06SHA512\x10\x31\x12\r\n\tSplitPart\x10\x32\x12\x0e\n\nStartsWith\x10\x33\x12\n\n\x06Strpos\x10\x34\x12\n\n\x06Substr\x10\x35\x12\t\n\x05ToHex\x10\x36\x12\x0f\n\x0bToTimestamp\x10\x37\x12\x15\n\x11ToTimestampMillis\x10\x38\x12\x15\n\x11ToTimestampMicros\x10\x39\x12\x16\n\x12ToTimestampSeconds\x10:\x12\x07\n\x03Now\x10;\x12\r\n\tTranslate\x10<\x12\x08\n\x04Trim\x10=\x12\t\n\x05Upper\x10>\x12\x0c\n\x08\x43oalesce\x10?\x12\t\n\x05Power\x10@\x12\r\n\tStructFun\x10\x41\x12\x10\n\x0c\x46romUnixtime\x10\x42\x12\t\n\x05\x41tan2\x10\x43\x12\x0b\n\x07\x44\x61teBin\x10\x44\x12\x0f\n\x0b\x41rrowTypeof\x10\x45\x12\x0f\n\x0b\x43urrentDate\x10\x46\x12\x0f\n\x0b\x43urrentTime\x10G\x12\x08\n\x04Uuid\x10H\x12\x08\n\x04\x43\x62rt\x10I\x12\t\n\x05\x41\x63osh\x10J\x12\t\n\x05\x41sinh\x10K\x12\t\n\x05\x41tanh\x10L\x12\x08\n\x04Sinh\x10M\x12\x08\n\x04\x43osh\x10N\x12\x08\n\x04Tanh\x10O\x12\x06\n\x02Pi\x10P\x12\x0b\n\x07\x44\x65grees\x10Q\x12\x0b\n\x07Radians\x10R\x12\r\n\tFactorial\x10S\x12\x07\n\x03Lcm\x10T\x12\x07\n\x03Gcd\x10U\x12\x0f\n\x0b\x41rrayAppend\x10V\x12\x0f\n\x0b\x41rrayConcat\x10W\x12\r\n\tArrayDims\x10X\x12\x0f\n\x0b\x41rrayRepeat\x10Y\x12\x0f\n\x0b\x41rrayLength\x10Z\x12\x0e\n\nArrayNdims\x10[\x12\x11\n\rArrayPosition\x10\\\x12\x12\n\x0e\x41rrayPositions\x10]\x12\x10\n\x0c\x41rrayPrepend\x10^\x12\x0f\n\x0b\x41rrayRemove\x10_\x12\x10\n\x0c\x41rrayReplace\x10`\x12\x0f\n\x0b\x43\x61rdinality\x10\x62\x12\x10\n\x0c\x41rrayElement\x10\x63\x12\x0e\n\nArraySlice\x10\x64\x12\x07\n\x03\x43ot\x10g\x12\x0c\n\x08\x41rrayHas\x10h\x12\x0f\n\x0b\x41rrayHasAny\x10i\x12\x0f\n\x0b\x41rrayHasAll\x10j\x12\x10\n\x0c\x41rrayRemoveN\x10k\x12\x11\n\rArrayReplaceN\x10l\x12\x12\n\x0e\x41rrayRemoveAll\x10m\x12\x13\n\x0f\x41rrayReplaceAll\x10n\x12\t\n\x05Nanvl\x10o\x12\x0b\n\x07\x46latten\x10p\x12\n\n\x06Iszero\x10r\x12\x0e\n\nArrayEmpty\x10s\x12\x10\n\x0c\x41rrayPopBack\x10t\x12\x11\n\rStringToArray\x10u\x12\x14\n\x10ToTimestampNanos\x10v\x12\x12\n\x0e\x41rrayIntersect\x10w\x12\x0e\n\nArrayUnion\x10x\x12\x0b\n\x07OverLay\x10y\x12\t\n\x05Range\x10z\x12\x0f\n\x0b\x41rrayExcept\x10{\x12\x11\n\rArrayPopFront\x10|\x12\x0f\n\x0bLevenshtein\x10}\x12\x0f\n\x0bSubstrIndex\x10~\x12\r\n\tFindInSet\x10\x7f\x12\x0e\n\tArraySort\x10\x80\x01\x12\x12\n\rArrayDistinct\x10\x81\x01\x12\x10\n\x0b\x41rrayResize\x10\x82\x01\x12\r\n\x08\x45ndsWith\x10\x83\x01\x12\n\n\x05InStr\x10\x84\x01\x12\r\n\x08MakeDate\x10\x85\x01\x12\x11\n\x0c\x41rrayReverse\x10\x86\x01\x12\x0f\n\nRegexpLike\x10\x87\x01\x12\x0b\n\x06ToChar\x10\x88\x01*\xde\x04\n\x11\x41ggregateFunction\x12\x07\n\x03MIN\x10\x00\x12\x07\n\x03MAX\x10\x01\x12\x07\n\x03SUM\x10\x02\x12\x07\n\x03\x41VG\x10\x03\x12\t\n\x05\x43OUNT\x10\x04\x12\x13\n\x0f\x41PPROX_DISTINCT\x10\x05\x12\r\n\tARRAY_AGG\x10\x06\x12\x0c\n\x08VARIANCE\x10\x07\x12\x10\n\x0cVARIANCE_POP\x10\x08\x12\x0e\n\nCOVARIANCE\x10\t\x12\x12\n\x0e\x43OVARIANCE_POP\x10\n\x12\n\n\x06STDDEV\x10\x0b\x12\x0e\n\nSTDDEV_POP\x10\x0c\x12\x0f\n\x0b\x43ORRELATION\x10\r\x12\x1a\n\x16\x41PPROX_PERCENTILE_CONT\x10\x0e\x12\x11\n\rAPPROX_MEDIAN\x10\x0f\x12&\n\"APPROX_PERCENTILE_CONT_WITH_WEIGHT\x10\x10\x12\x0c\n\x08GROUPING\x10\x11\x12\n\n\x06MEDIAN\x10\x12\x12\x0b\n\x07\x42IT_AND\x10\x13\x12\n\n\x06\x42IT_OR\x10\x14\x12\x0b\n\x07\x42IT_XOR\x10\x15\x12\x0c\n\x08\x42OOL_AND\x10\x16\x12\x0b\n\x07\x42OOL_OR\x10\x17\x12\x13\n\x0f\x46IRST_VALUE_AGG\x10\x18\x12\x12\n\x0eLAST_VALUE_AGG\x10\x19\x12\x0e\n\nREGR_SLOPE\x10\x1a\x12\x12\n\x0eREGR_INTERCEPT\x10\x1b\x12\x0e\n\nREGR_COUNT\x10\x1c\x12\x0b\n\x07REGR_R2\x10\x1d\x12\r\n\tREGR_AVGX\x10\x1e\x12\r\n\tREGR_AVGY\x10\x1f\x12\x0c\n\x08REGR_SXX\x10 \x12\x0c\n\x08REGR_SYY\x10!\x12\x0c\n\x08REGR_SXY\x10\"\x12\x0e\n\nSTRING_AGG\x10#\x12\x11\n\rNTH_VALUE_AGG\x10$*\xb0\x01\n\x15\x42uiltInWindowFunction\x12\x0e\n\nROW_NUMBER\x10\x00\x12\x08\n\x04RANK\x10\x01\x12\x0e\n\nDENSE_RANK\x10\x02\x12\x10\n\x0cPERCENT_RANK\x10\x03\x12\r\n\tCUME_DIST\x10\x04\x12\t\n\x05NTILE\x10\x05\x12\x07\n\x03LAG\x10\x06\x12\x08\n\x04LEAD\x10\x07\x12\x0f\n\x0b\x46IRST_VALUE\x10\x08\x12\x0e\n\nLAST_VALUE\x10\t\x12\r\n\tNTH_VALUE\x10\n*3\n\x10WindowFrameUnits\x12\x08\n\x04ROWS\x10\x00\x12\t\n\x05RANGE\x10\x01\x12\n\n\x06GROUPS\x10\x02*E\n\x14WindowFrameBoundType\x12\x0f\n\x0b\x43URRENT_ROW\x10\x00\x12\r\n\tPRECEDING\x10\x01\x12\r\n\tFOLLOWING\x10\x02*(\n\x08\x44\x61teUnit\x12\x07\n\x03\x44\x61y\x10\x00\x12\x13\n\x0f\x44\x61teMillisecond\x10\x01*H\n\x08TimeUnit\x12\n\n\x06Second\x10\x00\x12\x0f\n\x0bMillisecond\x10\x01\x12\x0f\n\x0bMicrosecond\x10\x02\x12\x0e\n\nNanosecond\x10\x03*<\n\x0cIntervalUnit\x12\r\n\tYearMonth\x10\x00\x12\x0b\n\x07\x44\x61yTime\x10\x01\x12\x10\n\x0cMonthDayNano\x10\x02*\"\n\tUnionMode\x12\n\n\x06sparse\x10\x00\x12\t\n\x05\x64\x65nse\x10\x01*Q\n\x16\x43ompressionTypeVariant\x12\x08\n\x04GZIP\x10\x00\x12\t\n\x05\x42ZIP2\x10\x01\x12\x06\n\x02XZ\x10\x02\x12\x08\n\x04ZSTD\x10\x03\x12\x10\n\x0cUNCOMPRESSED\x10\x04*<\n\rPartitionMode\x12\x10\n\x0c\x43OLLECT_LEFT\x10\x00\x12\x0f\n\x0bPARTITIONED\x10\x01\x12\x08\n\x04\x41UTO\x10\x02*A\n\x13StreamPartitionMode\x12\x14\n\x10SINGLE_PARTITION\x10\x00\x12\x14\n\x10PARTITIONED_EXEC\x10\x01*b\n\rAggregateMode\x12\x0b\n\x07PARTIAL\x10\x00\x12\t\n\x05\x46INAL\x10\x01\x12\x15\n\x11\x46INAL_PARTITIONED\x10\x02\x12\n\n\x06SINGLE\x10\x03\x12\x16\n\x12SINGLE_PARTITIONED\x10\x04*)\n\x08JoinSide\x12\r\n\tLEFT_SIDE\x10\x00\x12\x0e\n\nRIGHT_SIDE\x10\x01*3\n\rPrecisionInfo\x12\t\n\x05\x45XACT\x10\x00\x12\x0b\n\x07INEXACT\x10\x01\x12\n\n\x06\x41\x42SENT\x10\x02\x42\x39\n$org.apache.arrow.datafusion.protobufB\x0f\x44\x61tafusionProtoP\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -26,38 +26,46 @@ _DFSCHEMA_METADATAENTRY._serialized_options = b'8\001' _CREATEEXTERNALTABLENODE_OPTIONSENTRY._options = None _CREATEEXTERNALTABLENODE_OPTIONSENTRY._serialized_options = b'8\001' + _CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY._options = None + _CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY._serialized_options = b'8\001' _SCHEMA_METADATAENTRY._options = None _SCHEMA_METADATAENTRY._serialized_options = b'8\001' _FIELD_METADATAENTRY._options = None _FIELD_METADATAENTRY._serialized_options = b'8\001' - _globals['_JOINTYPE']._serialized_start=26361 - _globals['_JOINTYPE']._serialized_end=26471 - _globals['_JOINCONSTRAINT']._serialized_start=26473 - _globals['_JOINCONSTRAINT']._serialized_end=26508 - _globals['_SCALARFUNCTION']._serialized_start=26511 - _globals['_SCALARFUNCTION']._serialized_end=27893 - _globals['_AGGREGATEFUNCTION']._serialized_start=27896 - _globals['_AGGREGATEFUNCTION']._serialized_end=28330 - _globals['_BUILTINWINDOWFUNCTION']._serialized_start=28333 - _globals['_BUILTINWINDOWFUNCTION']._serialized_end=28509 - _globals['_WINDOWFRAMEUNITS']._serialized_start=28511 - _globals['_WINDOWFRAMEUNITS']._serialized_end=28562 - _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_start=28564 - _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_end=28633 - _globals['_DATEUNIT']._serialized_start=28635 - _globals['_DATEUNIT']._serialized_end=28675 - _globals['_TIMEUNIT']._serialized_start=28677 - _globals['_TIMEUNIT']._serialized_end=28749 - _globals['_INTERVALUNIT']._serialized_start=28751 - _globals['_INTERVALUNIT']._serialized_end=28811 - _globals['_UNIONMODE']._serialized_start=28813 - _globals['_UNIONMODE']._serialized_end=28847 - _globals['_PARTITIONMODE']._serialized_start=28849 - _globals['_PARTITIONMODE']._serialized_end=28909 - _globals['_AGGREGATEMODE']._serialized_start=28911 - _globals['_AGGREGATEMODE']._serialized_end=29009 - _globals['_JOINSIDE']._serialized_start=29011 - _globals['_JOINSIDE']._serialized_end=29052 + _globals['_JOINTYPE']._serialized_start=32527 + _globals['_JOINTYPE']._serialized_end=32637 + _globals['_JOINCONSTRAINT']._serialized_start=32639 + _globals['_JOINCONSTRAINT']._serialized_end=32674 + _globals['_SCALARFUNCTION']._serialized_start=32677 + _globals['_SCALARFUNCTION']._serialized_end=34533 + _globals['_AGGREGATEFUNCTION']._serialized_start=34536 + _globals['_AGGREGATEFUNCTION']._serialized_end=35142 + _globals['_BUILTINWINDOWFUNCTION']._serialized_start=35145 + _globals['_BUILTINWINDOWFUNCTION']._serialized_end=35321 + _globals['_WINDOWFRAMEUNITS']._serialized_start=35323 + _globals['_WINDOWFRAMEUNITS']._serialized_end=35374 + _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_start=35376 + _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_end=35445 + _globals['_DATEUNIT']._serialized_start=35447 + _globals['_DATEUNIT']._serialized_end=35487 + _globals['_TIMEUNIT']._serialized_start=35489 + _globals['_TIMEUNIT']._serialized_end=35561 + _globals['_INTERVALUNIT']._serialized_start=35563 + _globals['_INTERVALUNIT']._serialized_end=35623 + _globals['_UNIONMODE']._serialized_start=35625 + _globals['_UNIONMODE']._serialized_end=35659 + _globals['_COMPRESSIONTYPEVARIANT']._serialized_start=35661 + _globals['_COMPRESSIONTYPEVARIANT']._serialized_end=35742 + _globals['_PARTITIONMODE']._serialized_start=35744 + _globals['_PARTITIONMODE']._serialized_end=35804 + _globals['_STREAMPARTITIONMODE']._serialized_start=35806 + _globals['_STREAMPARTITIONMODE']._serialized_end=35871 + _globals['_AGGREGATEMODE']._serialized_start=35873 + _globals['_AGGREGATEMODE']._serialized_end=35971 + _globals['_JOINSIDE']._serialized_start=35973 + _globals['_JOINSIDE']._serialized_end=36014 + _globals['_PRECISIONINFO']._serialized_start=36016 + _globals['_PRECISIONINFO']._serialized_end=36067 _globals['_COLUMNRELATION']._serialized_start=38 _globals['_COLUMNRELATION']._serialized_end=72 _globals['_COLUMN']._serialized_start=74 @@ -69,329 +77,403 @@ _globals['_DFSCHEMA_METADATAENTRY']._serialized_start=341 _globals['_DFSCHEMA_METADATAENTRY']._serialized_end=388 _globals['_LOGICALPLANNODE']._serialized_start=391 - _globals['_LOGICALPLANNODE']._serialized_end=1722 - _globals['_LOGICALEXTENSIONNODE']._serialized_start=1724 - _globals['_LOGICALEXTENSIONNODE']._serialized_end=1805 - _globals['_PROJECTIONCOLUMNS']._serialized_start=1807 - _globals['_PROJECTIONCOLUMNS']._serialized_end=1843 - _globals['_CSVFORMAT']._serialized_start=1845 - _globals['_CSVFORMAT']._serialized_end=1895 - _globals['_PARQUETFORMAT']._serialized_start=1897 - _globals['_PARQUETFORMAT']._serialized_end=1918 - _globals['_AVROFORMAT']._serialized_start=1920 - _globals['_AVROFORMAT']._serialized_end=1932 - _globals['_LOGICALEXPRNODECOLLECTION']._serialized_start=1934 - _globals['_LOGICALEXPRNODECOLLECTION']._serialized_end=2018 - _globals['_LISTINGTABLESCANNODE']._serialized_start=2021 - _globals['_LISTINGTABLESCANNODE']._serialized_end=2559 - _globals['_VIEWTABLESCANNODE']._serialized_start=2562 - _globals['_VIEWTABLESCANNODE']._serialized_end=2791 - _globals['_CUSTOMTABLESCANNODE']._serialized_start=2794 - _globals['_CUSTOMTABLESCANNODE']._serialized_end=3034 - _globals['_PROJECTIONNODE']._serialized_start=3037 - _globals['_PROJECTIONNODE']._serialized_end=3175 - _globals['_SELECTIONNODE']._serialized_start=3177 - _globals['_SELECTIONNODE']._serialized_end=3279 - _globals['_SORTNODE']._serialized_start=3281 - _globals['_SORTNODE']._serialized_end=3393 - _globals['_REPARTITIONNODE']._serialized_start=3396 - _globals['_REPARTITIONNODE']._serialized_end=3545 - _globals['_HASHREPARTITION']._serialized_start=3547 - _globals['_HASHREPARTITION']._serialized_end=3637 - _globals['_EMPTYRELATIONNODE']._serialized_start=3639 - _globals['_EMPTYRELATIONNODE']._serialized_end=3683 - _globals['_CREATEEXTERNALTABLENODE']._serialized_start=3686 - _globals['_CREATEEXTERNALTABLENODE']._serialized_end=4176 - _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_start=4124 - _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_end=4170 - _globals['_PREPARENODE']._serialized_start=4178 - _globals['_PREPARENODE']._serialized_end=4292 - _globals['_CREATECATALOGSCHEMANODE']._serialized_start=4294 - _globals['_CREATECATALOGSCHEMANODE']._serialized_end=4401 - _globals['_CREATECATALOGNODE']._serialized_start=4403 - _globals['_CREATECATALOGNODE']._serialized_end=4505 - _globals['_DROPVIEWNODE']._serialized_start=4507 - _globals['_DROPVIEWNODE']._serialized_end=4625 - _globals['_CREATEVIEWNODE']._serialized_start=4628 - _globals['_CREATEVIEWNODE']._serialized_end=4781 - _globals['_VALUESNODE']._serialized_start=4783 - _globals['_VALUESNODE']._serialized_end=4861 - _globals['_ANALYZENODE']._serialized_start=4863 - _globals['_ANALYZENODE']._serialized_end=4937 - _globals['_EXPLAINNODE']._serialized_start=4939 - _globals['_EXPLAINNODE']._serialized_end=5013 - _globals['_AGGREGATENODE']._serialized_start=5016 - _globals['_AGGREGATENODE']._serialized_end=5172 - _globals['_WINDOWNODE']._serialized_start=5174 - _globals['_WINDOWNODE']._serialized_end=5280 - _globals['_JOINNODE']._serialized_start=5283 - _globals['_JOINNODE']._serialized_end=5650 - _globals['_DISTINCTNODE']._serialized_start=5652 - _globals['_DISTINCTNODE']._serialized_end=5710 - _globals['_UNIONNODE']._serialized_start=5712 - _globals['_UNIONNODE']._serialized_end=5768 - _globals['_CROSSJOINNODE']._serialized_start=5770 - _globals['_CROSSJOINNODE']._serialized_end=5872 - _globals['_LIMITNODE']._serialized_start=5874 - _globals['_LIMITNODE']._serialized_end=5958 - _globals['_SELECTIONEXECNODE']._serialized_start=5960 - _globals['_SELECTIONEXECNODE']._serialized_end=6022 - _globals['_SUBQUERYALIASNODE']._serialized_start=6024 - _globals['_SUBQUERYALIASNODE']._serialized_end=6141 - _globals['_LOGICALEXPRNODE']._serialized_start=6144 - _globals['_LOGICALEXPRNODE']._serialized_end=7725 - _globals['_PLACEHOLDERNODE']._serialized_start=7727 - _globals['_PLACEHOLDERNODE']._serialized_end=7798 - _globals['_LOGICALEXPRLIST']._serialized_start=7800 - _globals['_LOGICALEXPRLIST']._serialized_end=7860 - _globals['_GROUPINGSETNODE']._serialized_start=7862 - _globals['_GROUPINGSETNODE']._serialized_end=7922 - _globals['_CUBENODE']._serialized_start=7924 - _globals['_CUBENODE']._serialized_end=7977 - _globals['_ROLLUPNODE']._serialized_start=7979 - _globals['_ROLLUPNODE']._serialized_end=8034 - _globals['_GETINDEXEDFIELD']._serialized_start=8036 - _globals['_GETINDEXEDFIELD']._serialized_end=8134 - _globals['_ISNULL']._serialized_start=8136 - _globals['_ISNULL']._serialized_end=8187 - _globals['_ISNOTNULL']._serialized_start=8189 - _globals['_ISNOTNULL']._serialized_end=8243 - _globals['_ISTRUE']._serialized_start=8245 - _globals['_ISTRUE']._serialized_end=8296 - _globals['_ISFALSE']._serialized_start=8298 - _globals['_ISFALSE']._serialized_end=8350 - _globals['_ISUNKNOWN']._serialized_start=8352 - _globals['_ISUNKNOWN']._serialized_end=8406 - _globals['_ISNOTTRUE']._serialized_start=8408 - _globals['_ISNOTTRUE']._serialized_end=8462 - _globals['_ISNOTFALSE']._serialized_start=8464 - _globals['_ISNOTFALSE']._serialized_end=8519 - _globals['_ISNOTUNKNOWN']._serialized_start=8521 - _globals['_ISNOTUNKNOWN']._serialized_end=8578 - _globals['_NOT']._serialized_start=8580 - _globals['_NOT']._serialized_end=8628 - _globals['_ALIASNODE']._serialized_start=8630 - _globals['_ALIASNODE']._serialized_end=8699 - _globals['_BINARYEXPRNODE']._serialized_start=8701 - _globals['_BINARYEXPRNODE']._serialized_end=8776 - _globals['_NEGATIVENODE']._serialized_start=8778 - _globals['_NEGATIVENODE']._serialized_end=8835 - _globals['_INLISTNODE']._serialized_start=8837 - _globals['_INLISTNODE']._serialized_end=8952 - _globals['_SCALARFUNCTIONNODE']._serialized_start=8954 - _globals['_SCALARFUNCTIONNODE']._serialized_end=9058 - _globals['_AGGREGATEEXPRNODE']._serialized_start=9061 - _globals['_AGGREGATEEXPRNODE']._serialized_end=9287 - _globals['_AGGREGATEUDFEXPRNODE']._serialized_start=9290 - _globals['_AGGREGATEUDFEXPRNODE']._serialized_end=9465 - _globals['_SCALARUDFEXPRNODE']._serialized_start=9467 - _globals['_SCALARUDFEXPRNODE']._serialized_end=9547 - _globals['_WINDOWEXPRNODE']._serialized_start=9550 - _globals['_WINDOWEXPRNODE']._serialized_end=9925 - _globals['_BETWEENNODE']._serialized_start=9928 - _globals['_BETWEENNODE']._serialized_end=10086 - _globals['_LIKENODE']._serialized_start=10089 - _globals['_LIKENODE']._serialized_end=10226 - _globals['_ILIKENODE']._serialized_start=10229 - _globals['_ILIKENODE']._serialized_end=10367 - _globals['_SIMILARTONODE']._serialized_start=10370 - _globals['_SIMILARTONODE']._serialized_end=10512 - _globals['_CASENODE']._serialized_start=10515 - _globals['_CASENODE']._serialized_end=10662 - _globals['_WHENTHEN']._serialized_start=10664 - _globals['_WHENTHEN']._serialized_end=10770 - _globals['_CASTNODE']._serialized_start=10772 - _globals['_CASTNODE']._serialized_end=10868 - _globals['_TRYCASTNODE']._serialized_start=10870 - _globals['_TRYCASTNODE']._serialized_end=10969 - _globals['_SORTEXPRNODE']._serialized_start=10971 - _globals['_SORTEXPRNODE']._serialized_end=11062 - _globals['_WINDOWFRAME']._serialized_start=11065 - _globals['_WINDOWFRAME']._serialized_end=11247 - _globals['_WINDOWFRAMEBOUND']._serialized_start=11250 - _globals['_WINDOWFRAMEBOUND']._serialized_end=11381 - _globals['_SCHEMA']._serialized_start=11384 - _globals['_SCHEMA']._serialized_end=11529 + _globals['_LOGICALPLANNODE']._serialized_end=1816 + _globals['_LOGICALEXTENSIONNODE']._serialized_start=1818 + _globals['_LOGICALEXTENSIONNODE']._serialized_end=1899 + _globals['_PROJECTIONCOLUMNS']._serialized_start=1901 + _globals['_PROJECTIONCOLUMNS']._serialized_end=1937 + _globals['_CSVFORMAT']._serialized_start=1939 + _globals['_CSVFORMAT']._serialized_end=2041 + _globals['_PARQUETFORMAT']._serialized_start=2043 + _globals['_PARQUETFORMAT']._serialized_end=2064 + _globals['_AVROFORMAT']._serialized_start=2066 + _globals['_AVROFORMAT']._serialized_end=2078 + _globals['_LOGICALEXPRNODECOLLECTION']._serialized_start=2080 + _globals['_LOGICALEXPRNODECOLLECTION']._serialized_end=2164 + _globals['_LISTINGTABLESCANNODE']._serialized_start=2167 + _globals['_LISTINGTABLESCANNODE']._serialized_end=2705 + _globals['_VIEWTABLESCANNODE']._serialized_start=2708 + _globals['_VIEWTABLESCANNODE']._serialized_end=2937 + _globals['_CUSTOMTABLESCANNODE']._serialized_start=2940 + _globals['_CUSTOMTABLESCANNODE']._serialized_end=3180 + _globals['_PROJECTIONNODE']._serialized_start=3183 + _globals['_PROJECTIONNODE']._serialized_end=3321 + _globals['_SELECTIONNODE']._serialized_start=3323 + _globals['_SELECTIONNODE']._serialized_end=3425 + _globals['_SORTNODE']._serialized_start=3427 + _globals['_SORTNODE']._serialized_end=3539 + _globals['_REPARTITIONNODE']._serialized_start=3542 + _globals['_REPARTITIONNODE']._serialized_end=3691 + _globals['_HASHREPARTITION']._serialized_start=3693 + _globals['_HASHREPARTITION']._serialized_end=3783 + _globals['_EMPTYRELATIONNODE']._serialized_start=3785 + _globals['_EMPTYRELATIONNODE']._serialized_end=3829 + _globals['_PRIMARYKEYCONSTRAINT']._serialized_start=3831 + _globals['_PRIMARYKEYCONSTRAINT']._serialized_end=3870 + _globals['_UNIQUECONSTRAINT']._serialized_start=3872 + _globals['_UNIQUECONSTRAINT']._serialized_end=3907 + _globals['_CONSTRAINT']._serialized_start=3910 + _globals['_CONSTRAINT']._serialized_end=4046 + _globals['_CONSTRAINTS']._serialized_start=4048 + _globals['_CONSTRAINTS']._serialized_end=4106 + _globals['_CREATEEXTERNALTABLENODE']._serialized_start=4109 + _globals['_CREATEEXTERNALTABLENODE']._serialized_end=4853 + _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_start=4711 + _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_end=4757 + _globals['_CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY']._serialized_start=4759 + _globals['_CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY']._serialized_end=4841 + _globals['_PREPARENODE']._serialized_start=4855 + _globals['_PREPARENODE']._serialized_end=4969 + _globals['_CREATECATALOGSCHEMANODE']._serialized_start=4971 + _globals['_CREATECATALOGSCHEMANODE']._serialized_end=5078 + _globals['_CREATECATALOGNODE']._serialized_start=5080 + _globals['_CREATECATALOGNODE']._serialized_end=5182 + _globals['_DROPVIEWNODE']._serialized_start=5184 + _globals['_DROPVIEWNODE']._serialized_end=5302 + _globals['_CREATEVIEWNODE']._serialized_start=5305 + _globals['_CREATEVIEWNODE']._serialized_end=5458 + _globals['_VALUESNODE']._serialized_start=5460 + _globals['_VALUESNODE']._serialized_end=5538 + _globals['_ANALYZENODE']._serialized_start=5540 + _globals['_ANALYZENODE']._serialized_end=5614 + _globals['_EXPLAINNODE']._serialized_start=5616 + _globals['_EXPLAINNODE']._serialized_end=5690 + _globals['_AGGREGATENODE']._serialized_start=5693 + _globals['_AGGREGATENODE']._serialized_end=5849 + _globals['_WINDOWNODE']._serialized_start=5851 + _globals['_WINDOWNODE']._serialized_end=5957 + _globals['_JOINNODE']._serialized_start=5960 + _globals['_JOINNODE']._serialized_end=6327 + _globals['_DISTINCTNODE']._serialized_start=6329 + _globals['_DISTINCTNODE']._serialized_end=6387 + _globals['_DISTINCTONNODE']._serialized_start=6390 + _globals['_DISTINCTONNODE']._serialized_end=6594 + _globals['_COPYTONODE']._serialized_start=6597 + _globals['_COPYTONODE']._serialized_end=6815 + _globals['_SQLOPTIONS']._serialized_start=6817 + _globals['_SQLOPTIONS']._serialized_end=6868 + _globals['_SQLOPTION']._serialized_start=6870 + _globals['_SQLOPTION']._serialized_end=6909 + _globals['_UNIONNODE']._serialized_start=6911 + _globals['_UNIONNODE']._serialized_end=6967 + _globals['_CROSSJOINNODE']._serialized_start=6969 + _globals['_CROSSJOINNODE']._serialized_end=7071 + _globals['_LIMITNODE']._serialized_start=7073 + _globals['_LIMITNODE']._serialized_end=7157 + _globals['_SELECTIONEXECNODE']._serialized_start=7159 + _globals['_SELECTIONEXECNODE']._serialized_end=7221 + _globals['_SUBQUERYALIASNODE']._serialized_start=7223 + _globals['_SUBQUERYALIASNODE']._serialized_end=7340 + _globals['_LOGICALEXPRNODE']._serialized_start=7343 + _globals['_LOGICALEXPRNODE']._serialized_end=8984 + _globals['_WILDCARD']._serialized_start=8986 + _globals['_WILDCARD']._serialized_end=9015 + _globals['_PLACEHOLDERNODE']._serialized_start=9017 + _globals['_PLACEHOLDERNODE']._serialized_end=9088 + _globals['_LOGICALEXPRLIST']._serialized_start=9090 + _globals['_LOGICALEXPRLIST']._serialized_end=9150 + _globals['_GROUPINGSETNODE']._serialized_start=9152 + _globals['_GROUPINGSETNODE']._serialized_end=9212 + _globals['_CUBENODE']._serialized_start=9214 + _globals['_CUBENODE']._serialized_end=9267 + _globals['_ROLLUPNODE']._serialized_start=9269 + _globals['_ROLLUPNODE']._serialized_end=9324 + _globals['_NAMEDSTRUCTFIELD']._serialized_start=9326 + _globals['_NAMEDSTRUCTFIELD']._serialized_end=9383 + _globals['_LISTINDEX']._serialized_start=9385 + _globals['_LISTINDEX']._serialized_end=9438 + _globals['_LISTRANGE']._serialized_start=9441 + _globals['_LISTRANGE']._serialized_end=9584 + _globals['_GETINDEXEDFIELD']._serialized_start=9587 + _globals['_GETINDEXEDFIELD']._serialized_end=9806 + _globals['_ISNULL']._serialized_start=9808 + _globals['_ISNULL']._serialized_end=9859 + _globals['_ISNOTNULL']._serialized_start=9861 + _globals['_ISNOTNULL']._serialized_end=9915 + _globals['_ISTRUE']._serialized_start=9917 + _globals['_ISTRUE']._serialized_end=9968 + _globals['_ISFALSE']._serialized_start=9970 + _globals['_ISFALSE']._serialized_end=10022 + _globals['_ISUNKNOWN']._serialized_start=10024 + _globals['_ISUNKNOWN']._serialized_end=10078 + _globals['_ISNOTTRUE']._serialized_start=10080 + _globals['_ISNOTTRUE']._serialized_end=10134 + _globals['_ISNOTFALSE']._serialized_start=10136 + _globals['_ISNOTFALSE']._serialized_end=10191 + _globals['_ISNOTUNKNOWN']._serialized_start=10193 + _globals['_ISNOTUNKNOWN']._serialized_end=10250 + _globals['_NOT']._serialized_start=10252 + _globals['_NOT']._serialized_end=10300 + _globals['_ALIASNODE']._serialized_start=10302 + _globals['_ALIASNODE']._serialized_end=10422 + _globals['_BINARYEXPRNODE']._serialized_start=10424 + _globals['_BINARYEXPRNODE']._serialized_end=10499 + _globals['_NEGATIVENODE']._serialized_start=10501 + _globals['_NEGATIVENODE']._serialized_end=10558 + _globals['_UNNEST']._serialized_start=10560 + _globals['_UNNEST']._serialized_end=10612 + _globals['_INLISTNODE']._serialized_start=10614 + _globals['_INLISTNODE']._serialized_end=10729 + _globals['_SCALARFUNCTIONNODE']._serialized_start=10731 + _globals['_SCALARFUNCTIONNODE']._serialized_end=10835 + _globals['_AGGREGATEEXPRNODE']._serialized_start=10838 + _globals['_AGGREGATEEXPRNODE']._serialized_end=11064 + _globals['_AGGREGATEUDFEXPRNODE']._serialized_start=11067 + _globals['_AGGREGATEUDFEXPRNODE']._serialized_end=11242 + _globals['_SCALARUDFEXPRNODE']._serialized_start=11244 + _globals['_SCALARUDFEXPRNODE']._serialized_end=11324 + _globals['_WINDOWEXPRNODE']._serialized_start=11327 + _globals['_WINDOWEXPRNODE']._serialized_end=11702 + _globals['_BETWEENNODE']._serialized_start=11705 + _globals['_BETWEENNODE']._serialized_end=11863 + _globals['_LIKENODE']._serialized_start=11866 + _globals['_LIKENODE']._serialized_end=12003 + _globals['_ILIKENODE']._serialized_start=12006 + _globals['_ILIKENODE']._serialized_end=12144 + _globals['_SIMILARTONODE']._serialized_start=12147 + _globals['_SIMILARTONODE']._serialized_end=12289 + _globals['_CASENODE']._serialized_start=12292 + _globals['_CASENODE']._serialized_end=12439 + _globals['_WHENTHEN']._serialized_start=12441 + _globals['_WHENTHEN']._serialized_end=12547 + _globals['_CASTNODE']._serialized_start=12549 + _globals['_CASTNODE']._serialized_end=12645 + _globals['_TRYCASTNODE']._serialized_start=12647 + _globals['_TRYCASTNODE']._serialized_end=12746 + _globals['_SORTEXPRNODE']._serialized_start=12748 + _globals['_SORTEXPRNODE']._serialized_end=12839 + _globals['_WINDOWFRAME']._serialized_start=12842 + _globals['_WINDOWFRAME']._serialized_end=13024 + _globals['_WINDOWFRAMEBOUND']._serialized_start=13027 + _globals['_WINDOWFRAMEBOUND']._serialized_end=13158 + _globals['_SCHEMA']._serialized_start=13161 + _globals['_SCHEMA']._serialized_end=13306 _globals['_SCHEMA_METADATAENTRY']._serialized_start=341 _globals['_SCHEMA_METADATAENTRY']._serialized_end=388 - _globals['_FIELD']._serialized_start=11532 - _globals['_FIELD']._serialized_end=11751 + _globals['_FIELD']._serialized_start=13309 + _globals['_FIELD']._serialized_end=13567 _globals['_FIELD_METADATAENTRY']._serialized_start=341 _globals['_FIELD_METADATAENTRY']._serialized_end=388 - _globals['_FIXEDSIZEBINARY']._serialized_start=11753 - _globals['_FIXEDSIZEBINARY']._serialized_end=11786 - _globals['_TIMESTAMP']._serialized_start=11788 - _globals['_TIMESTAMP']._serialized_end=11858 - _globals['_DECIMAL']._serialized_start=11860 - _globals['_DECIMAL']._serialized_end=11915 - _globals['_LIST']._serialized_start=11917 - _globals['_LIST']._serialized_end=11962 - _globals['_FIXEDSIZELIST']._serialized_start=11964 - _globals['_FIXEDSIZELIST']._serialized_end=12037 - _globals['_DICTIONARY']._serialized_start=12039 - _globals['_DICTIONARY']._serialized_end=12125 - _globals['_STRUCT']._serialized_start=12127 - _globals['_STRUCT']._serialized_end=12179 - _globals['_MAP']._serialized_start=12181 - _globals['_MAP']._serialized_end=12246 - _globals['_UNION']._serialized_start=12248 - _globals['_UNION']._serialized_end=12356 - _globals['_SCALARLISTVALUE']._serialized_start=12358 - _globals['_SCALARLISTVALUE']._serialized_end=12467 - _globals['_SCALARTIME32VALUE']._serialized_start=12469 - _globals['_SCALARTIME32VALUE']._serialized_end=12564 - _globals['_SCALARTIME64VALUE']._serialized_start=12566 - _globals['_SCALARTIME64VALUE']._serialized_end=12665 - _globals['_SCALARTIMESTAMPVALUE']._serialized_start=12668 - _globals['_SCALARTIMESTAMPVALUE']._serialized_end=12847 - _globals['_SCALARDICTIONARYVALUE']._serialized_start=12849 - _globals['_SCALARDICTIONARYVALUE']._serialized_end=12955 - _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_start=12957 - _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_end=13029 - _globals['_STRUCTVALUE']._serialized_start=13031 - _globals['_STRUCTVALUE']._serialized_end=13126 - _globals['_SCALARFIXEDSIZEBINARY']._serialized_start=13128 - _globals['_SCALARFIXEDSIZEBINARY']._serialized_end=13183 - _globals['_SCALARVALUE']._serialized_start=13186 - _globals['_SCALARVALUE']._serialized_end=14417 - _globals['_DECIMAL128']._serialized_start=14419 - _globals['_DECIMAL128']._serialized_end=14468 - _globals['_ARROWTYPE']._serialized_start=14471 - _globals['_ARROWTYPE']._serialized_end=15899 - _globals['_EMPTYMESSAGE']._serialized_start=15901 - _globals['_EMPTYMESSAGE']._serialized_end=15915 - _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_start=15917 - _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_end=15965 - _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_start=15967 - _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_end=16017 - _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_start=16019 - _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_end=16070 - _globals['_PLANTYPE']._serialized_start=16073 - _globals['_PLANTYPE']._serialized_end=16595 - _globals['_STRINGIFIEDPLAN']._serialized_start=16597 - _globals['_STRINGIFIEDPLAN']._serialized_end=16669 - _globals['_BARETABLEREFERENCE']._serialized_start=16671 - _globals['_BARETABLEREFERENCE']._serialized_end=16706 - _globals['_PARTIALTABLEREFERENCE']._serialized_start=16708 - _globals['_PARTIALTABLEREFERENCE']._serialized_end=16762 - _globals['_FULLTABLEREFERENCE']._serialized_start=16764 - _globals['_FULLTABLEREFERENCE']._serialized_end=16832 - _globals['_OWNEDTABLEREFERENCE']._serialized_start=16835 - _globals['_OWNEDTABLEREFERENCE']._serialized_end=17030 - _globals['_PHYSICALPLANNODE']._serialized_start=17033 - _globals['_PHYSICALPLANNODE']._serialized_end=18195 - _globals['_PHYSICALEXTENSIONNODE']._serialized_start=18197 - _globals['_PHYSICALEXTENSIONNODE']._serialized_end=18280 - _globals['_PHYSICALEXPRNODE']._serialized_start=18283 - _globals['_PHYSICALEXPRNODE']._serialized_end=19382 - _globals['_PHYSICALSCALARUDFNODE']._serialized_start=19384 - _globals['_PHYSICALSCALARUDFNODE']._serialized_end=19509 - _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_start=19512 - _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_end=19772 - _globals['_PHYSICALWINDOWEXPRNODE']._serialized_start=19775 - _globals['_PHYSICALWINDOWEXPRNODE']._serialized_end=19982 - _globals['_PHYSICALISNULL']._serialized_start=19984 - _globals['_PHYSICALISNULL']._serialized_end=20044 - _globals['_PHYSICALISNOTNULL']._serialized_start=20046 - _globals['_PHYSICALISNOTNULL']._serialized_end=20109 - _globals['_PHYSICALNOT']._serialized_start=20111 - _globals['_PHYSICALNOT']._serialized_end=20168 - _globals['_PHYSICALALIASNODE']._serialized_start=20170 - _globals['_PHYSICALALIASNODE']._serialized_end=20248 - _globals['_PHYSICALBINARYEXPRNODE']._serialized_start=20250 - _globals['_PHYSICALBINARYEXPRNODE']._serialized_end=20368 - _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_start=20371 - _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_end=20499 - _globals['_PHYSICALLIKEEXPRNODE']._serialized_start=20502 - _globals['_PHYSICALLIKEEXPRNODE']._serialized_end=20658 - _globals['_PHYSICALSORTEXPRNODE']._serialized_start=20660 - _globals['_PHYSICALSORTEXPRNODE']._serialized_end=20760 - _globals['_PHYSICALWHENTHEN']._serialized_start=20762 - _globals['_PHYSICALWHENTHEN']._serialized_end=20878 - _globals['_PHYSICALINLISTNODE']._serialized_start=20880 - _globals['_PHYSICALINLISTNODE']._serialized_end=21005 - _globals['_PHYSICALCASENODE']._serialized_start=21008 - _globals['_PHYSICALCASENODE']._serialized_end=21173 - _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_start=21176 - _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_end=21347 - _globals['_PHYSICALTRYCASTNODE']._serialized_start=21349 - _globals['_PHYSICALTRYCASTNODE']._serialized_end=21457 - _globals['_PHYSICALCASTNODE']._serialized_start=21459 - _globals['_PHYSICALCASTNODE']._serialized_end=21564 - _globals['_PHYSICALNEGATIVENODE']._serialized_start=21566 - _globals['_PHYSICALNEGATIVENODE']._serialized_end=21632 - _globals['_FILTEREXECNODE']._serialized_start=21634 - _globals['_FILTEREXECNODE']._serialized_end=21739 - _globals['_FILEGROUP']._serialized_start=21741 - _globals['_FILEGROUP']._serialized_end=21796 - _globals['_SCANLIMIT']._serialized_start=21798 - _globals['_SCANLIMIT']._serialized_end=21824 - _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_start=21826 - _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_end=21926 - _globals['_FILESCANEXECCONF']._serialized_start=21929 - _globals['_FILESCANEXECCONF']._serialized_end=22260 - _globals['_PARQUETSCANEXECNODE']._serialized_start=22262 - _globals['_PARQUETSCANEXECNODE']._serialized_end=22387 - _globals['_CSVSCANEXECNODE']._serialized_start=22389 - _globals['_CSVSCANEXECNODE']._serialized_end=22494 - _globals['_AVROSCANEXECNODE']._serialized_start=22496 - _globals['_AVROSCANEXECNODE']._serialized_end=22563 - _globals['_HASHJOINEXECNODE']._serialized_start=22566 - _globals['_HASHJOINEXECNODE']._serialized_end=22863 - _globals['_UNIONEXECNODE']._serialized_start=22865 - _globals['_UNIONEXECNODE']._serialized_end=22926 - _globals['_EXPLAINEXECNODE']._serialized_start=22928 - _globals['_EXPLAINEXECNODE']._serialized_end=23054 - _globals['_CROSSJOINEXECNODE']._serialized_start=23056 - _globals['_CROSSJOINEXECNODE']._serialized_end=23164 - _globals['_PHYSICALCOLUMN']._serialized_start=23166 - _globals['_PHYSICALCOLUMN']._serialized_end=23211 - _globals['_JOINON']._serialized_start=23213 - _globals['_JOINON']._serialized_end=23306 - _globals['_EMPTYEXECNODE']._serialized_start=23308 - _globals['_EMPTYEXECNODE']._serialized_end=23384 - _globals['_PROJECTIONEXECNODE']._serialized_start=23387 - _globals['_PROJECTIONEXECNODE']._serialized_end=23515 - _globals['_WINDOWAGGEXECNODE']._serialized_start=23518 - _globals['_WINDOWAGGEXECNODE']._serialized_end=23701 - _globals['_MAYBEFILTER']._serialized_start=23703 - _globals['_MAYBEFILTER']._serialized_end=23760 - _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_start=23762 - _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_end=23839 - _globals['_AGGREGATEEXECNODE']._serialized_start=23842 - _globals['_AGGREGATEEXECNODE']._serialized_end=24307 - _globals['_GLOBALLIMITEXECNODE']._serialized_start=24309 - _globals['_GLOBALLIMITEXECNODE']._serialized_end=24404 - _globals['_LOCALLIMITEXECNODE']._serialized_start=24406 - _globals['_LOCALLIMITEXECNODE']._serialized_end=24486 - _globals['_SORTEXECNODE']._serialized_start=24489 - _globals['_SORTEXECNODE']._serialized_end=24638 - _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_start=24641 - _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_end=24774 - _globals['_NESTEDLOOPJOINEXECNODE']._serialized_start=24777 - _globals['_NESTEDLOOPJOINEXECNODE']._serialized_end=24971 - _globals['_COALESCEBATCHESEXECNODE']._serialized_start=24973 - _globals['_COALESCEBATCHESEXECNODE']._serialized_end=25070 - _globals['_COALESCEPARTITIONSEXECNODE']._serialized_start=25072 - _globals['_COALESCEPARTITIONSEXECNODE']._serialized_end=25145 - _globals['_PHYSICALHASHREPARTITION']._serialized_start=25147 - _globals['_PHYSICALHASHREPARTITION']._serialized_end=25246 - _globals['_REPARTITIONEXECNODE']._serialized_start=25249 - _globals['_REPARTITIONEXECNODE']._serialized_end=25430 - _globals['_JOINFILTER']._serialized_start=25433 - _globals['_JOINFILTER']._serialized_end=25580 - _globals['_COLUMNINDEX']._serialized_start=25582 - _globals['_COLUMNINDEX']._serialized_end=25646 - _globals['_PARTITIONEDFILE']._serialized_start=25649 - _globals['_PARTITIONEDFILE']._serialized_end=25809 - _globals['_FILERANGE']._serialized_start=25811 - _globals['_FILERANGE']._serialized_end=25850 - _globals['_PARTITIONSTATS']._serialized_start=25852 - _globals['_PARTITIONSTATS']._serialized_end=25973 - _globals['_STATISTICS']._serialized_start=25975 - _globals['_STATISTICS']._serialized_end=26095 - _globals['_COLUMNSTATS']._serialized_start=26098 - _globals['_COLUMNSTATS']._serialized_end=26243 - _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_start=26245 - _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_end=26359 + _globals['_FIXEDSIZEBINARY']._serialized_start=13569 + _globals['_FIXEDSIZEBINARY']._serialized_end=13602 + _globals['_TIMESTAMP']._serialized_start=13604 + _globals['_TIMESTAMP']._serialized_end=13674 + _globals['_DECIMAL']._serialized_start=13676 + _globals['_DECIMAL']._serialized_end=13731 + _globals['_LIST']._serialized_start=13733 + _globals['_LIST']._serialized_end=13778 + _globals['_FIXEDSIZELIST']._serialized_start=13780 + _globals['_FIXEDSIZELIST']._serialized_end=13853 + _globals['_DICTIONARY']._serialized_start=13855 + _globals['_DICTIONARY']._serialized_end=13941 + _globals['_STRUCT']._serialized_start=13943 + _globals['_STRUCT']._serialized_end=13995 + _globals['_MAP']._serialized_start=13997 + _globals['_MAP']._serialized_end=14062 + _globals['_UNION']._serialized_start=14064 + _globals['_UNION']._serialized_end=14172 + _globals['_SCALARNESTEDVALUE']._serialized_start=14174 + _globals['_SCALARNESTEDVALUE']._serialized_end=14270 + _globals['_SCALARTIME32VALUE']._serialized_start=14272 + _globals['_SCALARTIME32VALUE']._serialized_end=14367 + _globals['_SCALARTIME64VALUE']._serialized_start=14369 + _globals['_SCALARTIME64VALUE']._serialized_end=14468 + _globals['_SCALARTIMESTAMPVALUE']._serialized_start=14471 + _globals['_SCALARTIMESTAMPVALUE']._serialized_end=14650 + _globals['_SCALARDICTIONARYVALUE']._serialized_start=14652 + _globals['_SCALARDICTIONARYVALUE']._serialized_end=14758 + _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_start=14760 + _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_end=14832 + _globals['_SCALARFIXEDSIZEBINARY']._serialized_start=14834 + _globals['_SCALARFIXEDSIZEBINARY']._serialized_end=14889 + _globals['_SCALARVALUE']._serialized_start=14892 + _globals['_SCALARVALUE']._serialized_end=16306 + _globals['_DECIMAL128']._serialized_start=16308 + _globals['_DECIMAL128']._serialized_end=16357 + _globals['_DECIMAL256']._serialized_start=16359 + _globals['_DECIMAL256']._serialized_end=16408 + _globals['_ARROWTYPE']._serialized_start=16411 + _globals['_ARROWTYPE']._serialized_end=17839 + _globals['_EMPTYMESSAGE']._serialized_start=17841 + _globals['_EMPTYMESSAGE']._serialized_end=17855 + _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_start=17857 + _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_end=17905 + _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_start=17907 + _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_end=17957 + _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_start=17959 + _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_end=18010 + _globals['_PLANTYPE']._serialized_start=18013 + _globals['_PLANTYPE']._serialized_end=18665 + _globals['_STRINGIFIEDPLAN']._serialized_start=18667 + _globals['_STRINGIFIEDPLAN']._serialized_end=18739 + _globals['_BARETABLEREFERENCE']._serialized_start=18741 + _globals['_BARETABLEREFERENCE']._serialized_end=18776 + _globals['_PARTIALTABLEREFERENCE']._serialized_start=18778 + _globals['_PARTIALTABLEREFERENCE']._serialized_end=18832 + _globals['_FULLTABLEREFERENCE']._serialized_start=18834 + _globals['_FULLTABLEREFERENCE']._serialized_end=18902 + _globals['_OWNEDTABLEREFERENCE']._serialized_start=18905 + _globals['_OWNEDTABLEREFERENCE']._serialized_end=19100 + _globals['_PHYSICALPLANNODE']._serialized_start=19103 + _globals['_PHYSICALPLANNODE']._serialized_end=20657 + _globals['_PARTITIONCOLUMN']._serialized_start=20659 + _globals['_PARTITIONCOLUMN']._serialized_end=20733 + _globals['_FILETYPEWRITEROPTIONS']._serialized_start=20736 + _globals['_FILETYPEWRITEROPTIONS']._serialized_end=20997 + _globals['_JSONWRITEROPTIONS']._serialized_start=20999 + _globals['_JSONWRITEROPTIONS']._serialized_end=21075 + _globals['_PARQUETWRITEROPTIONS']._serialized_start=21077 + _globals['_PARQUETWRITEROPTIONS']._serialized_end=21156 + _globals['_CSVWRITEROPTIONS']._serialized_start=21159 + _globals['_CSVWRITEROPTIONS']._serialized_end=21386 + _globals['_ARROWWRITEROPTIONS']._serialized_start=21388 + _globals['_ARROWWRITEROPTIONS']._serialized_end=21408 + _globals['_WRITERPROPERTIES']._serialized_start=21411 + _globals['_WRITERPROPERTIES']._serialized_end=21628 + _globals['_FILESINKCONFIG']._serialized_start=21631 + _globals['_FILESINKCONFIG']._serialized_end=21940 + _globals['_JSONSINK']._serialized_start=21942 + _globals['_JSONSINK']._serialized_end=21996 + _globals['_JSONSINKEXECNODE']._serialized_start=21999 + _globals['_JSONSINKEXECNODE']._serialized_end=22203 + _globals['_CSVSINK']._serialized_start=22205 + _globals['_CSVSINK']._serialized_end=22258 + _globals['_CSVSINKEXECNODE']._serialized_start=22261 + _globals['_CSVSINKEXECNODE']._serialized_end=22463 + _globals['_PARQUETSINK']._serialized_start=22465 + _globals['_PARQUETSINK']._serialized_end=22522 + _globals['_PARQUETSINKEXECNODE']._serialized_start=22525 + _globals['_PARQUETSINKEXECNODE']._serialized_end=22735 + _globals['_PHYSICALEXTENSIONNODE']._serialized_start=22737 + _globals['_PHYSICALEXTENSIONNODE']._serialized_end=22820 + _globals['_PHYSICALEXPRNODE']._serialized_start=22823 + _globals['_PHYSICALEXPRNODE']._serialized_end=23847 + _globals['_PHYSICALSCALARUDFNODE']._serialized_start=23849 + _globals['_PHYSICALSCALARUDFNODE']._serialized_end=23974 + _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_start=23977 + _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_end=24237 + _globals['_PHYSICALWINDOWEXPRNODE']._serialized_start=24240 + _globals['_PHYSICALWINDOWEXPRNODE']._serialized_end=24612 + _globals['_PHYSICALISNULL']._serialized_start=24614 + _globals['_PHYSICALISNULL']._serialized_end=24674 + _globals['_PHYSICALISNOTNULL']._serialized_start=24676 + _globals['_PHYSICALISNOTNULL']._serialized_end=24739 + _globals['_PHYSICALNOT']._serialized_start=24741 + _globals['_PHYSICALNOT']._serialized_end=24798 + _globals['_PHYSICALALIASNODE']._serialized_start=24800 + _globals['_PHYSICALALIASNODE']._serialized_end=24878 + _globals['_PHYSICALBINARYEXPRNODE']._serialized_start=24880 + _globals['_PHYSICALBINARYEXPRNODE']._serialized_end=24998 + _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_start=25001 + _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_end=25129 + _globals['_PHYSICALLIKEEXPRNODE']._serialized_start=25132 + _globals['_PHYSICALLIKEEXPRNODE']._serialized_end=25288 + _globals['_PHYSICALSORTEXPRNODE']._serialized_start=25290 + _globals['_PHYSICALSORTEXPRNODE']._serialized_end=25390 + _globals['_PHYSICALWHENTHEN']._serialized_start=25392 + _globals['_PHYSICALWHENTHEN']._serialized_end=25508 + _globals['_PHYSICALINLISTNODE']._serialized_start=25510 + _globals['_PHYSICALINLISTNODE']._serialized_end=25635 + _globals['_PHYSICALCASENODE']._serialized_start=25638 + _globals['_PHYSICALCASENODE']._serialized_end=25803 + _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_start=25806 + _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_end=25977 + _globals['_PHYSICALTRYCASTNODE']._serialized_start=25979 + _globals['_PHYSICALTRYCASTNODE']._serialized_end=26087 + _globals['_PHYSICALCASTNODE']._serialized_start=26089 + _globals['_PHYSICALCASTNODE']._serialized_end=26194 + _globals['_PHYSICALNEGATIVENODE']._serialized_start=26196 + _globals['_PHYSICALNEGATIVENODE']._serialized_end=26262 + _globals['_FILTEREXECNODE']._serialized_start=26265 + _globals['_FILTEREXECNODE']._serialized_end=26406 + _globals['_FILEGROUP']._serialized_start=26408 + _globals['_FILEGROUP']._serialized_end=26463 + _globals['_SCANLIMIT']._serialized_start=26465 + _globals['_SCANLIMIT']._serialized_end=26491 + _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_start=26493 + _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_end=26593 + _globals['_FILESCANEXECCONF']._serialized_start=26596 + _globals['_FILESCANEXECCONF']._serialized_end=26927 + _globals['_PARQUETSCANEXECNODE']._serialized_start=26929 + _globals['_PARQUETSCANEXECNODE']._serialized_end=27054 + _globals['_CSVSCANEXECNODE']._serialized_start=27057 + _globals['_CSVSCANEXECNODE']._serialized_end=27214 + _globals['_AVROSCANEXECNODE']._serialized_start=27216 + _globals['_AVROSCANEXECNODE']._serialized_end=27283 + _globals['_HASHJOINEXECNODE']._serialized_start=27286 + _globals['_HASHJOINEXECNODE']._serialized_end=27583 + _globals['_SYMMETRICHASHJOINEXECNODE']._serialized_start=27586 + _globals['_SYMMETRICHASHJOINEXECNODE']._serialized_end=28017 + _globals['_INTERLEAVEEXECNODE']._serialized_start=28019 + _globals['_INTERLEAVEEXECNODE']._serialized_end=28085 + _globals['_UNIONEXECNODE']._serialized_start=28087 + _globals['_UNIONEXECNODE']._serialized_end=28148 + _globals['_EXPLAINEXECNODE']._serialized_start=28150 + _globals['_EXPLAINEXECNODE']._serialized_end=28276 + _globals['_ANALYZEEXECNODE']._serialized_start=28279 + _globals['_ANALYZEEXECNODE']._serialized_end=28419 + _globals['_CROSSJOINEXECNODE']._serialized_start=28421 + _globals['_CROSSJOINEXECNODE']._serialized_end=28529 + _globals['_PHYSICALCOLUMN']._serialized_start=28531 + _globals['_PHYSICALCOLUMN']._serialized_end=28576 + _globals['_JOINON']._serialized_start=28578 + _globals['_JOINON']._serialized_end=28675 + _globals['_EMPTYEXECNODE']._serialized_start=28677 + _globals['_EMPTYEXECNODE']._serialized_end=28728 + _globals['_PLACEHOLDERROWEXECNODE']._serialized_start=28730 + _globals['_PLACEHOLDERROWEXECNODE']._serialized_end=28790 + _globals['_PROJECTIONEXECNODE']._serialized_start=28793 + _globals['_PROJECTIONEXECNODE']._serialized_end=28921 + _globals['_PARTIALLYSORTEDINPUTORDERMODE']._serialized_start=28923 + _globals['_PARTIALLYSORTEDINPUTORDERMODE']._serialized_end=28971 + _globals['_WINDOWAGGEXECNODE']._serialized_start=28974 + _globals['_WINDOWAGGEXECNODE']._serialized_end=29328 + _globals['_MAYBEFILTER']._serialized_start=29330 + _globals['_MAYBEFILTER']._serialized_end=29387 + _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_start=29389 + _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_end=29466 + _globals['_AGGREGATEEXECNODE']._serialized_start=29469 + _globals['_AGGREGATEEXECNODE']._serialized_end=29875 + _globals['_GLOBALLIMITEXECNODE']._serialized_start=29877 + _globals['_GLOBALLIMITEXECNODE']._serialized_end=29972 + _globals['_LOCALLIMITEXECNODE']._serialized_start=29974 + _globals['_LOCALLIMITEXECNODE']._serialized_end=30054 + _globals['_SORTEXECNODE']._serialized_start=30057 + _globals['_SORTEXECNODE']._serialized_end=30206 + _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_start=30209 + _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_end=30342 + _globals['_NESTEDLOOPJOINEXECNODE']._serialized_start=30345 + _globals['_NESTEDLOOPJOINEXECNODE']._serialized_end=30539 + _globals['_COALESCEBATCHESEXECNODE']._serialized_start=30541 + _globals['_COALESCEBATCHESEXECNODE']._serialized_end=30638 + _globals['_COALESCEPARTITIONSEXECNODE']._serialized_start=30640 + _globals['_COALESCEPARTITIONSEXECNODE']._serialized_end=30713 + _globals['_PHYSICALHASHREPARTITION']._serialized_start=30715 + _globals['_PHYSICALHASHREPARTITION']._serialized_end=30814 + _globals['_REPARTITIONEXECNODE']._serialized_start=30817 + _globals['_REPARTITIONEXECNODE']._serialized_end=30998 + _globals['_JOINFILTER']._serialized_start=31001 + _globals['_JOINFILTER']._serialized_end=31148 + _globals['_COLUMNINDEX']._serialized_start=31150 + _globals['_COLUMNINDEX']._serialized_end=31214 + _globals['_PARTITIONEDFILE']._serialized_start=31217 + _globals['_PARTITIONEDFILE']._serialized_end=31377 + _globals['_FILERANGE']._serialized_start=31379 + _globals['_FILERANGE']._serialized_end=31418 + _globals['_PARTITIONSTATS']._serialized_start=31420 + _globals['_PARTITIONSTATS']._serialized_end=31541 + _globals['_PRECISION']._serialized_start=31543 + _globals['_PRECISION']._serialized_end=31643 + _globals['_STATISTICS']._serialized_start=31646 + _globals['_STATISTICS']._serialized_end=31794 + _globals['_COLUMNSTATS']._serialized_start=31797 + _globals['_COLUMNSTATS']._serialized_end=31984 + _globals['_NAMEDSTRUCTFIELDEXPR']._serialized_start=31986 + _globals['_NAMEDSTRUCTFIELDEXPR']._serialized_end=32047 + _globals['_LISTINDEXEXPR']._serialized_start=32049 + _globals['_LISTINDEXEXPR']._serialized_end=32107 + _globals['_LISTRANGEEXPR']._serialized_start=32110 + _globals['_LISTRANGEEXPR']._serialized_end=32260 + _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_start=32263 + _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_end=32525 # @@protoc_insertion_point(module_scope)