diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 879ffa031..c3fed0693 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -105,7 +105,7 @@ jobs: with: cache-on-failure: True - name: Build wheels (Linux) - uses: messense/maturin-action@9357a23898d2fca5ef8da7ac38aaaedab4ef218f # pin@v1.40.2 + uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 with: command: build manylinux: 2014 @@ -137,10 +137,10 @@ jobs: run: | rustup target add aarch64-unknown-linux-gnu - name: Build arm64 wheels - uses: messense/maturin-action@6d52485c3b3044e20b4c8ee6ce6f61e20a7645b0 # pin@v1 + uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 with: command: build - manylinux: 2014 + manylinux: 2_28 rust-toolchain: stable args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu - name: Upload artifacts diff --git a/Cargo.lock b/Cargo.lock index 5108479e1..4402b26fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -17,24 +17,31 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" dependencies = [ "cfg-if", "const-random", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -56,9 +63,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "android-tzdata" @@ -83,49 +90,57 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.2.6" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", - "is-terminal", + "colorchoice", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "anstyle-wincon" -version = "0.2.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arrayref" @@ -135,17 +150,16 @@ checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" +checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" dependencies = [ - "ahash", "arrow-arith", "arrow-array", "arrow-buffer", @@ -164,24 +178,24 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "half 2.2.1", + "half", "num", ] [[package]] name = "arrow-array" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ "ahash", "arrow-buffer", @@ -189,45 +203,46 @@ dependencies = [ "arrow-schema", "chrono", "chrono-tz", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", "num", ] [[package]] name = "arrow-buffer" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" dependencies = [ "bytes", - "half 2.2.1", + "half", "num", ] [[package]] name = "arrow-cast" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "base64", "chrono", "comfy-table", - "half 2.2.1", + "half", "lexical-core", "num", ] [[package]] name = "arrow-csv" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" +checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,21 +259,21 @@ dependencies = [ [[package]] name = "arrow-data" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" dependencies = [ "arrow-buffer", "arrow-schema", - "half 2.2.1", + "half", "num", ] [[package]] name = "arrow-ipc" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" dependencies = [ "arrow-array", "arrow-buffer", @@ -266,13 +281,14 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] name = "arrow-json" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" +checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -280,8 +296,8 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "half 2.2.1", - "indexmap 2.0.0", + "half", + "indexmap 2.2.3", "lexical-core", "num", "serde", @@ -290,48 +306,48 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "half 2.2.1", + "half", "num", ] [[package]] name = "arrow-row" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", ] [[package]] name = "arrow-schema" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" dependencies = [ - "bitflags 2.1.0", + "bitflags 2.4.2", ] [[package]] name = "arrow-select" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ "ahash", "arrow-array", @@ -343,9 +359,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" +checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -359,9 +375,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.10" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b2340f55d9661d76793b2bfc2eb0e62689bd79d067a95707ea762afd5e9dd" +checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" dependencies = [ "anstyle", "bstr", @@ -384,20 +400,33 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.8.0" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 5.1.0", + "event-listener-strategy 0.5.0", "futures-core", + "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.0" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "bzip2", "flate2", @@ -413,30 +442,30 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.5.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17adb73da160dfb475c183343c8cccd80721ea5a605d3eb57125f0a7b7a92d0b" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" dependencies = [ - "async-lock", + "async-lock 3.3.0", "async-task", "concurrent-queue", - "fastrand", - "futures-lite", + "fastrand 2.0.1", + "futures-lite 2.2.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel", + "async-channel 2.2.0", "async-executor", - "async-io", - "async-lock", + "async-io 2.3.1", + "async-lock 3.3.0", "blocking", - "futures-lite", + "futures-lite 2.2.0", "once_cell", ] @@ -446,27 +475,57 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ - "async-lock", + "async-lock 2.8.0", "autocfg", "cfg-if", "concurrent-queue", - "futures-lite", + "futures-lite 1.13.0", "log", "parking", - "polling", - "rustix", + "polling 2.8.0", + "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f97ab0c5b00a7cdbe5a371b9a782ee7be1316095885c8a4ea1daf490eb0ef65" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.2.0", + "parking", + "polling 3.5.0", + "rustix 0.38.31", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener", + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", + "pin-project-lite", ] [[package]] @@ -477,7 +536,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -487,15 +546,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ "async-attributes", - "async-channel", + "async-channel 1.9.0", "async-global-executor", - "async-io", - "async-lock", + "async-io 1.13.0", + "async-lock 2.8.0", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite", + "futures-lite 1.13.0", "gloo-timers", "kv-log-macro", "log", @@ -509,9 +568,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", @@ -520,37 +579,37 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "async-task" -version = "4.4.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "atomic-waker" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "atty" @@ -580,9 +639,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.12" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -608,9 +667,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", "bytes", @@ -625,24 +684,24 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", "cfg-if", "libc", - "miniz_oxide 0.7.1", + "miniz_oxide 0.7.2", "object", "rustc-demangle", ] [[package]] name = "base64" -version = "0.21.2" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -652,9 +711,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.1.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70beb79cbb5ce9c4f8e20849978f34225931f665bb49efa6982875a4d5facb3" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "blake2" @@ -667,16 +726,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.3.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "digest", ] [[package]] @@ -690,23 +748,25 @@ dependencies = [ [[package]] name = "blocking" -version = "1.3.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c67b173a56acffd6d2326fb7ab938ba0b00a71480e14902b2591c87bc5741e8" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel", - "async-lock", + "async-channel 2.2.0", + "async-lock 3.3.0", "async-task", - "atomic-waker", - "fastrand", - "futures-lite", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.2.0", + "piper", + "tracing", ] [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -715,9 +775,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -725,39 +785,38 @@ dependencies = [ [[package]] name = "bstr" -version = "1.4.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "once_cell", - "regex-automata 0.1.10", + "regex-automata", "serde", ] [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2" @@ -788,11 +847,11 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "3286b845d0fccbdd15af433f61c5970e711987036cb468f437ff6badd70f4e24" dependencies = [ - "jobserver", + "libc", ] [[package]] @@ -809,9 +868,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -819,14 +878,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.0", + "windows-targets 0.52.3", ] [[package]] name = "chrono-tz" -version = "0.8.1" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa48fa079165080f11d7753fd0bc175b7d391f276b965fe4b55bfad67856e463" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", "chrono-tz-build", @@ -835,9 +894,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" dependencies = [ "parse-zoneinfo", "phf", @@ -846,9 +905,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -857,25 +916,25 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 1.8.2", + "half", ] [[package]] name = "clap" -version = "3.2.23" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", @@ -885,38 +944,36 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", - "bitflags 1.3.2", - "clap_lex 0.4.1", + "clap_lex 0.7.0", "strsim", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -930,19 +987,21 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "color_quant" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "combine" @@ -956,35 +1015,20 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum 0.24.1", - "strum_macros 0.24.3", + "strum 0.25.0", + "strum_macros 0.25.3", "unicode-width", ] -[[package]] -name = "concolor-override" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] - [[package]] name = "concurrent-queue" -version = "2.1.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -1001,37 +1045,35 @@ dependencies = [ [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom", "once_cell", - "proc-macro-hack", "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1039,24 +1081,24 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1071,7 +1113,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 3.2.25", "criterion-plot", "futures", "itertools 0.10.5", @@ -1099,48 +1141,30 @@ dependencies = [ "itertools 0.10.5", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.8.0", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1160,9 +1184,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -1172,79 +1196,36 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.31", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.31", -] - [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.3", "lock_api", "once_cell", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.9", ] [[package]] name = "datafusion" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7014432223f4d721cb9786cd88bb89e7464e0ba984d4a7f49db7787f5f268674" +checksum = "b2b360b692bf6c6d6e6b6dbaf41a3be0020daeceac0f406aed54c75331e50dbb" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-ipc", "arrow-schema", "async-compression", "async-trait", @@ -1255,6 +1236,8 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions", + "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -1262,19 +1245,18 @@ dependencies = [ "flate2", "futures", "glob", - "half 2.2.1", - "hashbrown 0.14.0", - "indexmap 2.0.0", - "itertools 0.11.0", + "half", + "hashbrown 0.14.3", + "indexmap 2.2.3", + "itertools 0.12.1", "log", "num_cpus", "object_store", "parking_lot 0.12.1", "parquet", - "percent-encoding", "pin-project-lite", "rand", - "sqlparser 0.38.0", + "sqlparser 0.43.1", "tempfile", "tokio", "tokio-util", @@ -1286,9 +1268,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3903ed8f102892f17b48efa437f3542159241d41c564f0d1e78efdc5e663aa" +checksum = "37f343ccc298f440e25aa38ff82678291a7acc24061c7370ba6c0ff5cc811412" dependencies = [ "ahash", "arrow", @@ -1296,19 +1278,20 @@ dependencies = [ "arrow-buffer", "arrow-schema", "chrono", - "half 2.2.1", + "half", + "libc", "num_cpus", "object_store", "parquet", "pyo3", - "sqlparser 0.38.0", + "sqlparser 0.43.1", ] [[package]] name = "datafusion-execution" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780b73b2407050e53f51a9781868593f694102c59e622de9a8aafc0343c4f237" +checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" dependencies = [ "arrow", "chrono", @@ -1316,7 +1299,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "log", "object_store", "parking_lot 0.12.1", @@ -1327,24 +1310,54 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c382676338d8caba6c027ba0da47260f65ffedab38fda78f6d8043f607557c" +checksum = "e204d89909e678846b6a95f156aafc1ee5b36cb6c9e37ec2e1449b078a38c818" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", - "sqlparser 0.38.0", - "strum 0.25.0", - "strum_macros 0.25.1", + "paste", + "sqlparser 0.43.1", + "strum 0.26.1", + "strum_macros 0.26.1", +] + +[[package]] +name = "datafusion-functions" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98f1c73f7801b2b8ba2297b3ad78ffcf6c1fc6b8171f502987eb9ad5cb244ee7" +dependencies = [ + "arrow", + "base64", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hex", + "log", +] + +[[package]] +name = "datafusion-functions-array" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d16a0ddf2c991526f6ffe2f47a72c6da0b7354d6c32411dd20631fe2e38937" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "log", + "paste", ] [[package]] name = "datafusion-optimizer" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2904a432f795484fd45e29ded4537152adb60f636c05691db34fcd94c92c96" +checksum = "5ae27e07bf1f04d327be5c2a293470879801ab5535204dc3b16b062fda195496" dependencies = [ "arrow", "async-trait", @@ -1352,35 +1365,37 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.0", - "itertools 0.11.0", + "hashbrown 0.14.3", + "itertools 0.12.1", "log", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b4968e9a998dc0476c4db7a82f280e2026b25f464e4aa0c3bb9807ee63ddfd" +checksum = "dde620cd9ef76a3bca9c754fb68854bd2349c49f55baf97e08001f9e967f6d6b" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", + "arrow-string", "base64", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-execution", "datafusion-expr", - "half 2.2.1", - "hashbrown 0.14.0", + "half", + "hashbrown 0.14.3", "hex", - "indexmap 2.0.0", - "itertools 0.11.0", - "libc", + "indexmap 2.2.3", + "itertools 0.12.1", "log", "md-5", "paste", @@ -1394,9 +1409,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd0d1fe54e37a47a2d58a1232c22786f2c28ad35805fdcd08f0253a8b0aaa90" +checksum = "9a4c75fba9ea99d64b2246cbd2fcae2e6fc973e6616b1015237a616036506dd4" dependencies = [ "ahash", "arrow", @@ -1410,10 +1425,10 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "futures", - "half 2.2.1", - "hashbrown 0.14.0", - "indexmap 2.0.0", - "itertools 0.11.0", + "half", + "hashbrown 0.14.3", + "indexmap 2.2.3", + "itertools 0.12.1", "log", "once_cell", "parking_lot 0.12.1", @@ -1425,9 +1440,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca1db9605e1f545b852fa9dd05f03339b333c9f2814a4e33b9ac7e9e586a1014" +checksum = "2292251d5441d88d92a90d1511d5a8c88759a6562ff38ac1711b1587e6bf19c4" dependencies = [ "arrow", "chrono", @@ -1440,16 +1455,26 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "32.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b568d44c87ead99604d704f942e257c8a236ee1bbf890ee3e034ad659dcb2c21" +checksum = "21474a95c3a62d113599d21b439fa15091b538bac06bd20be0bb2e7d22903c09" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.38.0", + "sqlparser 0.43.1", +] + +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", ] [[package]] @@ -1481,59 +1506,26 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dssim" -version = "3.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea54acfcd238d680c2c720a5eef56d210aaabc7582a48615602bcdf0c9a5894" -dependencies = [ - "dssim-core", - "getopts", - "imgref", - "load_image", - "lodepng", - "rayon", - "rgb", -] - -[[package]] -name = "dssim-core" -version = "3.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0286ae9482caf2cc3fb8ce55c5ff7dc1986425f36fd8423f00e38a83b4169fb" -dependencies = [ - "imgref", - "itertools 0.10.5", - "rayon", - "rgb", -] - -[[package]] -name = "dunce" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" - [[package]] name = "either" -version = "1.8.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -1550,36 +1542,67 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.0" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "event-listener" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] name = "event-listener" -version = "2.5.3" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "b7ad6fd685ce13acd6d9541a30f6db6567a7a24c9ffd4ba2955d29e3f22c8b27" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" +dependencies = [ + "event-listener 5.1.0", + "pin-project-lite", +] [[package]] name = "fallible_collections" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acf77205554f3cfeca94a4b910e159ad9824e8c2d164de02b3f12495cc1074d" +checksum = "a88c69768c0a15262df21899142bc6df9b9b823546d4b4b9a7bc2d6c448ec6fd" dependencies = [ "hashbrown 0.13.2", ] @@ -1593,6 +1616,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1601,9 +1630,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.1.21" +version = "23.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1611,12 +1640,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", - "miniz_oxide 0.6.2", + "miniz_oxide 0.7.2", ] [[package]] @@ -1634,47 +1663,20 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" -dependencies = [ - "foreign-types-macros", - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-macros" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.31", -] - -[[package]] -name = "foreign-types-shared" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" - [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1687,9 +1689,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1697,15 +1699,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1714,17 +1716,17 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -1733,40 +1735,53 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1790,20 +1805,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -1812,11 +1818,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gif" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -1838,9 +1854,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -1848,7 +1864,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.2.3", "slab", "tokio", "tokio-util", @@ -1857,16 +1873,11 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" - -[[package]] -name = "half" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" dependencies = [ + "cfg-if", "crunchy", "num-traits", ] @@ -1888,9 +1899,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ "ahash", "allocator-api2", @@ -1913,18 +1924,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "379dada1584ad501b383485dd706b8afb7a70fcbc7f4da7d780638a5a6124a60" [[package]] name = "hex" @@ -1932,11 +1934,20 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1945,9 +1956,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1956,9 +1967,9 @@ dependencies = [ [[package]] name = "http-range-header" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" @@ -1968,9 +1979,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -1980,9 +1991,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1995,7 +2006,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2030,43 +2041,55 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] name = "idna" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", ] [[package]] -name = "imgref" -version = "1.9.4" +name = "image" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cf49df1085dcfb171460e4592597b84abe50d900fb83efb6e41b20fefd6c2c" +checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational 0.3.2", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] [[package]] name = "indexmap" @@ -2080,19 +2103,19 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "instant" @@ -2114,31 +2137,30 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi 0.3.8", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.7.2" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.6" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.45.0", + "hermit-abi 0.3.8", + "libc", + "windows-sys 0.52.0", ] [[package]] @@ -2159,11 +2181,20 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jni" @@ -2187,38 +2218,29 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - [[package]] name = "jpeg-decoder" -version = "0.3.0" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" dependencies = [ "rayon", ] [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] [[package]] name = "json-patch" -version = "1.0.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f54898088ccb91df1b492cc80029a6fdf1c48ca0db7c6822a8babad69c94658" +checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6" dependencies = [ "serde", "serde_json", @@ -2241,28 +2263,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lcms2" -version = "5.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e736cf1cff040514bc97fd74ab45d622bba780e8799e43efba3269cecaaa45b" -dependencies = [ - "foreign-types", - "lcms2-sys", -] - -[[package]] -name = "lcms2-sys" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908236fb4fa8f210c5aefbec04e0bd6fea22b7f2796d479130037244d39078a8" -dependencies = [ - "cc", - "dunce", - "libc", - "pkg-config", -] - [[package]] name = "lexical-core" version = "0.8.5" @@ -2329,52 +2329,33 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libm" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" - -[[package]] -name = "link-cplusplus" -version = "1.0.8" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] -name = "load_image" -version = "3.0.3" +name = "linux-raw-sys" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f87f0f354b901237de8e6d4c0f050a3aa85b83a458fea9796f993a08a67c3cc" -dependencies = [ - "bytemuck", - "imgref", - "jpeg-decoder", - "lcms2", - "lodepng", - "quick-error", - "rexif", - "rgb", -] +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -2382,9 +2363,9 @@ dependencies = [ [[package]] name = "lodepng" -version = "3.7.2" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0ad39f75bbaa4b10bb6f2316543632a8046a5bcf9c785488d79720b21f044f8" +checksum = "a42d298694b14401847de29abd44adf278b42e989e516deac7b72018400002d8" dependencies = [ "crc32fast", "fallible_collections", @@ -2408,27 +2389,16 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" dependencies = [ - "hashbrown 0.14.0", -] - -[[package]] -name = "lz4" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" -dependencies = [ - "libc", - "lz4-sys", + "hashbrown 0.14.3", ] [[package]] -name = "lz4-sys" -version = "1.9.4" +name = "lz4_flex" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" dependencies = [ - "cc", - "libc", + "twox-hash", ] [[package]] @@ -2444,33 +2414,25 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -2499,32 +2461,41 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", + "autocfg", ] [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", - "log", "wasi", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2549,15 +2520,15 @@ dependencies = [ "num-complex", "num-integer", "num-iter", - "num-rational", + "num-rational 0.4.1", "num-traits", ] [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -2566,28 +2537,38 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", "num-integer", @@ -2608,9 +2589,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -2618,28 +2599,28 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.8", "libc", ] [[package]] name = "object" -version = "0.31.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d359e231e5451f4f9fa889d56e3ce34f8724f1a61db2107739359717cf2bbf08" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" dependencies = [ "async-trait", "base64", @@ -2648,7 +2629,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.10.5", + "itertools 0.12.1", "parking_lot 0.12.1", "percent-encoding", "quick-xml", @@ -2666,9 +2647,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -2676,35 +2657,41 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] [[package]] name = "ordered-float" -version = "3.6.0" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a384337e997e6860ffbaa83708b2ef329fd8c54cb67a5f64d421e0f943254f" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" dependencies = [ "num-traits", ] [[package]] name = "os_str_bytes" -version = "6.5.0" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" [[package]] name = "parking" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -2724,7 +2711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.9", ] [[package]] @@ -2743,22 +2730,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.4.1", "smallvec", - "windows-sys 0.45.0", + "windows-targets 0.48.5", ] [[package]] name = "parquet" -version = "47.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ "ahash", "arrow-array", @@ -2774,8 +2761,9 @@ dependencies = [ "chrono", "flate2", "futures", - "hashbrown 0.14.0", - "lz4", + "half", + "hashbrown 0.14.3", + "lz4_flex", "num", "num-bigint", "object_store", @@ -2799,40 +2787,40 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap 2.2.3", ] [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" dependencies = [ "phf_generator", "phf_shared", @@ -2840,9 +2828,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand", @@ -2850,31 +2838,31 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] @@ -2889,17 +2877,37 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + +[[package]] +name = "pixelmatch" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b15774a2acbbefbf12db95ef5582f80a3c9628dc48b6397e563c08640799764" +dependencies = [ + "image", +] + [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -2910,24 +2918,36 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "polling" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e1f879b2998099c2d69ab9605d145d5b661195627eccc680002c4918a7fb6fa" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", @@ -2936,9 +2956,29 @@ dependencies = [ "libc", "log", "pin-project-lite", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] +[[package]] +name = "polling" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.31", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2947,14 +2987,13 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c575290b64d24745b6c57a12a31465f0a66f3a4799686a6921526a33b0797965" +checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" dependencies = [ "anstyle", "difflib", "float-cmp", - "itertools 0.10.5", "normalize-line-endings", "predicates-core", "regex", @@ -2977,59 +3016,29 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" -dependencies = [ - "proc-macro2", - "syn 2.0.31", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" +name = "prettyplease" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +dependencies = [ + "proc-macro2", + "syn 2.0.50", +] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", "prost-derive", @@ -3037,9 +3046,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck", @@ -3052,29 +3061,29 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.31", + "syn 2.0.50", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "prost-types" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ "prost", ] @@ -3090,15 +3099,16 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb88ae05f306b4bfcde40ac4a51dc0b05936a9207a4b75b798c7729c4258a59" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.9.0", + "memoffset", "parking_lot 0.12.1", + "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -3107,9 +3117,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554db24f0b3c180a9c0b1268f91287ab3f17c162e15b54caaae5a6b3773396b0" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" dependencies = [ "once_cell", "target-lexicon", @@ -3117,9 +3127,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "922ede8759e8600ad4da3195ae41259654b9c55da4f7eec84a0ccc7d067a70a4" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" dependencies = [ "libc", "pyo3-build-config", @@ -3127,48 +3137,44 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5caec6a1dd355964a841fcbeeb1b89fe4146c87295573f94228911af3cc5a2" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b78ccbb160db1556cdb6fd96c50334c5d4ec44dc5e0a968d0a1208fa0efa8b" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" dependencies = [ + "heck", "proc-macro2", + "pyo3-build-config", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "pythonize" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e35b716d430ace57e2d1b4afb51c9e5b7c46d2bce72926e07f9be6a98ced03e" +checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850" dependencies = [ "pyo3", "serde", ] -[[package]] -name = "quick-error" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", "serde", @@ -3176,9 +3182,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -3215,9 +3221,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -3225,14 +3231,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -3246,36 +3250,30 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.5" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.8", + "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - -[[package]] -name = "regex-automata" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -3284,21 +3282,21 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "relative-path" -version = "1.9.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" +checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64", "bytes", @@ -3319,10 +3317,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", + "rustls-native-certs", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls", @@ -3339,9 +3339,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c50db2c7ccd815f976473dd7d0bde296f8c3b77c383acf4fc021cdcf10852b" +checksum = "88a3e86aa6053e59030e7ce2d2a3b258dd08fc2d337d52f73f6cb480f5858690" dependencies = [ "anyhow", "async-trait", @@ -3377,46 +3377,37 @@ dependencies = [ [[package]] name = "retry-policies" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a715dc4d0e8aea3085a9a94d76e79c79c7df7c9f6be609da841a6d2489ca3687" +checksum = "17dd00bff1d737c40dbcd47d4375281bf4c17933f9eef0a185fc7bacca23ecbd" dependencies = [ "anyhow", "chrono", "rand", ] -[[package]] -name = "rexif" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49352965b70522af9085d7a8c2a6df7494713c67ac58b9af02bcff7fb4ca1483" -dependencies = [ - "num", -] - [[package]] name = "rgb" -version = "0.8.36" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" dependencies = [ "bytemuck", ] [[package]] name = "ring" -version = "0.16.20" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", + "getrandom", "libc", - "once_cell", "spin", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -3444,7 +3435,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.31", + "syn 2.0.50", "unicode-ident", ] @@ -3457,7 +3448,7 @@ dependencies = [ "quote", "rand", "rustc_version", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -3477,23 +3468,36 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.6" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys 0.45.0", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring", @@ -3501,20 +3505,32 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", "untrusted", @@ -3522,15 +3538,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -3541,6 +3557,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3548,66 +3573,89 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] -name = "scopeguard" -version = "1.1.0" +name = "scoped_threadpool" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" [[package]] -name = "scratch" -version = "1.0.5" +name = "scopeguard" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", "untrusted", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" -version = "1.0.17" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "seq-macro" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.2.3", "itoa", "ryu", "serde", @@ -3615,9 +3663,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -3636,9 +3684,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -3647,30 +3695,30 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "snafu" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", "snafu-derive", @@ -3678,9 +3726,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck", "proc-macro2", @@ -3690,15 +3738,15 @@ dependencies = [ [[package]] name = "snap" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -3706,34 +3754,34 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.37.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ae05a8250b968a3f7db93155a84d68b2e6cea1583949af5ca5b5170c76c075" +checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" dependencies = [ "log", ] [[package]] name = "sqlparser" -version = "0.38.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" +checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" dependencies = [ "log", "sqlparser_derive", @@ -3741,13 +3789,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.1.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] @@ -3758,56 +3806,56 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "strum" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "723b93e8addf9aa965ebe2d11da6d7540fa2283fcea14b3371ff055f7ba13f5f" dependencies = [ - "strum_macros 0.25.1", + "strum_macros 0.26.1", ] [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "strum_macros" -version = "0.25.1" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" +checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -3822,9 +3870,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" dependencies = [ "proc-macro2", "quote", @@ -3860,9 +3908,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "task-local-extensions" @@ -3875,22 +3923,21 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", - "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", + "fastrand 2.0.1", + "rustix 0.38.31", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -3903,63 +3950,61 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "test-case" -version = "3.1.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a1d6e7bde536b0412f20765b76e921028059adfd1b90d8974d33fd3c91b25df" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" dependencies = [ "test-case-macros", ] [[package]] name = "test-case-core" -version = "3.1.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10394d5d1e27794f772b6fc854c7e91a2dc26e2cbf807ad523370c2a59c0cee" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" dependencies = [ "cfg-if", - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "test-case-macros" -version = "3.1.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb9a44b1c6a54c1ba58b152797739dba2a83ca74e18168a68c980eb142f9404" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", "test-case-core", ] [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -3970,7 +4015,18 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float 2.10.0", + "ordered-float 2.10.1", +] + +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.4", + "weezl", ] [[package]] @@ -4009,18 +4065,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", - "parking_lot 0.12.1", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.6", "tokio-macros", "windows-sys 0.48.0", ] @@ -4037,13 +4092,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -4058,9 +4113,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -4069,9 +4124,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -4083,9 +4138,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", @@ -4095,20 +4150,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.2.3", "serde", "serde_spanned", "toml_datetime", @@ -4155,7 +4210,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.31", + "syn 2.0.50", ] [[package]] @@ -4200,11 +4255,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "bytes", "futures-core", "futures-util", @@ -4230,11 +4285,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4242,38 +4296,38 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "treediff" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" +checksum = "4d127780145176e2b5d16611cc25a900150e86e9fd79d3bde6ff3a37359c9cb5" dependencies = [ "serde_json", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" @@ -4287,69 +4341,69 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.3.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -4364,18 +4418,18 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", ] [[package]] name = "value-bag" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" +checksum = "126e423afe2dd9ac52142e7e9d5ce4135d7e13776c529d27fd6bc49f19e3280b" [[package]] name = "vegafusion-common" @@ -4391,7 +4445,7 @@ dependencies = [ "object_store", "pyo3", "serde_json", - "sqlparser 0.37.0", + "sqlparser 0.41.0", "thiserror", ] @@ -4407,7 +4461,7 @@ dependencies = [ "json-patch", "lazy_static", "num-complex", - "ordered-float 3.6.0", + "ordered-float 3.9.2", "petgraph", "prost", "prost-build", @@ -4418,7 +4472,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser 0.41.0", "thiserror", "tonic", "tonic-build", @@ -4445,7 +4499,7 @@ dependencies = [ "chrono-tz", "datafusion-physical-expr", "lazy_static", - "ordered-float 3.6.0", + "ordered-float 3.9.2", "regex", "vegafusion-common", ] @@ -4491,7 +4545,7 @@ dependencies = [ name = "vegafusion-runtime" version = "1.6.4" dependencies = [ - "async-lock", + "async-lock 2.8.0", "async-recursion", "async-trait", "base64", @@ -4501,10 +4555,10 @@ dependencies = [ "criterion", "datafusion-common", "datafusion-expr", + "datafusion-functions", "datafusion-optimizer", "datafusion-physical-expr", "deterministic-hash", - "dssim", "env_logger", "float-cmp", "futures", @@ -4516,7 +4570,8 @@ dependencies = [ "lru", "num-traits", "object_store", - "ordered-float 3.6.0", + "ordered-float 3.9.2", + "pixelmatch", "prost", "prost-types", "regex", @@ -4527,7 +4582,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser 0.41.0", "tempfile", "test-case", "tokio", @@ -4544,7 +4599,7 @@ name = "vegafusion-server" version = "1.6.4" dependencies = [ "assert_cmd", - "clap 4.2.1", + "clap 4.5.1", "futures-util", "h2", "predicates", @@ -4587,7 +4642,7 @@ dependencies = [ "rstest_reuse", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser 0.41.0", "tempfile", "tokio", "toml", @@ -4636,15 +4691,15 @@ dependencies = [ [[package]] name = "waker-fn" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -4652,11 +4707,10 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -4668,9 +4722,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4678,24 +4732,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -4705,9 +4759,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4715,28 +4769,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.50", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "wasm-bindgen-test" -version = "0.3.34" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" +checksum = "143ddeb4f833e2ed0d252e618986e18bfc7b0e52f2d28d77d05b2f045dd8eb61" dependencies = [ "console_error_panic_hook", "js-sys", @@ -4748,19 +4802,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.34" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0734759ae6b3b1717d661fe4f016efcfb9828f5edb4520c18eaee05af3b43be9" +checksum = "a5211b7550606857312bba1d978a8ec75692eae187becc5e680444fffc5e6f89" dependencies = [ "proc-macro2", "quote", + "syn 2.0.50", ] [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -4786,9 +4841,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -4800,15 +4855,22 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.31", ] [[package]] @@ -4829,9 +4891,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -4843,12 +4905,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.52.3", ] [[package]] @@ -4866,7 +4928,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.3", ] [[package]] @@ -4886,17 +4957,32 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +dependencies = [ + "windows_aarch64_gnullvm 0.52.3", + "windows_aarch64_msvc 0.52.3", + "windows_i686_gnu 0.52.3", + "windows_i686_msvc 0.52.3", + "windows_x86_64_gnu 0.52.3", + "windows_x86_64_gnullvm 0.52.3", + "windows_x86_64_msvc 0.52.3", ] [[package]] @@ -4907,9 +4993,15 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" [[package]] name = "windows_aarch64_msvc" @@ -4919,9 +5011,15 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" [[package]] name = "windows_i686_gnu" @@ -4931,9 +5029,15 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" [[package]] name = "windows_i686_msvc" @@ -4943,9 +5047,15 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" [[package]] name = "windows_x86_64_gnu" @@ -4955,9 +5065,15 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" [[package]] name = "windows_x86_64_gnullvm" @@ -4967,9 +5083,15 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" [[package]] name = "windows_x86_64_msvc" @@ -4979,15 +5101,21 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" [[package]] name = "winnow" -version = "0.4.1" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -5011,32 +5139,50 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.50", +] + [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index e69b749f8..0645f107f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,22 +14,37 @@ members = [ ] [workspace.dependencies] -arrow = { version = "47.0.0", default_features = false } -sqlparser = { version = "0.37.0" } +arrow = { version = "50.0.0", default_features = false } +sqlparser = { version = "0.41.0" } chrono = { version = "0.4.31", default_features = false } reqwest = { version = "0.11.22", default-features = false } -tokio = { version = "1.32.0" } -pyo3 = { version = "0.19" } +tokio = { version = "1.36.0" } +pyo3 = { version = "0.20.0" } +pythonize = { version = "0.20.0" } prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } -object_store = { version="0.7" } - -datafusion = { version = "32.0.0" } -datafusion-common = { version = "32.0.0", default_features = false} -datafusion-expr = { version = "32.0.0" } -datafusion-proto = { version = "32.0.0" } -datafusion-physical-expr = { version = "32.0.0" } -datafusion-optimizer = { version = "32.0.0" } +object_store = { version= "0.9.0" } + +[workspace.dependencies.datafusion] +version = "36.0.0" + +[workspace.dependencies.datafusion-common] +version = "36.0.0" + +[workspace.dependencies.datafusion-expr] +version = "36.0.0" + +[workspace.dependencies.datafusion-proto] +version = "36.0.0" + +[workspace.dependencies.datafusion-physical-expr] +version = "36.0.0" + +[workspace.dependencies.datafusion-optimizer] +version = "36.0.0" + +[workspace.dependencies.datafusion-functions] +version = "36.0.0" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/python/vegafusion/proto/datafusion.proto b/python/vegafusion/proto/datafusion.proto index a1caa4c62..7707ed388 100644 --- a/python/vegafusion/proto/datafusion.proto +++ b/python/vegafusion/proto/datafusion.proto @@ -73,6 +73,8 @@ message LogicalPlanNode { CustomTableScanNode custom_scan = 25; PrepareNode prepare = 26; DropViewNode drop_view = 27; + DistinctOnNode distinct_on = 28; + CopyToNode copy_to = 29; } } @@ -88,6 +90,10 @@ message ProjectionColumns { message CsvFormat { bool has_header = 1; string delimiter = 2; + string quote = 3; + oneof optional_escape { + string escape = 4; + } } message ParquetFormat { @@ -176,6 +182,25 @@ message EmptyRelationNode { bool produce_one_row = 1; } +message PrimaryKeyConstraint{ + repeated uint64 indices = 1; +} + +message UniqueConstraint{ + repeated uint64 indices = 1; +} + +message Constraint{ + oneof constraint_mode{ + PrimaryKeyConstraint primary_key = 1; + UniqueConstraint unique = 2; + } +} + +message Constraints{ + repeated Constraint constraints = 1; +} + message CreateExternalTableNode { reserved 1; // was string name OwnedTableReference name = 12; @@ -187,10 +212,13 @@ message CreateExternalTableNode { bool if_not_exists = 7; string delimiter = 8; string definition = 9; - string file_compression_type = 10; + reserved 10; // was string file_compression_type + CompressionTypeVariant file_compression_type = 17; repeated LogicalExprNodeCollection order_exprs = 13; bool unbounded = 14; map options = 11; + Constraints constraints = 15; + map column_defaults = 16; } message PrepareNode { @@ -284,6 +312,32 @@ message DistinctNode { LogicalPlanNode input = 1; } +message DistinctOnNode { + repeated LogicalExprNode on_expr = 1; + repeated LogicalExprNode select_expr = 2; + repeated LogicalExprNode sort_expr = 3; + LogicalPlanNode input = 4; +} + +message CopyToNode { + LogicalPlanNode input = 1; + string output_url = 2; + oneof CopyOptions { + SQLOptions sql_options = 4; + FileTypeWriterOptions writer_options = 5; + } + string file_type = 6; +} + +message SQLOptions { + repeated SQLOption option = 1; +} + +message SQLOption { + string key = 1; + string value = 2; +} + message UnionNode { repeated LogicalPlanNode inputs = 1; } @@ -339,7 +393,7 @@ message LogicalExprNode { SortExprNode sort = 12; NegativeNode negative = 13; InListNode in_list = 14; - bool wildcard = 15; + Wildcard wildcard = 15; ScalarFunctionNode scalar_function = 16; TryCastNode try_cast = 17; @@ -372,9 +426,15 @@ message LogicalExprNode { PlaceholderNode placeholder = 34; + Unnest unnest = 35; + } } +message Wildcard { + string qualifier = 1; +} + message PlaceholderNode { string id = 1; ArrowType data_type = 2; @@ -396,11 +456,27 @@ message RollupNode { repeated LogicalExprNode expr = 1; } +message NamedStructField { + ScalarValue name = 1; +} + +message ListIndex { + LogicalExprNode key = 1; +} +message ListRange { + LogicalExprNode start = 1; + LogicalExprNode stop = 2; + LogicalExprNode stride = 3; +} message GetIndexedField { LogicalExprNode expr = 1; - ScalarValue key = 2; + oneof field { + NamedStructField named_struct_field = 2; + ListIndex list_index = 3; + ListRange list_range = 4; + } } message IsNull { @@ -442,6 +518,7 @@ message Not { message AliasNode { LogicalExprNode expr = 1; string alias = 2; + repeated OwnedTableReference relation = 3; } message BinaryExprNode { @@ -456,6 +533,10 @@ message NegativeNode { LogicalExprNode expr = 1; } +message Unnest { + repeated LogicalExprNode exprs = 1; +} + message InListNode { LogicalExprNode expr = 1; repeated LogicalExprNode list = 2; @@ -499,7 +580,7 @@ enum ScalarFunction { Lower = 33; Ltrim = 34; MD5 = 35; - NullIf = 36; + // 36 was NullIf OctetLength = 37; Random = 38; RegexpReplace = 39; @@ -552,7 +633,7 @@ enum ScalarFunction { ArrayAppend = 86; ArrayConcat = 87; ArrayDims = 88; - ArrayFill = 89; + ArrayRepeat = 89; ArrayLength = 90; ArrayNdims = 91; ArrayPosition = 92; @@ -560,13 +641,44 @@ enum ScalarFunction { ArrayPrepend = 94; ArrayRemove = 95; ArrayReplace = 96; - ArrayToString = 97; + // 97 was ArrayToString Cardinality = 98; - TrimArray = 99; - ArrayContains = 100; - Encode = 101; - Decode = 102; + ArrayElement = 99; + ArraySlice = 100; Cot = 103; + ArrayHas = 104; + ArrayHasAny = 105; + ArrayHasAll = 106; + ArrayRemoveN = 107; + ArrayReplaceN = 108; + ArrayRemoveAll = 109; + ArrayReplaceAll = 110; + Nanvl = 111; + Flatten = 112; + // 113 was IsNan + Iszero = 114; + ArrayEmpty = 115; + ArrayPopBack = 116; + StringToArray = 117; + ToTimestampNanos = 118; + ArrayIntersect = 119; + ArrayUnion = 120; + OverLay = 121; + Range = 122; + ArrayExcept = 123; + ArrayPopFront = 124; + Levenshtein = 125; + SubstrIndex = 126; + FindInSet = 127; + ArraySort = 128; + ArrayDistinct = 129; + ArrayResize = 130; + EndsWith = 131; + InStr = 132; + MakeDate = 133; + ArrayReverse = 134; + RegexpLike = 135; + ToChar = 136; } message ScalarFunctionNode { @@ -603,6 +715,17 @@ enum AggregateFunction { // we append "_AGG" to obey name scoping rules. FIRST_VALUE_AGG = 24; LAST_VALUE_AGG = 25; + REGR_SLOPE = 26; + REGR_INTERCEPT = 27; + REGR_COUNT = 28; + REGR_R2 = 29; + REGR_AVGX = 30; + REGR_AVGY = 31; + REGR_SXX = 32; + REGR_SYY = 33; + REGR_SXY = 34; + STRING_AGG = 35; + NTH_VALUE_AGG = 36; } message AggregateExprNode { @@ -752,6 +875,8 @@ message Field { // for complex data types like structs, unions repeated Field children = 4; map metadata = 5; + int64 dict_id = 6; + bool dict_ordered = 7; } message FixedSizeBinary{ @@ -821,12 +946,11 @@ message Union{ repeated int32 type_ids = 3; } -message ScalarListValue{ - // encode null explicitly to distinguish a list with a null value - // from a list with no values) - bool is_null = 3; - Field field = 1; - repeated ScalarValue values = 2; +// Used for List/FixedSizeList/LargeList/Struct +message ScalarNestedValue { + bytes ipc_message = 1; + bytes arrow_data = 2; + Schema schema = 3; } message ScalarTime32Value { @@ -864,14 +988,6 @@ message IntervalMonthDayNanoValue { int64 nanos = 3; } -message StructValue { - // Note that a null struct value must have one or more fields, so we - // encode a null StructValue as one witth an empty field_values - // list. - repeated ScalarValue field_values = 2; - repeated Field fields = 3; -} - message ScalarFixedSizeBinary{ bytes values = 1; int32 length = 2; @@ -902,10 +1018,14 @@ message ScalarValue{ // Literal Date32 value always has a unit of day int32 date_32_value = 14; ScalarTime32Value time32_value = 15; - ScalarListValue list_value = 17; - //WAS: ScalarType null_list_value = 18; + ScalarNestedValue large_list_value = 16; + ScalarNestedValue list_value = 17; + ScalarNestedValue fixed_size_list_value = 18; + ScalarNestedValue struct_value = 32; Decimal128 decimal128_value = 20; + Decimal256 decimal256_value = 39; + int64 date_64_value = 21; int32 interval_yearmonth_value = 24; int64 interval_daytime_value = 25; @@ -921,7 +1041,6 @@ message ScalarValue{ bytes large_binary_value = 29; ScalarTime64Value time64_value = 30; IntervalMonthDayNanoValue interval_month_day_nano = 31; - StructValue struct_value = 32; ScalarFixedSizeBinary fixed_size_binary_value = 34; } } @@ -932,6 +1051,12 @@ message Decimal128{ int64 s = 3; } +message Decimal256{ + bytes value = 1; + int64 p = 2; + int64 s = 3; +} + // Serialized data type message ArrowType{ oneof arrow_type_enum { @@ -1002,8 +1127,10 @@ message PlanType { OptimizedLogicalPlanType OptimizedLogicalPlan = 2; EmptyMessage FinalLogicalPlan = 3; EmptyMessage InitialPhysicalPlan = 4; + EmptyMessage InitialPhysicalPlanWithStats = 9; OptimizedPhysicalPlanType OptimizedPhysicalPlan = 5; EmptyMessage FinalPhysicalPlan = 6; + EmptyMessage FinalPhysicalPlanWithStats = 10; } } @@ -1061,9 +1188,122 @@ message PhysicalPlanNode { ExplainExecNode explain = 20; SortPreservingMergeExecNode sort_preserving_merge = 21; NestedLoopJoinExecNode nested_loop_join = 22; + AnalyzeExecNode analyze = 23; + JsonSinkExecNode json_sink = 24; + SymmetricHashJoinExecNode symmetric_hash_join = 25; + InterleaveExecNode interleave = 26; + PlaceholderRowExecNode placeholder_row = 27; + CsvSinkExecNode csv_sink = 28; + ParquetSinkExecNode parquet_sink = 29; } } +enum CompressionTypeVariant { + GZIP = 0; + BZIP2 = 1; + XZ = 2; + ZSTD = 3; + UNCOMPRESSED = 4; +} + +message PartitionColumn { + string name = 1; + ArrowType arrow_type = 2; +} + +message FileTypeWriterOptions { + oneof FileType { + JsonWriterOptions json_options = 1; + ParquetWriterOptions parquet_options = 2; + CsvWriterOptions csv_options = 3; + ArrowWriterOptions arrow_options = 4; + } +} + +message JsonWriterOptions { + CompressionTypeVariant compression = 1; +} + +message ParquetWriterOptions { + WriterProperties writer_properties = 1; +} + +message CsvWriterOptions { + // Compression type + CompressionTypeVariant compression = 1; + // Optional column delimiter. Defaults to `b','` + string delimiter = 2; + // Whether to write column names as file headers. Defaults to `true` + bool has_header = 3; + // Optional date format for date arrays + string date_format = 4; + // Optional datetime format for datetime arrays + string datetime_format = 5; + // Optional timestamp format for timestamp arrays + string timestamp_format = 6; + // Optional time format for time arrays + string time_format = 7; + // Optional value to represent null + string null_value = 8; +} + +message ArrowWriterOptions {} + +message WriterProperties { + uint64 data_page_size_limit = 1; + uint64 dictionary_page_size_limit = 2; + uint64 data_page_row_count_limit = 3; + uint64 write_batch_size = 4; + uint64 max_row_group_size = 5; + string writer_version = 6; + string created_by = 7; +} + +message FileSinkConfig { + reserved 6; // writer_mode + + string object_store_url = 1; + repeated PartitionedFile file_groups = 2; + repeated string table_paths = 3; + Schema output_schema = 4; + repeated PartitionColumn table_partition_cols = 5; + bool overwrite = 8; + FileTypeWriterOptions file_type_writer_options = 9; +} + +message JsonSink { + FileSinkConfig config = 1; +} + +message JsonSinkExecNode { + PhysicalPlanNode input = 1; + JsonSink sink = 2; + Schema sink_schema = 3; + PhysicalSortExprNodeCollection sort_order = 4; +} + +message CsvSink { + FileSinkConfig config = 1; +} + +message CsvSinkExecNode { + PhysicalPlanNode input = 1; + CsvSink sink = 2; + Schema sink_schema = 3; + PhysicalSortExprNodeCollection sort_order = 4; +} + +message ParquetSink { + FileSinkConfig config = 1; +} + +message ParquetSinkExecNode { + PhysicalPlanNode input = 1; + ParquetSink sink = 2; + Schema sink_schema = 3; + PhysicalSortExprNodeCollection sort_order = 4; +} + message PhysicalExtensionNode { bytes node = 1; repeated PhysicalPlanNode inputs = 2; @@ -1071,6 +1311,9 @@ message PhysicalExtensionNode { // physical expressions message PhysicalExprNode { + // Was date_time_interval_expr + reserved 17; + oneof ExprType { // column references PhysicalColumn column = 1; @@ -1101,8 +1344,6 @@ message PhysicalExprNode { PhysicalScalarUdfNode scalar_udf = 16; - PhysicalDateTimeIntervalExprNode date_time_interval_expr = 17; - PhysicalLikeExprNode like_expr = 18; PhysicalGetIndexedFieldExprNode get_indexed_field_expr = 19; @@ -1131,7 +1372,11 @@ message PhysicalWindowExprNode { BuiltInWindowFunction built_in_function = 2; // udaf = 3 } - PhysicalExprNode expr = 4; + repeated PhysicalExprNode args = 4; + repeated PhysicalExprNode partition_by = 5; + repeated PhysicalSortExprNode order_by = 6; + WindowFrame window_frame = 7; + string name = 8; } message PhysicalIsNull { @@ -1217,6 +1462,7 @@ message PhysicalNegativeNode { message FilterExecNode { PhysicalPlanNode input = 1; PhysicalExprNode expr = 2; + uint32 default_filter_selectivity = 3; } message FileGroup { @@ -1259,6 +1505,10 @@ message CsvScanExecNode { FileScanExecConf base_conf = 1; bool has_header = 2; string delimiter = 3; + string quote = 4; + oneof optional_escape { + string escape = 5; + } } message AvroScanExecNode { @@ -1281,6 +1531,27 @@ message HashJoinExecNode { JoinFilter filter = 8; } +enum StreamPartitionMode { + SINGLE_PARTITION = 0; + PARTITIONED_EXEC = 1; +} + +message SymmetricHashJoinExecNode { + PhysicalPlanNode left = 1; + PhysicalPlanNode right = 2; + repeated JoinOn on = 3; + JoinType join_type = 4; + StreamPartitionMode partition_mode = 6; + bool null_equals_null = 7; + JoinFilter filter = 8; + repeated PhysicalSortExprNode left_sort_exprs = 9; + repeated PhysicalSortExprNode right_sort_exprs = 10; +} + +message InterleaveExecNode { + repeated PhysicalPlanNode inputs = 1; +} + message UnionExecNode { repeated PhysicalPlanNode inputs = 1; } @@ -1291,6 +1562,13 @@ message ExplainExecNode { bool verbose = 3; } +message AnalyzeExecNode { + bool verbose = 1; + bool show_statistics = 2; + PhysicalPlanNode input = 3; + Schema schema = 4; +} + message CrossJoinExecNode { PhysicalPlanNode left = 1; PhysicalPlanNode right = 2; @@ -1302,13 +1580,16 @@ message PhysicalColumn { } message JoinOn { - PhysicalColumn left = 1; - PhysicalColumn right = 2; + PhysicalExprNode left = 1; + PhysicalExprNode right = 2; } message EmptyExecNode { - bool produce_one_row = 1; - Schema schema = 2; + Schema schema = 1; +} + +message PlaceholderRowExecNode { + Schema schema = 1; } message ProjectionExecNode { @@ -1325,11 +1606,20 @@ enum AggregateMode { SINGLE_PARTITIONED = 4; } +message PartiallySortedInputOrderMode { + repeated uint64 columns = 6; +} + message WindowAggExecNode { PhysicalPlanNode input = 1; - repeated PhysicalExprNode window_expr = 2; - repeated string window_expr_name = 3; - Schema input_schema = 4; + repeated PhysicalWindowExprNode window_expr = 2; + repeated PhysicalExprNode partition_keys = 5; + // Set optional to `None` for `BoundedWindowAggExec`. + oneof input_order_mode { + EmptyMessage linear = 7; + PartiallySortedInputOrderMode partially_sorted = 8; + EmptyMessage sorted = 9; + } } message MaybeFilter { @@ -1352,7 +1642,6 @@ message AggregateExecNode { repeated PhysicalExprNode null_expr = 8; repeated bool groups = 9; repeated MaybeFilter filter_expr = 10; - repeated MaybePhysicalSortExprs order_by_expr = 11; } message GlobalLimitExecNode { @@ -1449,21 +1738,49 @@ message PartitionStats { repeated ColumnStats column_stats = 4; } +message Precision{ + PrecisionInfo precision_info = 1; + ScalarValue val = 2; +} + +enum PrecisionInfo { + EXACT = 0; + INEXACT = 1; + ABSENT = 2; +} + message Statistics { - int64 num_rows = 1; - int64 total_byte_size = 2; + Precision num_rows = 1; + Precision total_byte_size = 2; repeated ColumnStats column_stats = 3; - bool is_exact = 4; } message ColumnStats { - ScalarValue min_value = 1; - ScalarValue max_value = 2; - uint32 null_count = 3; - uint32 distinct_count = 4; + Precision min_value = 1; + Precision max_value = 2; + Precision null_count = 3; + Precision distinct_count = 4; +} + +message NamedStructFieldExpr { + ScalarValue name = 1; +} + +message ListIndexExpr { + PhysicalExprNode key = 1; +} + +message ListRangeExpr { + PhysicalExprNode start = 1; + PhysicalExprNode stop = 2; + PhysicalExprNode stride = 3; } message PhysicalGetIndexedFieldExprNode { PhysicalExprNode arg = 1; - ScalarValue key = 2; -} + oneof field { + NamedStructFieldExpr named_struct_field_expr = 2; + ListIndexExpr list_index_expr = 3; + ListRangeExpr list_range_expr = 4; + } +} \ No newline at end of file diff --git a/python/vegafusion/vegafusion/proto/datafusion_pb2.py b/python/vegafusion/vegafusion/proto/datafusion_pb2.py index 6cd5755e2..530601492 100644 --- a/python/vegafusion/vegafusion/proto/datafusion_pb2.py +++ b/python/vegafusion/vegafusion/proto/datafusion_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16proto/datafusion.proto\x12\ndatafusion\"\"\n\x0e\x43olumnRelation\x12\x10\n\x08relation\x18\x01 \x01(\t\"D\n\x06\x43olumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\x08relation\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"Z\n\x07\x44\x66\x46ield\x12 \n\x05\x66ield\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12-\n\tqualifier\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"\x97\x01\n\x08\x44\x66Schema\x12$\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x13.datafusion.DfField\x12\x34\n\x08metadata\x18\x02 \x03(\x0b\x32\".datafusion.DfSchema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb3\n\n\x0fLogicalPlanNode\x12\x38\n\x0clisting_scan\x18\x01 \x01(\x0b\x32 .datafusion.ListingTableScanNodeH\x00\x12\x30\n\nprojection\x18\x03 \x01(\x0b\x32\x1a.datafusion.ProjectionNodeH\x00\x12.\n\tselection\x18\x04 \x01(\x0b\x32\x19.datafusion.SelectionNodeH\x00\x12&\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.LimitNodeH\x00\x12.\n\taggregate\x18\x06 \x01(\x0b\x32\x19.datafusion.AggregateNodeH\x00\x12$\n\x04join\x18\x07 \x01(\x0b\x32\x14.datafusion.JoinNodeH\x00\x12$\n\x04sort\x18\x08 \x01(\x0b\x32\x14.datafusion.SortNodeH\x00\x12\x32\n\x0brepartition\x18\t \x01(\x0b\x32\x1b.datafusion.RepartitionNodeH\x00\x12\x37\n\x0e\x65mpty_relation\x18\n \x01(\x0b\x32\x1d.datafusion.EmptyRelationNodeH\x00\x12\x44\n\x15\x63reate_external_table\x18\x0b \x01(\x0b\x32#.datafusion.CreateExternalTableNodeH\x00\x12*\n\x07\x65xplain\x18\x0c \x01(\x0b\x32\x17.datafusion.ExplainNodeH\x00\x12(\n\x06window\x18\r \x01(\x0b\x32\x16.datafusion.WindowNodeH\x00\x12*\n\x07\x61nalyze\x18\x0e \x01(\x0b\x32\x17.datafusion.AnalyzeNodeH\x00\x12/\n\ncross_join\x18\x0f \x01(\x0b\x32\x19.datafusion.CrossJoinNodeH\x00\x12(\n\x06values\x18\x10 \x01(\x0b\x32\x16.datafusion.ValuesNodeH\x00\x12\x35\n\textension\x18\x11 \x01(\x0b\x32 .datafusion.LogicalExtensionNodeH\x00\x12\x44\n\x15\x63reate_catalog_schema\x18\x12 \x01(\x0b\x32#.datafusion.CreateCatalogSchemaNodeH\x00\x12&\n\x05union\x18\x13 \x01(\x0b\x32\x15.datafusion.UnionNodeH\x00\x12\x37\n\x0e\x63reate_catalog\x18\x14 \x01(\x0b\x32\x1d.datafusion.CreateCatalogNodeH\x00\x12\x37\n\x0esubquery_alias\x18\x15 \x01(\x0b\x32\x1d.datafusion.SubqueryAliasNodeH\x00\x12\x31\n\x0b\x63reate_view\x18\x16 \x01(\x0b\x32\x1a.datafusion.CreateViewNodeH\x00\x12,\n\x08\x64istinct\x18\x17 \x01(\x0b\x32\x18.datafusion.DistinctNodeH\x00\x12\x32\n\tview_scan\x18\x18 \x01(\x0b\x32\x1d.datafusion.ViewTableScanNodeH\x00\x12\x36\n\x0b\x63ustom_scan\x18\x19 \x01(\x0b\x32\x1f.datafusion.CustomTableScanNodeH\x00\x12*\n\x07prepare\x18\x1a \x01(\x0b\x32\x17.datafusion.PrepareNodeH\x00\x12-\n\tdrop_view\x18\x1b \x01(\x0b\x32\x18.datafusion.DropViewNodeH\x00\x42\x11\n\x0fLogicalPlanType\"Q\n\x14LogicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"$\n\x11ProjectionColumns\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\"2\n\tCsvFormat\x12\x12\n\nhas_header\x18\x01 \x01(\x08\x12\x11\n\tdelimiter\x18\x02 \x01(\t\"\x15\n\rParquetFormatJ\x04\x08\x01\x10\x02\"\x0c\n\nAvroFormat\"T\n\x19LogicalExprNodeCollection\x12\x37\n\x12logical_expr_nodes\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x9a\x04\n\x14ListingTableScanNode\x12\x33\n\ntable_name\x18\x0e \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\r\n\x05paths\x18\x02 \x03(\t\x12\x16\n\x0e\x66ile_extension\x18\x03 \x01(\t\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x05 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x14\n\x0c\x63ollect_stat\x18\x08 \x01(\x08\x12\x19\n\x11target_partitions\x18\t \x01(\r\x12$\n\x03\x63sv\x18\n \x01(\x0b\x32\x15.datafusion.CsvFormatH\x00\x12,\n\x07parquet\x18\x0b \x01(\x0b\x32\x19.datafusion.ParquetFormatH\x00\x12&\n\x04\x61vro\x18\x0c \x01(\x0b\x32\x16.datafusion.AvroFormatH\x00\x12>\n\x0f\x66ile_sort_order\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollectionB\x10\n\x0e\x46ileFormatTypeJ\x04\x08\x01\x10\x02\"\xe5\x01\n\x11ViewTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\x12\n\ndefinition\x18\x05 \x01(\tJ\x04\x08\x01\x10\x02\"\xf0\x01\n\x13\x43ustomTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x31\n\nprojection\x18\x02 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x19\n\x11\x63ustom_table_data\x18\x05 \x01(\x0cJ\x04\x08\x01\x10\x02\"\x8a\x01\n\x0eProjectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x05\x61lias\x18\x03 \x01(\tH\x00\x42\x10\n\x0eoptional_alias\"f\n\rSelectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"p\n\x08SortNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\x95\x01\n\x0fRepartitionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12+\n\x04hash\x18\x03 \x01(\x0b\x32\x1b.datafusion.HashRepartitionH\x00\x42\x12\n\x10partition_method\"Z\n\x0fHashRepartition\x12.\n\thash_expr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\",\n\x11\x45mptyRelationNode\x12\x17\n\x0fproduce_one_row\x18\x01 \x01(\x08\"\xea\x03\n\x17\x43reateExternalTableNode\x12-\n\x04name\x18\x0c \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x10\n\x08location\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x12\n\nhas_header\x18\x04 \x01(\x08\x12$\n\x06schema\x18\x05 \x01(\x0b\x32\x14.datafusion.DfSchema\x12\x1c\n\x14table_partition_cols\x18\x06 \x03(\t\x12\x15\n\rif_not_exists\x18\x07 \x01(\x08\x12\x11\n\tdelimiter\x18\x08 \x01(\t\x12\x12\n\ndefinition\x18\t \x01(\t\x12\x1d\n\x15\x66ile_compression_type\x18\n \x01(\t\x12:\n\x0border_exprs\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollection\x12\x11\n\tunbounded\x18\x0e \x01(\x08\x12\x41\n\x07options\x18\x0b \x03(\x0b\x32\x30.datafusion.CreateExternalTableNode.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x01\x10\x02\"r\n\x0bPrepareNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\ndata_types\x18\x02 \x03(\x0b\x32\x15.datafusion.ArrowType\x12*\n\x05input\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"k\n\x17\x43reateCatalogSchemaNode\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"f\n\x11\x43reateCatalogNode\x12\x14\n\x0c\x63\x61talog_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"v\n\x0c\x44ropViewNode\x12-\n\x04name\x18\x01 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x11\n\tif_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"\x99\x01\n\x0e\x43reateViewNode\x12-\n\x04name\x18\x05 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x12\n\nor_replace\x18\x03 \x01(\x08\x12\x12\n\ndefinition\x18\x04 \x01(\tJ\x04\x08\x01\x10\x02\"N\n\nValuesNode\x12\x0e\n\x06n_cols\x18\x01 \x01(\x04\x12\x30\n\x0bvalues_list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"J\n\x0b\x41nalyzeNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"J\n\x0b\x45xplainNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"\x9c\x01\n\rAggregateNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12/\n\ngroup_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\taggr_expr\x18\x03 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\nWindowNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x30\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xef\x02\n\x08JoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x33\n\x0fjoin_constraint\x18\x04 \x01(\x0e\x32\x1a.datafusion.JoinConstraint\x12\x32\n\rleft_join_key\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x33\n\x0eright_join_key\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12+\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\":\n\x0c\x44istinctNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"8\n\tUnionNode\x12+\n\x06inputs\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"f\n\rCrossJoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"T\n\tLimitNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\x03\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\">\n\x11SelectionExecNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"u\n\x11SubqueryAliasNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12.\n\x05\x61lias\x18\x03 \x01(\x0b\x32\x1f.datafusion.OwnedTableReferenceJ\x04\x08\x02\x10\x03\"\xad\x0c\n\x0fLogicalExprNode\x12$\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x12.datafusion.ColumnH\x00\x12&\n\x05\x61lias\x18\x02 \x01(\x0b\x32\x15.datafusion.AliasNodeH\x00\x12*\n\x07literal\x18\x03 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x31\n\x0b\x62inary_expr\x18\x04 \x01(\x0b\x32\x1a.datafusion.BinaryExprNodeH\x00\x12\x37\n\x0e\x61ggregate_expr\x18\x05 \x01(\x0b\x32\x1d.datafusion.AggregateExprNodeH\x00\x12*\n\x0cis_null_expr\x18\x06 \x01(\x0b\x32\x12.datafusion.IsNullH\x00\x12\x31\n\x10is_not_null_expr\x18\x07 \x01(\x0b\x32\x15.datafusion.IsNotNullH\x00\x12#\n\x08not_expr\x18\x08 \x01(\x0b\x32\x0f.datafusion.NotH\x00\x12*\n\x07\x62\x65tween\x18\t \x01(\x0b\x32\x17.datafusion.BetweenNodeH\x00\x12%\n\x05\x63\x61se_\x18\n \x01(\x0b\x32\x14.datafusion.CaseNodeH\x00\x12$\n\x04\x63\x61st\x18\x0b \x01(\x0b\x32\x14.datafusion.CastNodeH\x00\x12(\n\x04sort\x18\x0c \x01(\x0b\x32\x18.datafusion.SortExprNodeH\x00\x12,\n\x08negative\x18\r \x01(\x0b\x32\x18.datafusion.NegativeNodeH\x00\x12)\n\x07in_list\x18\x0e \x01(\x0b\x32\x16.datafusion.InListNodeH\x00\x12\x12\n\x08wildcard\x18\x0f \x01(\x08H\x00\x12\x39\n\x0fscalar_function\x18\x10 \x01(\x0b\x32\x1e.datafusion.ScalarFunctionNodeH\x00\x12+\n\x08try_cast\x18\x11 \x01(\x0b\x32\x17.datafusion.TryCastNodeH\x00\x12\x31\n\x0bwindow_expr\x18\x12 \x01(\x0b\x32\x1a.datafusion.WindowExprNodeH\x00\x12>\n\x12\x61ggregate_udf_expr\x18\x13 \x01(\x0b\x32 .datafusion.AggregateUDFExprNodeH\x00\x12\x38\n\x0fscalar_udf_expr\x18\x14 \x01(\x0b\x32\x1d.datafusion.ScalarUDFExprNodeH\x00\x12\x38\n\x11get_indexed_field\x18\x15 \x01(\x0b\x32\x1b.datafusion.GetIndexedFieldH\x00\x12\x33\n\x0cgrouping_set\x18\x16 \x01(\x0b\x32\x1b.datafusion.GroupingSetNodeH\x00\x12$\n\x04\x63ube\x18\x17 \x01(\x0b\x32\x14.datafusion.CubeNodeH\x00\x12(\n\x06rollup\x18\x18 \x01(\x0b\x32\x16.datafusion.RollupNodeH\x00\x12%\n\x07is_true\x18\x19 \x01(\x0b\x32\x12.datafusion.IsTrueH\x00\x12\'\n\x08is_false\x18\x1a \x01(\x0b\x32\x13.datafusion.IsFalseH\x00\x12+\n\nis_unknown\x18\x1b \x01(\x0b\x32\x15.datafusion.IsUnknownH\x00\x12,\n\x0bis_not_true\x18\x1c \x01(\x0b\x32\x15.datafusion.IsNotTrueH\x00\x12.\n\x0cis_not_false\x18\x1d \x01(\x0b\x32\x16.datafusion.IsNotFalseH\x00\x12\x32\n\x0eis_not_unknown\x18\x1e \x01(\x0b\x32\x18.datafusion.IsNotUnknownH\x00\x12$\n\x04like\x18\x1f \x01(\x0b\x32\x14.datafusion.LikeNodeH\x00\x12&\n\x05ilike\x18 \x01(\x0b\x32\x15.datafusion.ILikeNodeH\x00\x12/\n\nsimilar_to\x18! \x01(\x0b\x32\x19.datafusion.SimilarToNodeH\x00\x12\x32\n\x0bplaceholder\x18\" \x01(\x0b\x32\x1b.datafusion.PlaceholderNodeH\x00\x42\n\n\x08\x45xprType\"G\n\x0fPlaceholderNode\x12\n\n\x02id\x18\x01 \x01(\t\x12(\n\tdata_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"<\n\x0fLogicalExprList\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"<\n\x0fGroupingSetNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprList\"5\n\x08\x43ubeNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nRollupNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"b\n\x0fGetIndexedField\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12$\n\x03key\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"3\n\x06IsNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"3\n\x06IsTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"4\n\x07IsFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nIsNotFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"9\n\x0cIsNotUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"0\n\x03Not\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"E\n\tAliasNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"K\n\x0e\x42inaryExprNode\x12-\n\x08operands\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"9\n\x0cNegativeNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"s\n\nInListNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"h\n\x12ScalarFunctionNode\x12\'\n\x03\x66un\x18\x01 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xe2\x01\n\x11\x41ggregateExprNode\x12\x34\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunction\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x12+\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xaf\x01\n\x14\x41ggregateUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12+\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"P\n\x11ScalarUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xf7\x02\n\x0eWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12\x0e\n\x04udaf\x18\x03 \x01(\tH\x00\x12\x0e\n\x04udwf\x18\t \x01(\tH\x00\x12)\n\x04\x65xpr\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x31\n\x0cpartition_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x0cwindow_frame\x18\x08 \x01(\x0b\x32\x17.datafusion.WindowFrameB\x11\n\x0fwindow_function\"\x9e\x01\n\x0b\x42\x65tweenNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x02 \x01(\x08\x12(\n\x03low\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04high\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x89\x01\n\x08LikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8a\x01\n\tILikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8e\x01\n\rSimilarToNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x93\x01\n\x08\x43\x61seNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x14.datafusion.WhenThen\x12.\n\telse_expr\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\x08WhenThen\x12.\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"`\n\x08\x43\x61stNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"c\n\x0bTryCastNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"[\n\x0cSortExprNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"\xb6\x01\n\x0bWindowFrame\x12\x38\n\x12window_frame_units\x18\x01 \x01(\x0e\x32\x1c.datafusion.WindowFrameUnits\x12\x31\n\x0bstart_bound\x18\x02 \x01(\x0b\x32\x1c.datafusion.WindowFrameBound\x12-\n\x05\x62ound\x18\x03 \x01(\x0b\x32\x1c.datafusion.WindowFrameBoundH\x00\x42\x0b\n\tend_bound\"\x83\x01\n\x10WindowFrameBound\x12\x41\n\x17window_frame_bound_type\x18\x01 \x01(\x0e\x32 .datafusion.WindowFrameBoundType\x12,\n\x0b\x62ound_value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"\x91\x01\n\x06Schema\x12\"\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12\x32\n\x08metadata\x18\x02 \x03(\x0b\x32 .datafusion.Schema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xdb\x01\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\x12\x10\n\x08nullable\x18\x03 \x01(\x08\x12#\n\x08\x63hildren\x18\x04 \x03(\x0b\x32\x11.datafusion.Field\x12\x31\n\x08metadata\x18\x05 \x03(\x0b\x32\x1f.datafusion.Field.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"!\n\x0f\x46ixedSizeBinary\x12\x0e\n\x06length\x18\x01 \x01(\x05\"F\n\tTimestamp\x12\'\n\ttime_unit\x18\x01 \x01(\x0e\x32\x14.datafusion.TimeUnit\x12\x10\n\x08timezone\x18\x02 \x01(\t\"7\n\x07\x44\x65\x63imal\x12\x11\n\tprecision\x18\x03 \x01(\r\x12\r\n\x05scale\x18\x04 \x01(\x05J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03\"-\n\x04List\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\"I\n\rFixedSizeList\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x11\n\tlist_size\x18\x02 \x01(\x05\"V\n\nDictionary\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"4\n\x06Struct\x12*\n\x0fsub_field_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\"A\n\x03Map\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x13\n\x0bkeys_sorted\x18\x02 \x01(\x08\"l\n\x05Union\x12&\n\x0bunion_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12)\n\nunion_mode\x18\x02 \x01(\x0e\x32\x15.datafusion.UnionMode\x12\x10\n\x08type_ids\x18\x03 \x03(\x05\"m\n\x0fScalarListValue\x12\x0f\n\x07is_null\x18\x03 \x01(\x08\x12 \n\x05\x66ield\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\'\n\x06values\x18\x02 \x03(\x0b\x32\x17.datafusion.ScalarValue\"_\n\x11ScalarTime32Value\x12\x1d\n\x13time32_second_value\x18\x01 \x01(\x05H\x00\x12\"\n\x18time32_millisecond_value\x18\x02 \x01(\x05H\x00\x42\x07\n\x05value\"c\n\x11ScalarTime64Value\x12\"\n\x18time64_microsecond_value\x18\x01 \x01(\x03H\x00\x12!\n\x17time64_nanosecond_value\x18\x02 \x01(\x03H\x00\x42\x07\n\x05value\"\xb3\x01\n\x14ScalarTimestampValue\x12 \n\x16time_microsecond_value\x18\x01 \x01(\x03H\x00\x12\x1f\n\x15time_nanosecond_value\x18\x02 \x01(\x03H\x00\x12\x1b\n\x11time_second_value\x18\x03 \x01(\x03H\x00\x12 \n\x16time_millisecond_value\x18\x04 \x01(\x03H\x00\x12\x10\n\x08timezone\x18\x05 \x01(\tB\x07\n\x05value\"j\n\x15ScalarDictionaryValue\x12)\n\nindex_type\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"H\n\x19IntervalMonthDayNanoValue\x12\x0e\n\x06months\x18\x01 \x01(\x05\x12\x0c\n\x04\x64\x61ys\x18\x02 \x01(\x05\x12\r\n\x05nanos\x18\x03 \x01(\x03\"_\n\x0bStructValue\x12-\n\x0c\x66ield_values\x18\x02 \x03(\x0b\x32\x17.datafusion.ScalarValue\x12!\n\x06\x66ields\x18\x03 \x03(\x0b\x32\x11.datafusion.Field\"7\n\x15ScalarFixedSizeBinary\x12\x0e\n\x06values\x18\x01 \x01(\x0c\x12\x0e\n\x06length\x18\x02 \x01(\x05\"\xcf\t\n\x0bScalarValue\x12+\n\nnull_value\x18! \x01(\x0b\x32\x15.datafusion.ArrowTypeH\x00\x12\x14\n\nbool_value\x18\x01 \x01(\x08H\x00\x12\x14\n\nutf8_value\x18\x02 \x01(\tH\x00\x12\x1a\n\x10large_utf8_value\x18\x03 \x01(\tH\x00\x12\x14\n\nint8_value\x18\x04 \x01(\x05H\x00\x12\x15\n\x0bint16_value\x18\x05 \x01(\x05H\x00\x12\x15\n\x0bint32_value\x18\x06 \x01(\x05H\x00\x12\x15\n\x0bint64_value\x18\x07 \x01(\x03H\x00\x12\x15\n\x0buint8_value\x18\x08 \x01(\rH\x00\x12\x16\n\x0cuint16_value\x18\t \x01(\rH\x00\x12\x16\n\x0cuint32_value\x18\n \x01(\rH\x00\x12\x16\n\x0cuint64_value\x18\x0b \x01(\x04H\x00\x12\x17\n\rfloat32_value\x18\x0c \x01(\x02H\x00\x12\x17\n\rfloat64_value\x18\r \x01(\x01H\x00\x12\x17\n\rdate_32_value\x18\x0e \x01(\x05H\x00\x12\x35\n\x0ctime32_value\x18\x0f \x01(\x0b\x32\x1d.datafusion.ScalarTime32ValueH\x00\x12\x31\n\nlist_value\x18\x11 \x01(\x0b\x32\x1b.datafusion.ScalarListValueH\x00\x12\x32\n\x10\x64\x65\x63imal128_value\x18\x14 \x01(\x0b\x32\x16.datafusion.Decimal128H\x00\x12\x17\n\rdate_64_value\x18\x15 \x01(\x03H\x00\x12\"\n\x18interval_yearmonth_value\x18\x18 \x01(\x05H\x00\x12 \n\x16interval_daytime_value\x18\x19 \x01(\x03H\x00\x12\x1f\n\x15\x64uration_second_value\x18# \x01(\x03H\x00\x12$\n\x1a\x64uration_millisecond_value\x18$ \x01(\x03H\x00\x12$\n\x1a\x64uration_microsecond_value\x18% \x01(\x03H\x00\x12#\n\x19\x64uration_nanosecond_value\x18& \x01(\x03H\x00\x12;\n\x0ftimestamp_value\x18\x1a \x01(\x0b\x32 .datafusion.ScalarTimestampValueH\x00\x12=\n\x10\x64ictionary_value\x18\x1b \x01(\x0b\x32!.datafusion.ScalarDictionaryValueH\x00\x12\x16\n\x0c\x62inary_value\x18\x1c \x01(\x0cH\x00\x12\x1c\n\x12large_binary_value\x18\x1d \x01(\x0cH\x00\x12\x35\n\x0ctime64_value\x18\x1e \x01(\x0b\x32\x1d.datafusion.ScalarTime64ValueH\x00\x12H\n\x17interval_month_day_nano\x18\x1f \x01(\x0b\x32%.datafusion.IntervalMonthDayNanoValueH\x00\x12/\n\x0cstruct_value\x18 \x01(\x0b\x32\x17.datafusion.StructValueH\x00\x12\x44\n\x17\x66ixed_size_binary_value\x18\" \x01(\x0b\x32!.datafusion.ScalarFixedSizeBinaryH\x00\x42\x07\n\x05valueJ\x04\x08\x13\x10\x14\"1\n\nDecimal128\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\t\n\x01p\x18\x02 \x01(\x03\x12\t\n\x01s\x18\x03 \x01(\x03\"\x94\x0b\n\tArrowType\x12(\n\x04NONE\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04\x42OOL\x18\x02 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05UINT8\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04INT8\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT16\x18\x05 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT16\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT32\x18\x07 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT32\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT64\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT64\x18\n \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT16\x18\x0b \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT32\x18\x0c \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT64\x18\r \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04UTF8\x18\x0e \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12.\n\nLARGE_UTF8\x18 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x42INARY\x18\x0f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x1b\n\x11\x46IXED_SIZE_BINARY\x18\x10 \x01(\x05H\x00\x12\x30\n\x0cLARGE_BINARY\x18\x1f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE32\x18\x11 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE64\x18\x12 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x08\x44URATION\x18\x13 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12*\n\tTIMESTAMP\x18\x14 \x01(\x0b\x32\x15.datafusion.TimestampH\x00\x12&\n\x06TIME32\x18\x15 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12&\n\x06TIME64\x18\x16 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12,\n\x08INTERVAL\x18\x17 \x01(\x0e\x32\x18.datafusion.IntervalUnitH\x00\x12&\n\x07\x44\x45\x43IMAL\x18\x18 \x01(\x0b\x32\x13.datafusion.DecimalH\x00\x12 \n\x04LIST\x18\x19 \x01(\x0b\x32\x10.datafusion.ListH\x00\x12&\n\nLARGE_LIST\x18\x1a \x01(\x0b\x32\x10.datafusion.ListH\x00\x12\x34\n\x0f\x46IXED_SIZE_LIST\x18\x1b \x01(\x0b\x32\x19.datafusion.FixedSizeListH\x00\x12$\n\x06STRUCT\x18\x1c \x01(\x0b\x32\x12.datafusion.StructH\x00\x12\"\n\x05UNION\x18\x1d \x01(\x0b\x32\x11.datafusion.UnionH\x00\x12,\n\nDICTIONARY\x18\x1e \x01(\x0b\x32\x16.datafusion.DictionaryH\x00\x12\x1e\n\x03MAP\x18! \x01(\x0b\x32\x0f.datafusion.MapH\x00\x42\x11\n\x0f\x61rrow_type_enum\"\x0e\n\x0c\x45mptyMessage\"0\n\x17\x41nalyzedLogicalPlanType\x12\x15\n\ranalyzer_name\x18\x01 \x01(\t\"2\n\x18OptimizedLogicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"3\n\x19OptimizedPhysicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"\x8a\x04\n\x08PlanType\x12\x36\n\x12InitialLogicalPlan\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x42\n\x13\x41nalyzedLogicalPlan\x18\x07 \x01(\x0b\x32#.datafusion.AnalyzedLogicalPlanTypeH\x00\x12<\n\x18\x46inalAnalyzedLogicalPlan\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x44\n\x14OptimizedLogicalPlan\x18\x02 \x01(\x0b\x32$.datafusion.OptimizedLogicalPlanTypeH\x00\x12\x34\n\x10\x46inalLogicalPlan\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x37\n\x13InitialPhysicalPlan\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x46\n\x15OptimizedPhysicalPlan\x18\x05 \x01(\x0b\x32%.datafusion.OptimizedPhysicalPlanTypeH\x00\x12\x35\n\x11\x46inalPhysicalPlan\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x42\x10\n\x0eplan_type_enum\"H\n\x0fStringifiedPlan\x12\'\n\tplan_type\x18\x01 \x01(\x0b\x32\x14.datafusion.PlanType\x12\x0c\n\x04plan\x18\x02 \x01(\t\"#\n\x12\x42\x61reTableReference\x12\r\n\x05table\x18\x01 \x01(\t\"6\n\x15PartialTableReference\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\r\n\x05table\x18\x02 \x01(\t\"D\n\x12\x46ullTableReference\x12\x0f\n\x07\x63\x61talog\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\xc3\x01\n\x13OwnedTableReference\x12.\n\x04\x62\x61re\x18\x01 \x01(\x0b\x32\x1e.datafusion.BareTableReferenceH\x00\x12\x34\n\x07partial\x18\x02 \x01(\x0b\x32!.datafusion.PartialTableReferenceH\x00\x12.\n\x04\x66ull\x18\x03 \x01(\x0b\x32\x1e.datafusion.FullTableReferenceH\x00\x42\x16\n\x14table_reference_enum\"\x8a\t\n\x10PhysicalPlanNode\x12\x37\n\x0cparquet_scan\x18\x01 \x01(\x0b\x32\x1f.datafusion.ParquetScanExecNodeH\x00\x12/\n\x08\x63sv_scan\x18\x02 \x01(\x0b\x32\x1b.datafusion.CsvScanExecNodeH\x00\x12*\n\x05\x65mpty\x18\x03 \x01(\x0b\x32\x19.datafusion.EmptyExecNodeH\x00\x12\x34\n\nprojection\x18\x04 \x01(\x0b\x32\x1e.datafusion.ProjectionExecNodeH\x00\x12\x37\n\x0cglobal_limit\x18\x06 \x01(\x0b\x32\x1f.datafusion.GlobalLimitExecNodeH\x00\x12\x35\n\x0blocal_limit\x18\x07 \x01(\x0b\x32\x1e.datafusion.LocalLimitExecNodeH\x00\x12\x32\n\taggregate\x18\x08 \x01(\x0b\x32\x1d.datafusion.AggregateExecNodeH\x00\x12\x31\n\thash_join\x18\t \x01(\x0b\x32\x1c.datafusion.HashJoinExecNodeH\x00\x12(\n\x04sort\x18\n \x01(\x0b\x32\x18.datafusion.SortExecNodeH\x00\x12?\n\x10\x63oalesce_batches\x18\x0b \x01(\x0b\x32#.datafusion.CoalesceBatchesExecNodeH\x00\x12,\n\x06\x66ilter\x18\x0c \x01(\x0b\x32\x1a.datafusion.FilterExecNodeH\x00\x12\x37\n\x05merge\x18\r \x01(\x0b\x32&.datafusion.CoalescePartitionsExecNodeH\x00\x12\x36\n\x0brepartition\x18\x0e \x01(\x0b\x32\x1f.datafusion.RepartitionExecNodeH\x00\x12/\n\x06window\x18\x0f \x01(\x0b\x32\x1d.datafusion.WindowAggExecNodeH\x00\x12\x33\n\ncross_join\x18\x10 \x01(\x0b\x32\x1d.datafusion.CrossJoinExecNodeH\x00\x12\x31\n\tavro_scan\x18\x11 \x01(\x0b\x32\x1c.datafusion.AvroScanExecNodeH\x00\x12\x36\n\textension\x18\x12 \x01(\x0b\x32!.datafusion.PhysicalExtensionNodeH\x00\x12*\n\x05union\x18\x13 \x01(\x0b\x32\x19.datafusion.UnionExecNodeH\x00\x12.\n\x07\x65xplain\x18\x14 \x01(\x0b\x32\x1b.datafusion.ExplainExecNodeH\x00\x12H\n\x15sort_preserving_merge\x18\x15 \x01(\x0b\x32\'.datafusion.SortPreservingMergeExecNodeH\x00\x12>\n\x10nested_loop_join\x18\x16 \x01(\x0b\x32\".datafusion.NestedLoopJoinExecNodeH\x00\x42\x12\n\x10PhysicalPlanType\"S\n\x15PhysicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12,\n\x06inputs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"\xcb\x08\n\x10PhysicalExprNode\x12,\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x1a.datafusion.PhysicalColumnH\x00\x12*\n\x07literal\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x39\n\x0b\x62inary_expr\x18\x03 \x01(\x0b\x32\".datafusion.PhysicalBinaryExprNodeH\x00\x12?\n\x0e\x61ggregate_expr\x18\x04 \x01(\x0b\x32%.datafusion.PhysicalAggregateExprNodeH\x00\x12\x32\n\x0cis_null_expr\x18\x05 \x01(\x0b\x32\x1a.datafusion.PhysicalIsNullH\x00\x12\x39\n\x10is_not_null_expr\x18\x06 \x01(\x0b\x32\x1d.datafusion.PhysicalIsNotNullH\x00\x12+\n\x08not_expr\x18\x07 \x01(\x0b\x32\x17.datafusion.PhysicalNotH\x00\x12-\n\x05\x63\x61se_\x18\x08 \x01(\x0b\x32\x1c.datafusion.PhysicalCaseNodeH\x00\x12,\n\x04\x63\x61st\x18\t \x01(\x0b\x32\x1c.datafusion.PhysicalCastNodeH\x00\x12\x30\n\x04sort\x18\n \x01(\x0b\x32 .datafusion.PhysicalSortExprNodeH\x00\x12\x34\n\x08negative\x18\x0b \x01(\x0b\x32 .datafusion.PhysicalNegativeNodeH\x00\x12\x31\n\x07in_list\x18\x0c \x01(\x0b\x32\x1e.datafusion.PhysicalInListNodeH\x00\x12\x41\n\x0fscalar_function\x18\r \x01(\x0b\x32&.datafusion.PhysicalScalarFunctionNodeH\x00\x12\x33\n\x08try_cast\x18\x0e \x01(\x0b\x32\x1f.datafusion.PhysicalTryCastNodeH\x00\x12\x39\n\x0bwindow_expr\x18\x0f \x01(\x0b\x32\".datafusion.PhysicalWindowExprNodeH\x00\x12\x37\n\nscalar_udf\x18\x10 \x01(\x0b\x32!.datafusion.PhysicalScalarUdfNodeH\x00\x12O\n\x17\x64\x61te_time_interval_expr\x18\x11 \x01(\x0b\x32,.datafusion.PhysicalDateTimeIntervalExprNodeH\x00\x12\x35\n\tlike_expr\x18\x12 \x01(\x0b\x32 .datafusion.PhysicalLikeExprNodeH\x00\x12M\n\x16get_indexed_field_expr\x18\x13 \x01(\x0b\x32+.datafusion.PhysicalGetIndexedFieldExprNodeH\x00\x42\n\n\x08\x45xprType\"}\n\x15PhysicalScalarUdfNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12*\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"\x84\x02\n\x19PhysicalAggregateExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12$\n\x1auser_defined_aggr_function\x18\x04 \x01(\tH\x00\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x36\n\x0cordering_req\x18\x05 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x42\x13\n\x11\x41ggregateFunction\"\xcf\x01\n\x16PhysicalWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12*\n\x04\x65xpr\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNodeB\x11\n\x0fwindow_function\"<\n\x0ePhysicalIsNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"?\n\x11PhysicalIsNotNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"9\n\x0bPhysicalNot\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"N\n\x11PhysicalAliasNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"v\n\x16PhysicalBinaryExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x80\x01\n PhysicalDateTimeIntervalExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x9c\x01\n\x14PhysicalLikeExprNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12\x18\n\x10\x63\x61se_insensitive\x18\x02 \x01(\x08\x12*\n\x04\x65xpr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12-\n\x07pattern\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"d\n\x14PhysicalSortExprNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"t\n\x10PhysicalWhenThen\x12/\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"}\n\x12PhysicalInListNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x04list\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"\xa5\x01\n\x10PhysicalCaseNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x34\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalWhenThen\x12/\n\telse_expr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\xab\x01\n\x1aPhysicalScalarFunctionNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\'\n\x03\x66un\x18\x02 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12*\n\x04\x61rgs\x18\x03 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"l\n\x13PhysicalTryCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"i\n\x10PhysicalCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"B\n\x14PhysicalNegativeNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"i\n\x0e\x46ilterExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"7\n\tFileGroup\x12*\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x1b.datafusion.PartitionedFile\"\x1a\n\tScanLimit\x12\r\n\x05limit\x18\x01 \x01(\r\"d\n\x1ePhysicalSortExprNodeCollection\x12\x42\n\x18physical_sort_expr_nodes\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\xcb\x02\n\x10\x46ileScanExecConf\x12*\n\x0b\x66ile_groups\x18\x01 \x03(\x0b\x32\x15.datafusion.FileGroup\x12\"\n\x06schema\x18\x02 \x01(\x0b\x32\x12.datafusion.Schema\x12\x12\n\nprojection\x18\x04 \x03(\r\x12$\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.ScanLimit\x12*\n\nstatistics\x18\x06 \x01(\x0b\x32\x16.datafusion.Statistics\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x18\n\x10object_store_url\x18\x08 \x01(\t\x12\x43\n\x0foutput_ordering\x18\t \x03(\x0b\x32*.datafusion.PhysicalSortExprNodeCollectionJ\x04\x08\n\x10\x0b\"}\n\x13ParquetScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12/\n\tpredicate\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNodeJ\x04\x08\x02\x10\x03\"i\n\x0f\x43svScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12\x12\n\nhas_header\x18\x02 \x01(\x08\x12\x11\n\tdelimiter\x18\x03 \x01(\t\"C\n\x10\x41vroScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\"\xa9\x02\n\x10HashJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x1e\n\x02on\x18\x03 \x03(\x0b\x32\x12.datafusion.JoinOn\x12\'\n\tjoin_type\x18\x04 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x31\n\x0epartition_mode\x18\x06 \x01(\x0e\x32\x19.datafusion.PartitionMode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12&\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x16.datafusion.JoinFilter\"=\n\rUnionExecNode\x12,\n\x06inputs\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"~\n\x0f\x45xplainExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\x12\x36\n\x11stringified_plans\x18\x02 \x03(\x0b\x32\x1b.datafusion.StringifiedPlan\x12\x0f\n\x07verbose\x18\x03 \x01(\x08\"l\n\x11\x43rossJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"-\n\x0ePhysicalColumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\r\"]\n\x06JoinOn\x12(\n\x04left\x18\x01 \x01(\x0b\x32\x1a.datafusion.PhysicalColumn\x12)\n\x05right\x18\x02 \x01(\x0b\x32\x1a.datafusion.PhysicalColumn\"L\n\rEmptyExecNode\x12\x17\n\x0fproduce_one_row\x18\x01 \x01(\x08\x12\"\n\x06schema\x18\x02 \x01(\x0b\x32\x12.datafusion.Schema\"\x80\x01\n\x12ProjectionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x11\n\texpr_name\x18\x03 \x03(\t\"\xb7\x01\n\x11WindowAggExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x31\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x18\n\x10window_expr_name\x18\x03 \x03(\t\x12(\n\x0cinput_schema\x18\x04 \x01(\x0b\x32\x12.datafusion.Schema\"9\n\x0bMaybeFilter\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"M\n\x16MaybePhysicalSortExprs\x12\x33\n\tsort_expr\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\xd1\x03\n\x11\x41ggregateExecNode\x12\x30\n\ngroup_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\taggr_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x04mode\x18\x03 \x01(\x0e\x32\x19.datafusion.AggregateMode\x12+\n\x05input\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x17\n\x0fgroup_expr_name\x18\x05 \x03(\t\x12\x16\n\x0e\x61ggr_expr_name\x18\x06 \x03(\t\x12(\n\x0cinput_schema\x18\x07 \x01(\x0b\x32\x12.datafusion.Schema\x12/\n\tnull_expr\x18\x08 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0e\n\x06groups\x18\t \x03(\x08\x12,\n\x0b\x66ilter_expr\x18\n \x03(\x0b\x32\x17.datafusion.MaybeFilter\x12\x39\n\rorder_by_expr\x18\x0b \x03(\x0b\x32\".datafusion.MaybePhysicalSortExprs\"_\n\x13GlobalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\r\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"P\n\x12LocalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\r\n\x05\x66\x65tch\x18\x02 \x01(\r\"\x95\x01\n\x0cSortExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\x12\x1d\n\x15preserve_partitioning\x18\x04 \x01(\x08\"\x85\x01\n\x1bSortPreservingMergeExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\xc2\x01\n\x16NestedLoopJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12&\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x16.datafusion.JoinFilter\"a\n\x17\x43oalesceBatchesExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x19\n\x11target_batch_size\x18\x02 \x01(\r\"I\n\x1a\x43oalescePartitionsExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"c\n\x17PhysicalHashRepartition\x12/\n\thash_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\"\xb5\x01\n\x13RepartitionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12\x33\n\x04hash\x18\x03 \x01(\x0b\x32#.datafusion.PhysicalHashRepartitionH\x00\x12\x11\n\x07unknown\x18\x04 \x01(\x04H\x00\x42\x12\n\x10partition_method\"\x93\x01\n\nJoinFilter\x12\x30\n\nexpression\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\x0e\x63olumn_indices\x18\x02 \x03(\x0b\x32\x17.datafusion.ColumnIndex\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\"@\n\x0b\x43olumnIndex\x12\r\n\x05index\x18\x01 \x01(\r\x12\"\n\x04side\x18\x02 \x01(\x0e\x32\x14.datafusion.JoinSide\"\xa0\x01\n\x0fPartitionedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\x12\x18\n\x10last_modified_ns\x18\x03 \x01(\x04\x12\x31\n\x10partition_values\x18\x04 \x03(\x0b\x32\x17.datafusion.ScalarValue\x12$\n\x05range\x18\x05 \x01(\x0b\x32\x15.datafusion.FileRange\"\'\n\tFileRange\x12\r\n\x05start\x18\x01 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x03\"y\n\x0ePartitionStats\x12\x10\n\x08num_rows\x18\x01 \x01(\x03\x12\x13\n\x0bnum_batches\x18\x02 \x01(\x03\x12\x11\n\tnum_bytes\x18\x03 \x01(\x03\x12-\n\x0c\x63olumn_stats\x18\x04 \x03(\x0b\x32\x17.datafusion.ColumnStats\"x\n\nStatistics\x12\x10\n\x08num_rows\x18\x01 \x01(\x03\x12\x17\n\x0ftotal_byte_size\x18\x02 \x01(\x03\x12-\n\x0c\x63olumn_stats\x18\x03 \x03(\x0b\x32\x17.datafusion.ColumnStats\x12\x10\n\x08is_exact\x18\x04 \x01(\x08\"\x91\x01\n\x0b\x43olumnStats\x12*\n\tmin_value\x18\x01 \x01(\x0b\x32\x17.datafusion.ScalarValue\x12*\n\tmax_value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\x12\x12\n\nnull_count\x18\x03 \x01(\r\x12\x16\n\x0e\x64istinct_count\x18\x04 \x01(\r\"r\n\x1fPhysicalGetIndexedFieldExprNode\x12)\n\x03\x61rg\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12$\n\x03key\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue*n\n\x08JoinType\x12\t\n\x05INNER\x10\x00\x12\x08\n\x04LEFT\x10\x01\x12\t\n\x05RIGHT\x10\x02\x12\x08\n\x04\x46ULL\x10\x03\x12\x0c\n\x08LEFTSEMI\x10\x04\x12\x0c\n\x08LEFTANTI\x10\x05\x12\r\n\tRIGHTSEMI\x10\x06\x12\r\n\tRIGHTANTI\x10\x07*#\n\x0eJoinConstraint\x12\x06\n\x02ON\x10\x00\x12\t\n\x05USING\x10\x01*\xe6\n\n\x0eScalarFunction\x12\x07\n\x03\x41\x62s\x10\x00\x12\x08\n\x04\x41\x63os\x10\x01\x12\x08\n\x04\x41sin\x10\x02\x12\x08\n\x04\x41tan\x10\x03\x12\t\n\x05\x41scii\x10\x04\x12\x08\n\x04\x43\x65il\x10\x05\x12\x07\n\x03\x43os\x10\x06\x12\n\n\x06\x44igest\x10\x07\x12\x07\n\x03\x45xp\x10\x08\x12\t\n\x05\x46loor\x10\t\x12\x06\n\x02Ln\x10\n\x12\x07\n\x03Log\x10\x0b\x12\t\n\x05Log10\x10\x0c\x12\x08\n\x04Log2\x10\r\x12\t\n\x05Round\x10\x0e\x12\n\n\x06Signum\x10\x0f\x12\x07\n\x03Sin\x10\x10\x12\x08\n\x04Sqrt\x10\x11\x12\x07\n\x03Tan\x10\x12\x12\t\n\x05Trunc\x10\x13\x12\t\n\x05\x41rray\x10\x14\x12\x0f\n\x0bRegexpMatch\x10\x15\x12\r\n\tBitLength\x10\x16\x12\t\n\x05\x42trim\x10\x17\x12\x13\n\x0f\x43haracterLength\x10\x18\x12\x07\n\x03\x43hr\x10\x19\x12\n\n\x06\x43oncat\x10\x1a\x12\x17\n\x13\x43oncatWithSeparator\x10\x1b\x12\x0c\n\x08\x44\x61tePart\x10\x1c\x12\r\n\tDateTrunc\x10\x1d\x12\x0b\n\x07InitCap\x10\x1e\x12\x08\n\x04Left\x10\x1f\x12\x08\n\x04Lpad\x10 \x12\t\n\x05Lower\x10!\x12\t\n\x05Ltrim\x10\"\x12\x07\n\x03MD5\x10#\x12\n\n\x06NullIf\x10$\x12\x0f\n\x0bOctetLength\x10%\x12\n\n\x06Random\x10&\x12\x11\n\rRegexpReplace\x10\'\x12\n\n\x06Repeat\x10(\x12\x0b\n\x07Replace\x10)\x12\x0b\n\x07Reverse\x10*\x12\t\n\x05Right\x10+\x12\x08\n\x04Rpad\x10,\x12\t\n\x05Rtrim\x10-\x12\n\n\x06SHA224\x10.\x12\n\n\x06SHA256\x10/\x12\n\n\x06SHA384\x10\x30\x12\n\n\x06SHA512\x10\x31\x12\r\n\tSplitPart\x10\x32\x12\x0e\n\nStartsWith\x10\x33\x12\n\n\x06Strpos\x10\x34\x12\n\n\x06Substr\x10\x35\x12\t\n\x05ToHex\x10\x36\x12\x0f\n\x0bToTimestamp\x10\x37\x12\x15\n\x11ToTimestampMillis\x10\x38\x12\x15\n\x11ToTimestampMicros\x10\x39\x12\x16\n\x12ToTimestampSeconds\x10:\x12\x07\n\x03Now\x10;\x12\r\n\tTranslate\x10<\x12\x08\n\x04Trim\x10=\x12\t\n\x05Upper\x10>\x12\x0c\n\x08\x43oalesce\x10?\x12\t\n\x05Power\x10@\x12\r\n\tStructFun\x10\x41\x12\x10\n\x0c\x46romUnixtime\x10\x42\x12\t\n\x05\x41tan2\x10\x43\x12\x0b\n\x07\x44\x61teBin\x10\x44\x12\x0f\n\x0b\x41rrowTypeof\x10\x45\x12\x0f\n\x0b\x43urrentDate\x10\x46\x12\x0f\n\x0b\x43urrentTime\x10G\x12\x08\n\x04Uuid\x10H\x12\x08\n\x04\x43\x62rt\x10I\x12\t\n\x05\x41\x63osh\x10J\x12\t\n\x05\x41sinh\x10K\x12\t\n\x05\x41tanh\x10L\x12\x08\n\x04Sinh\x10M\x12\x08\n\x04\x43osh\x10N\x12\x08\n\x04Tanh\x10O\x12\x06\n\x02Pi\x10P\x12\x0b\n\x07\x44\x65grees\x10Q\x12\x0b\n\x07Radians\x10R\x12\r\n\tFactorial\x10S\x12\x07\n\x03Lcm\x10T\x12\x07\n\x03Gcd\x10U\x12\x0f\n\x0b\x41rrayAppend\x10V\x12\x0f\n\x0b\x41rrayConcat\x10W\x12\r\n\tArrayDims\x10X\x12\r\n\tArrayFill\x10Y\x12\x0f\n\x0b\x41rrayLength\x10Z\x12\x0e\n\nArrayNdims\x10[\x12\x11\n\rArrayPosition\x10\\\x12\x12\n\x0e\x41rrayPositions\x10]\x12\x10\n\x0c\x41rrayPrepend\x10^\x12\x0f\n\x0b\x41rrayRemove\x10_\x12\x10\n\x0c\x41rrayReplace\x10`\x12\x11\n\rArrayToString\x10\x61\x12\x0f\n\x0b\x43\x61rdinality\x10\x62\x12\r\n\tTrimArray\x10\x63\x12\x11\n\rArrayContains\x10\x64\x12\n\n\x06\x45ncode\x10\x65\x12\n\n\x06\x44\x65\x63ode\x10\x66\x12\x07\n\x03\x43ot\x10g*\xb2\x03\n\x11\x41ggregateFunction\x12\x07\n\x03MIN\x10\x00\x12\x07\n\x03MAX\x10\x01\x12\x07\n\x03SUM\x10\x02\x12\x07\n\x03\x41VG\x10\x03\x12\t\n\x05\x43OUNT\x10\x04\x12\x13\n\x0f\x41PPROX_DISTINCT\x10\x05\x12\r\n\tARRAY_AGG\x10\x06\x12\x0c\n\x08VARIANCE\x10\x07\x12\x10\n\x0cVARIANCE_POP\x10\x08\x12\x0e\n\nCOVARIANCE\x10\t\x12\x12\n\x0e\x43OVARIANCE_POP\x10\n\x12\n\n\x06STDDEV\x10\x0b\x12\x0e\n\nSTDDEV_POP\x10\x0c\x12\x0f\n\x0b\x43ORRELATION\x10\r\x12\x1a\n\x16\x41PPROX_PERCENTILE_CONT\x10\x0e\x12\x11\n\rAPPROX_MEDIAN\x10\x0f\x12&\n\"APPROX_PERCENTILE_CONT_WITH_WEIGHT\x10\x10\x12\x0c\n\x08GROUPING\x10\x11\x12\n\n\x06MEDIAN\x10\x12\x12\x0b\n\x07\x42IT_AND\x10\x13\x12\n\n\x06\x42IT_OR\x10\x14\x12\x0b\n\x07\x42IT_XOR\x10\x15\x12\x0c\n\x08\x42OOL_AND\x10\x16\x12\x0b\n\x07\x42OOL_OR\x10\x17\x12\x13\n\x0f\x46IRST_VALUE_AGG\x10\x18\x12\x12\n\x0eLAST_VALUE_AGG\x10\x19*\xb0\x01\n\x15\x42uiltInWindowFunction\x12\x0e\n\nROW_NUMBER\x10\x00\x12\x08\n\x04RANK\x10\x01\x12\x0e\n\nDENSE_RANK\x10\x02\x12\x10\n\x0cPERCENT_RANK\x10\x03\x12\r\n\tCUME_DIST\x10\x04\x12\t\n\x05NTILE\x10\x05\x12\x07\n\x03LAG\x10\x06\x12\x08\n\x04LEAD\x10\x07\x12\x0f\n\x0b\x46IRST_VALUE\x10\x08\x12\x0e\n\nLAST_VALUE\x10\t\x12\r\n\tNTH_VALUE\x10\n*3\n\x10WindowFrameUnits\x12\x08\n\x04ROWS\x10\x00\x12\t\n\x05RANGE\x10\x01\x12\n\n\x06GROUPS\x10\x02*E\n\x14WindowFrameBoundType\x12\x0f\n\x0b\x43URRENT_ROW\x10\x00\x12\r\n\tPRECEDING\x10\x01\x12\r\n\tFOLLOWING\x10\x02*(\n\x08\x44\x61teUnit\x12\x07\n\x03\x44\x61y\x10\x00\x12\x13\n\x0f\x44\x61teMillisecond\x10\x01*H\n\x08TimeUnit\x12\n\n\x06Second\x10\x00\x12\x0f\n\x0bMillisecond\x10\x01\x12\x0f\n\x0bMicrosecond\x10\x02\x12\x0e\n\nNanosecond\x10\x03*<\n\x0cIntervalUnit\x12\r\n\tYearMonth\x10\x00\x12\x0b\n\x07\x44\x61yTime\x10\x01\x12\x10\n\x0cMonthDayNano\x10\x02*\"\n\tUnionMode\x12\n\n\x06sparse\x10\x00\x12\t\n\x05\x64\x65nse\x10\x01*<\n\rPartitionMode\x12\x10\n\x0c\x43OLLECT_LEFT\x10\x00\x12\x0f\n\x0bPARTITIONED\x10\x01\x12\x08\n\x04\x41UTO\x10\x02*b\n\rAggregateMode\x12\x0b\n\x07PARTIAL\x10\x00\x12\t\n\x05\x46INAL\x10\x01\x12\x15\n\x11\x46INAL_PARTITIONED\x10\x02\x12\n\n\x06SINGLE\x10\x03\x12\x16\n\x12SINGLE_PARTITIONED\x10\x04*)\n\x08JoinSide\x12\r\n\tLEFT_SIDE\x10\x00\x12\x0e\n\nRIGHT_SIDE\x10\x01\x42\x39\n$org.apache.arrow.datafusion.protobufB\x0f\x44\x61tafusionProtoP\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16proto/datafusion.proto\x12\ndatafusion\"\"\n\x0e\x43olumnRelation\x12\x10\n\x08relation\x18\x01 \x01(\t\"D\n\x06\x43olumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\x08relation\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"Z\n\x07\x44\x66\x46ield\x12 \n\x05\x66ield\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12-\n\tqualifier\x18\x02 \x01(\x0b\x32\x1a.datafusion.ColumnRelation\"\x97\x01\n\x08\x44\x66Schema\x12$\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x13.datafusion.DfField\x12\x34\n\x08metadata\x18\x02 \x03(\x0b\x32\".datafusion.DfSchema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x91\x0b\n\x0fLogicalPlanNode\x12\x38\n\x0clisting_scan\x18\x01 \x01(\x0b\x32 .datafusion.ListingTableScanNodeH\x00\x12\x30\n\nprojection\x18\x03 \x01(\x0b\x32\x1a.datafusion.ProjectionNodeH\x00\x12.\n\tselection\x18\x04 \x01(\x0b\x32\x19.datafusion.SelectionNodeH\x00\x12&\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.LimitNodeH\x00\x12.\n\taggregate\x18\x06 \x01(\x0b\x32\x19.datafusion.AggregateNodeH\x00\x12$\n\x04join\x18\x07 \x01(\x0b\x32\x14.datafusion.JoinNodeH\x00\x12$\n\x04sort\x18\x08 \x01(\x0b\x32\x14.datafusion.SortNodeH\x00\x12\x32\n\x0brepartition\x18\t \x01(\x0b\x32\x1b.datafusion.RepartitionNodeH\x00\x12\x37\n\x0e\x65mpty_relation\x18\n \x01(\x0b\x32\x1d.datafusion.EmptyRelationNodeH\x00\x12\x44\n\x15\x63reate_external_table\x18\x0b \x01(\x0b\x32#.datafusion.CreateExternalTableNodeH\x00\x12*\n\x07\x65xplain\x18\x0c \x01(\x0b\x32\x17.datafusion.ExplainNodeH\x00\x12(\n\x06window\x18\r \x01(\x0b\x32\x16.datafusion.WindowNodeH\x00\x12*\n\x07\x61nalyze\x18\x0e \x01(\x0b\x32\x17.datafusion.AnalyzeNodeH\x00\x12/\n\ncross_join\x18\x0f \x01(\x0b\x32\x19.datafusion.CrossJoinNodeH\x00\x12(\n\x06values\x18\x10 \x01(\x0b\x32\x16.datafusion.ValuesNodeH\x00\x12\x35\n\textension\x18\x11 \x01(\x0b\x32 .datafusion.LogicalExtensionNodeH\x00\x12\x44\n\x15\x63reate_catalog_schema\x18\x12 \x01(\x0b\x32#.datafusion.CreateCatalogSchemaNodeH\x00\x12&\n\x05union\x18\x13 \x01(\x0b\x32\x15.datafusion.UnionNodeH\x00\x12\x37\n\x0e\x63reate_catalog\x18\x14 \x01(\x0b\x32\x1d.datafusion.CreateCatalogNodeH\x00\x12\x37\n\x0esubquery_alias\x18\x15 \x01(\x0b\x32\x1d.datafusion.SubqueryAliasNodeH\x00\x12\x31\n\x0b\x63reate_view\x18\x16 \x01(\x0b\x32\x1a.datafusion.CreateViewNodeH\x00\x12,\n\x08\x64istinct\x18\x17 \x01(\x0b\x32\x18.datafusion.DistinctNodeH\x00\x12\x32\n\tview_scan\x18\x18 \x01(\x0b\x32\x1d.datafusion.ViewTableScanNodeH\x00\x12\x36\n\x0b\x63ustom_scan\x18\x19 \x01(\x0b\x32\x1f.datafusion.CustomTableScanNodeH\x00\x12*\n\x07prepare\x18\x1a \x01(\x0b\x32\x17.datafusion.PrepareNodeH\x00\x12-\n\tdrop_view\x18\x1b \x01(\x0b\x32\x18.datafusion.DropViewNodeH\x00\x12\x31\n\x0b\x64istinct_on\x18\x1c \x01(\x0b\x32\x1a.datafusion.DistinctOnNodeH\x00\x12)\n\x07\x63opy_to\x18\x1d \x01(\x0b\x32\x16.datafusion.CopyToNodeH\x00\x42\x11\n\x0fLogicalPlanType\"Q\n\x14LogicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"$\n\x11ProjectionColumns\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\"f\n\tCsvFormat\x12\x12\n\nhas_header\x18\x01 \x01(\x08\x12\x11\n\tdelimiter\x18\x02 \x01(\t\x12\r\n\x05quote\x18\x03 \x01(\t\x12\x10\n\x06\x65scape\x18\x04 \x01(\tH\x00\x42\x11\n\x0foptional_escape\"\x15\n\rParquetFormatJ\x04\x08\x01\x10\x02\"\x0c\n\nAvroFormat\"T\n\x19LogicalExprNodeCollection\x12\x37\n\x12logical_expr_nodes\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x9a\x04\n\x14ListingTableScanNode\x12\x33\n\ntable_name\x18\x0e \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\r\n\x05paths\x18\x02 \x03(\t\x12\x16\n\x0e\x66ile_extension\x18\x03 \x01(\t\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x05 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x14\n\x0c\x63ollect_stat\x18\x08 \x01(\x08\x12\x19\n\x11target_partitions\x18\t \x01(\r\x12$\n\x03\x63sv\x18\n \x01(\x0b\x32\x15.datafusion.CsvFormatH\x00\x12,\n\x07parquet\x18\x0b \x01(\x0b\x32\x19.datafusion.ParquetFormatH\x00\x12&\n\x04\x61vro\x18\x0c \x01(\x0b\x32\x16.datafusion.AvroFormatH\x00\x12>\n\x0f\x66ile_sort_order\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollectionB\x10\n\x0e\x46ileFormatTypeJ\x04\x08\x01\x10\x02\"\xe5\x01\n\x11ViewTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12\x31\n\nprojection\x18\x04 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\x12\n\ndefinition\x18\x05 \x01(\tJ\x04\x08\x01\x10\x02\"\xf0\x01\n\x13\x43ustomTableScanNode\x12\x33\n\ntable_name\x18\x06 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x31\n\nprojection\x18\x02 \x01(\x0b\x32\x1d.datafusion.ProjectionColumns\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12,\n\x07\x66ilters\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x19\n\x11\x63ustom_table_data\x18\x05 \x01(\x0cJ\x04\x08\x01\x10\x02\"\x8a\x01\n\x0eProjectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x05\x61lias\x18\x03 \x01(\tH\x00\x42\x10\n\x0eoptional_alias\"f\n\rSelectionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"p\n\x08SortNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\x95\x01\n\x0fRepartitionNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12+\n\x04hash\x18\x03 \x01(\x0b\x32\x1b.datafusion.HashRepartitionH\x00\x42\x12\n\x10partition_method\"Z\n\x0fHashRepartition\x12.\n\thash_expr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\",\n\x11\x45mptyRelationNode\x12\x17\n\x0fproduce_one_row\x18\x01 \x01(\x08\"\'\n\x14PrimaryKeyConstraint\x12\x0f\n\x07indices\x18\x01 \x03(\x04\"#\n\x10UniqueConstraint\x12\x0f\n\x07indices\x18\x01 \x03(\x04\"\x88\x01\n\nConstraint\x12\x37\n\x0bprimary_key\x18\x01 \x01(\x0b\x32 .datafusion.PrimaryKeyConstraintH\x00\x12.\n\x06unique\x18\x02 \x01(\x0b\x32\x1c.datafusion.UniqueConstraintH\x00\x42\x11\n\x0f\x63onstraint_mode\":\n\x0b\x43onstraints\x12+\n\x0b\x63onstraints\x18\x01 \x03(\x0b\x32\x16.datafusion.Constraint\"\xe8\x05\n\x17\x43reateExternalTableNode\x12-\n\x04name\x18\x0c \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x10\n\x08location\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x12\n\nhas_header\x18\x04 \x01(\x08\x12$\n\x06schema\x18\x05 \x01(\x0b\x32\x14.datafusion.DfSchema\x12\x1c\n\x14table_partition_cols\x18\x06 \x03(\t\x12\x15\n\rif_not_exists\x18\x07 \x01(\x08\x12\x11\n\tdelimiter\x18\x08 \x01(\t\x12\x12\n\ndefinition\x18\t \x01(\t\x12\x41\n\x15\x66ile_compression_type\x18\x11 \x01(\x0e\x32\".datafusion.CompressionTypeVariant\x12:\n\x0border_exprs\x18\r \x03(\x0b\x32%.datafusion.LogicalExprNodeCollection\x12\x11\n\tunbounded\x18\x0e \x01(\x08\x12\x41\n\x07options\x18\x0b \x03(\x0b\x32\x30.datafusion.CreateExternalTableNode.OptionsEntry\x12,\n\x0b\x63onstraints\x18\x0f \x01(\x0b\x32\x17.datafusion.Constraints\x12P\n\x0f\x63olumn_defaults\x18\x10 \x03(\x0b\x32\x37.datafusion.CreateExternalTableNode.ColumnDefaultsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aR\n\x13\x43olumnDefaultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode:\x02\x38\x01J\x04\x08\x01\x10\x02J\x04\x08\n\x10\x0b\"r\n\x0bPrepareNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\ndata_types\x18\x02 \x03(\x0b\x32\x15.datafusion.ArrowType\x12*\n\x05input\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"k\n\x17\x43reateCatalogSchemaNode\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"f\n\x11\x43reateCatalogNode\x12\x14\n\x0c\x63\x61talog_name\x18\x01 \x01(\t\x12\x15\n\rif_not_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"v\n\x0c\x44ropViewNode\x12-\n\x04name\x18\x01 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12\x11\n\tif_exists\x18\x02 \x01(\x08\x12$\n\x06schema\x18\x03 \x01(\x0b\x32\x14.datafusion.DfSchema\"\x99\x01\n\x0e\x43reateViewNode\x12-\n\x04name\x18\x05 \x01(\x0b\x32\x1f.datafusion.OwnedTableReference\x12*\n\x05input\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x12\n\nor_replace\x18\x03 \x01(\x08\x12\x12\n\ndefinition\x18\x04 \x01(\tJ\x04\x08\x01\x10\x02\"N\n\nValuesNode\x12\x0e\n\x06n_cols\x18\x01 \x01(\x04\x12\x30\n\x0bvalues_list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"J\n\x0b\x41nalyzeNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"J\n\x0b\x45xplainNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0f\n\x07verbose\x18\x02 \x01(\x08\"\x9c\x01\n\rAggregateNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12/\n\ngroup_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\taggr_expr\x18\x03 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\nWindowNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x30\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xef\x02\n\x08JoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x33\n\x0fjoin_constraint\x18\x04 \x01(\x0e\x32\x1a.datafusion.JoinConstraint\x12\x32\n\rleft_join_key\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x33\n\x0eright_join_key\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12+\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\":\n\x0c\x44istinctNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"\xcc\x01\n\x0e\x44istinctOnNode\x12,\n\x07on_expr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x30\n\x0bselect_expr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\tsort_expr\x18\x03 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12*\n\x05input\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"\xda\x01\n\nCopyToNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x12\n\noutput_url\x18\x02 \x01(\t\x12-\n\x0bsql_options\x18\x04 \x01(\x0b\x32\x16.datafusion.SQLOptionsH\x00\x12;\n\x0ewriter_options\x18\x05 \x01(\x0b\x32!.datafusion.FileTypeWriterOptionsH\x00\x12\x11\n\tfile_type\x18\x06 \x01(\tB\r\n\x0b\x43opyOptions\"3\n\nSQLOptions\x12%\n\x06option\x18\x01 \x03(\x0b\x32\x15.datafusion.SQLOption\"\'\n\tSQLOption\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"8\n\tUnionNode\x12+\n\x06inputs\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalPlanNode\"f\n\rCrossJoinNode\x12)\n\x04left\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12*\n\x05right\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\"T\n\tLimitNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\x03\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\">\n\x11SelectionExecNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"u\n\x11SubqueryAliasNode\x12*\n\x05input\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalPlanNode\x12.\n\x05\x61lias\x18\x03 \x01(\x0b\x32\x1f.datafusion.OwnedTableReferenceJ\x04\x08\x02\x10\x03\"\xe9\x0c\n\x0fLogicalExprNode\x12$\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x12.datafusion.ColumnH\x00\x12&\n\x05\x61lias\x18\x02 \x01(\x0b\x32\x15.datafusion.AliasNodeH\x00\x12*\n\x07literal\x18\x03 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x31\n\x0b\x62inary_expr\x18\x04 \x01(\x0b\x32\x1a.datafusion.BinaryExprNodeH\x00\x12\x37\n\x0e\x61ggregate_expr\x18\x05 \x01(\x0b\x32\x1d.datafusion.AggregateExprNodeH\x00\x12*\n\x0cis_null_expr\x18\x06 \x01(\x0b\x32\x12.datafusion.IsNullH\x00\x12\x31\n\x10is_not_null_expr\x18\x07 \x01(\x0b\x32\x15.datafusion.IsNotNullH\x00\x12#\n\x08not_expr\x18\x08 \x01(\x0b\x32\x0f.datafusion.NotH\x00\x12*\n\x07\x62\x65tween\x18\t \x01(\x0b\x32\x17.datafusion.BetweenNodeH\x00\x12%\n\x05\x63\x61se_\x18\n \x01(\x0b\x32\x14.datafusion.CaseNodeH\x00\x12$\n\x04\x63\x61st\x18\x0b \x01(\x0b\x32\x14.datafusion.CastNodeH\x00\x12(\n\x04sort\x18\x0c \x01(\x0b\x32\x18.datafusion.SortExprNodeH\x00\x12,\n\x08negative\x18\r \x01(\x0b\x32\x18.datafusion.NegativeNodeH\x00\x12)\n\x07in_list\x18\x0e \x01(\x0b\x32\x16.datafusion.InListNodeH\x00\x12(\n\x08wildcard\x18\x0f \x01(\x0b\x32\x14.datafusion.WildcardH\x00\x12\x39\n\x0fscalar_function\x18\x10 \x01(\x0b\x32\x1e.datafusion.ScalarFunctionNodeH\x00\x12+\n\x08try_cast\x18\x11 \x01(\x0b\x32\x17.datafusion.TryCastNodeH\x00\x12\x31\n\x0bwindow_expr\x18\x12 \x01(\x0b\x32\x1a.datafusion.WindowExprNodeH\x00\x12>\n\x12\x61ggregate_udf_expr\x18\x13 \x01(\x0b\x32 .datafusion.AggregateUDFExprNodeH\x00\x12\x38\n\x0fscalar_udf_expr\x18\x14 \x01(\x0b\x32\x1d.datafusion.ScalarUDFExprNodeH\x00\x12\x38\n\x11get_indexed_field\x18\x15 \x01(\x0b\x32\x1b.datafusion.GetIndexedFieldH\x00\x12\x33\n\x0cgrouping_set\x18\x16 \x01(\x0b\x32\x1b.datafusion.GroupingSetNodeH\x00\x12$\n\x04\x63ube\x18\x17 \x01(\x0b\x32\x14.datafusion.CubeNodeH\x00\x12(\n\x06rollup\x18\x18 \x01(\x0b\x32\x16.datafusion.RollupNodeH\x00\x12%\n\x07is_true\x18\x19 \x01(\x0b\x32\x12.datafusion.IsTrueH\x00\x12\'\n\x08is_false\x18\x1a \x01(\x0b\x32\x13.datafusion.IsFalseH\x00\x12+\n\nis_unknown\x18\x1b \x01(\x0b\x32\x15.datafusion.IsUnknownH\x00\x12,\n\x0bis_not_true\x18\x1c \x01(\x0b\x32\x15.datafusion.IsNotTrueH\x00\x12.\n\x0cis_not_false\x18\x1d \x01(\x0b\x32\x16.datafusion.IsNotFalseH\x00\x12\x32\n\x0eis_not_unknown\x18\x1e \x01(\x0b\x32\x18.datafusion.IsNotUnknownH\x00\x12$\n\x04like\x18\x1f \x01(\x0b\x32\x14.datafusion.LikeNodeH\x00\x12&\n\x05ilike\x18 \x01(\x0b\x32\x15.datafusion.ILikeNodeH\x00\x12/\n\nsimilar_to\x18! \x01(\x0b\x32\x19.datafusion.SimilarToNodeH\x00\x12\x32\n\x0bplaceholder\x18\" \x01(\x0b\x32\x1b.datafusion.PlaceholderNodeH\x00\x12$\n\x06unnest\x18# \x01(\x0b\x32\x12.datafusion.UnnestH\x00\x42\n\n\x08\x45xprType\"\x1d\n\x08Wildcard\x12\x11\n\tqualifier\x18\x01 \x01(\t\"G\n\x0fPlaceholderNode\x12\n\n\x02id\x18\x01 \x01(\t\x12(\n\tdata_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"<\n\x0fLogicalExprList\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"<\n\x0fGroupingSetNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprList\"5\n\x08\x43ubeNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nRollupNode\x12)\n\x04\x65xpr\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"9\n\x10NamedStructField\x12%\n\x04name\x18\x01 \x01(\x0b\x32\x17.datafusion.ScalarValue\"5\n\tListIndex\x12(\n\x03key\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x8f\x01\n\tListRange\x12*\n\x05start\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04stop\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12+\n\x06stride\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xdb\x01\n\x0fGetIndexedField\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12:\n\x12named_struct_field\x18\x02 \x01(\x0b\x32\x1c.datafusion.NamedStructFieldH\x00\x12+\n\nlist_index\x18\x03 \x01(\x0b\x32\x15.datafusion.ListIndexH\x00\x12+\n\nlist_range\x18\x04 \x01(\x0b\x32\x15.datafusion.ListRangeH\x00\x42\x07\n\x05\x66ield\"3\n\x06IsNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotNull\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"3\n\x06IsTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"4\n\x07IsFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"6\n\tIsNotTrue\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"7\n\nIsNotFalse\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"9\n\x0cIsNotUnknown\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"0\n\x03Not\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"x\n\tAliasNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\x12\x31\n\x08relation\x18\x03 \x03(\x0b\x32\x1f.datafusion.OwnedTableReference\"K\n\x0e\x42inaryExprNode\x12-\n\x08operands\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"9\n\x0cNegativeNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"4\n\x06Unnest\x12*\n\x05\x65xprs\x18\x01 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"s\n\nInListNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04list\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"h\n\x12ScalarFunctionNode\x12\'\n\x03\x66un\x18\x01 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xe2\x01\n\x11\x41ggregateExprNode\x12\x34\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunction\x12)\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x12+\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xaf\x01\n\x14\x41ggregateUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12+\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x04 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"P\n\x11ScalarUDFExprNode\x12\x10\n\x08\x66un_name\x18\x01 \x01(\t\x12)\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\"\xf7\x02\n\x0eWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12\x0e\n\x04udaf\x18\x03 \x01(\tH\x00\x12\x0e\n\x04udwf\x18\t \x01(\tH\x00\x12)\n\x04\x65xpr\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x31\n\x0cpartition_by\x18\x05 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x08order_by\x18\x06 \x03(\x0b\x32\x1b.datafusion.LogicalExprNode\x12-\n\x0cwindow_frame\x18\x08 \x01(\x0b\x32\x17.datafusion.WindowFrameB\x11\n\x0fwindow_function\"\x9e\x01\n\x0b\x42\x65tweenNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0f\n\x07negated\x18\x02 \x01(\x08\x12(\n\x03low\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\x04high\x18\x04 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"\x89\x01\n\x08LikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8a\x01\n\tILikeNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x8e\x01\n\rSimilarToNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12)\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x07pattern\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x13\n\x0b\x65scape_char\x18\x04 \x01(\t\"\x93\x01\n\x08\x43\x61seNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12,\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x14.datafusion.WhenThen\x12.\n\telse_expr\x18\x03 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"j\n\x08WhenThen\x12.\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12.\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\"`\n\x08\x43\x61stNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"c\n\x0bTryCastNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"[\n\x0cSortExprNode\x12)\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1b.datafusion.LogicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"\xb6\x01\n\x0bWindowFrame\x12\x38\n\x12window_frame_units\x18\x01 \x01(\x0e\x32\x1c.datafusion.WindowFrameUnits\x12\x31\n\x0bstart_bound\x18\x02 \x01(\x0b\x32\x1c.datafusion.WindowFrameBound\x12-\n\x05\x62ound\x18\x03 \x01(\x0b\x32\x1c.datafusion.WindowFrameBoundH\x00\x42\x0b\n\tend_bound\"\x83\x01\n\x10WindowFrameBound\x12\x41\n\x17window_frame_bound_type\x18\x01 \x01(\x0e\x32 .datafusion.WindowFrameBoundType\x12,\n\x0b\x62ound_value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"\x91\x01\n\x06Schema\x12\"\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12\x32\n\x08metadata\x18\x02 \x03(\x0b\x32 .datafusion.Schema.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x82\x02\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\x12\x10\n\x08nullable\x18\x03 \x01(\x08\x12#\n\x08\x63hildren\x18\x04 \x03(\x0b\x32\x11.datafusion.Field\x12\x31\n\x08metadata\x18\x05 \x03(\x0b\x32\x1f.datafusion.Field.MetadataEntry\x12\x0f\n\x07\x64ict_id\x18\x06 \x01(\x03\x12\x14\n\x0c\x64ict_ordered\x18\x07 \x01(\x08\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"!\n\x0f\x46ixedSizeBinary\x12\x0e\n\x06length\x18\x01 \x01(\x05\"F\n\tTimestamp\x12\'\n\ttime_unit\x18\x01 \x01(\x0e\x32\x14.datafusion.TimeUnit\x12\x10\n\x08timezone\x18\x02 \x01(\t\"7\n\x07\x44\x65\x63imal\x12\x11\n\tprecision\x18\x03 \x01(\r\x12\r\n\x05scale\x18\x04 \x01(\x05J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03\"-\n\x04List\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\"I\n\rFixedSizeList\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x11\n\tlist_size\x18\x02 \x01(\x05\"V\n\nDictionary\x12\"\n\x03key\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"4\n\x06Struct\x12*\n\x0fsub_field_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\"A\n\x03Map\x12%\n\nfield_type\x18\x01 \x01(\x0b\x32\x11.datafusion.Field\x12\x13\n\x0bkeys_sorted\x18\x02 \x01(\x08\"l\n\x05Union\x12&\n\x0bunion_types\x18\x01 \x03(\x0b\x32\x11.datafusion.Field\x12)\n\nunion_mode\x18\x02 \x01(\x0e\x32\x15.datafusion.UnionMode\x12\x10\n\x08type_ids\x18\x03 \x03(\x05\"`\n\x11ScalarNestedValue\x12\x13\n\x0bipc_message\x18\x01 \x01(\x0c\x12\x12\n\narrow_data\x18\x02 \x01(\x0c\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\"_\n\x11ScalarTime32Value\x12\x1d\n\x13time32_second_value\x18\x01 \x01(\x05H\x00\x12\"\n\x18time32_millisecond_value\x18\x02 \x01(\x05H\x00\x42\x07\n\x05value\"c\n\x11ScalarTime64Value\x12\"\n\x18time64_microsecond_value\x18\x01 \x01(\x03H\x00\x12!\n\x17time64_nanosecond_value\x18\x02 \x01(\x03H\x00\x42\x07\n\x05value\"\xb3\x01\n\x14ScalarTimestampValue\x12 \n\x16time_microsecond_value\x18\x01 \x01(\x03H\x00\x12\x1f\n\x15time_nanosecond_value\x18\x02 \x01(\x03H\x00\x12\x1b\n\x11time_second_value\x18\x03 \x01(\x03H\x00\x12 \n\x16time_millisecond_value\x18\x04 \x01(\x03H\x00\x12\x10\n\x08timezone\x18\x05 \x01(\tB\x07\n\x05value\"j\n\x15ScalarDictionaryValue\x12)\n\nindex_type\x18\x01 \x01(\x0b\x32\x15.datafusion.ArrowType\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"H\n\x19IntervalMonthDayNanoValue\x12\x0e\n\x06months\x18\x01 \x01(\x05\x12\x0c\n\x04\x64\x61ys\x18\x02 \x01(\x05\x12\r\n\x05nanos\x18\x03 \x01(\x03\"7\n\x15ScalarFixedSizeBinary\x12\x0e\n\x06values\x18\x01 \x01(\x0c\x12\x0e\n\x06length\x18\x02 \x01(\x05\"\x86\x0b\n\x0bScalarValue\x12+\n\nnull_value\x18! \x01(\x0b\x32\x15.datafusion.ArrowTypeH\x00\x12\x14\n\nbool_value\x18\x01 \x01(\x08H\x00\x12\x14\n\nutf8_value\x18\x02 \x01(\tH\x00\x12\x1a\n\x10large_utf8_value\x18\x03 \x01(\tH\x00\x12\x14\n\nint8_value\x18\x04 \x01(\x05H\x00\x12\x15\n\x0bint16_value\x18\x05 \x01(\x05H\x00\x12\x15\n\x0bint32_value\x18\x06 \x01(\x05H\x00\x12\x15\n\x0bint64_value\x18\x07 \x01(\x03H\x00\x12\x15\n\x0buint8_value\x18\x08 \x01(\rH\x00\x12\x16\n\x0cuint16_value\x18\t \x01(\rH\x00\x12\x16\n\x0cuint32_value\x18\n \x01(\rH\x00\x12\x16\n\x0cuint64_value\x18\x0b \x01(\x04H\x00\x12\x17\n\rfloat32_value\x18\x0c \x01(\x02H\x00\x12\x17\n\rfloat64_value\x18\r \x01(\x01H\x00\x12\x17\n\rdate_32_value\x18\x0e \x01(\x05H\x00\x12\x35\n\x0ctime32_value\x18\x0f \x01(\x0b\x32\x1d.datafusion.ScalarTime32ValueH\x00\x12\x39\n\x10large_list_value\x18\x10 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12\x33\n\nlist_value\x18\x11 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12>\n\x15\x66ixed_size_list_value\x18\x12 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12\x35\n\x0cstruct_value\x18 \x01(\x0b\x32\x1d.datafusion.ScalarNestedValueH\x00\x12\x32\n\x10\x64\x65\x63imal128_value\x18\x14 \x01(\x0b\x32\x16.datafusion.Decimal128H\x00\x12\x32\n\x10\x64\x65\x63imal256_value\x18\' \x01(\x0b\x32\x16.datafusion.Decimal256H\x00\x12\x17\n\rdate_64_value\x18\x15 \x01(\x03H\x00\x12\"\n\x18interval_yearmonth_value\x18\x18 \x01(\x05H\x00\x12 \n\x16interval_daytime_value\x18\x19 \x01(\x03H\x00\x12\x1f\n\x15\x64uration_second_value\x18# \x01(\x03H\x00\x12$\n\x1a\x64uration_millisecond_value\x18$ \x01(\x03H\x00\x12$\n\x1a\x64uration_microsecond_value\x18% \x01(\x03H\x00\x12#\n\x19\x64uration_nanosecond_value\x18& \x01(\x03H\x00\x12;\n\x0ftimestamp_value\x18\x1a \x01(\x0b\x32 .datafusion.ScalarTimestampValueH\x00\x12=\n\x10\x64ictionary_value\x18\x1b \x01(\x0b\x32!.datafusion.ScalarDictionaryValueH\x00\x12\x16\n\x0c\x62inary_value\x18\x1c \x01(\x0cH\x00\x12\x1c\n\x12large_binary_value\x18\x1d \x01(\x0cH\x00\x12\x35\n\x0ctime64_value\x18\x1e \x01(\x0b\x32\x1d.datafusion.ScalarTime64ValueH\x00\x12H\n\x17interval_month_day_nano\x18\x1f \x01(\x0b\x32%.datafusion.IntervalMonthDayNanoValueH\x00\x12\x44\n\x17\x66ixed_size_binary_value\x18\" \x01(\x0b\x32!.datafusion.ScalarFixedSizeBinaryH\x00\x42\x07\n\x05valueJ\x04\x08\x13\x10\x14\"1\n\nDecimal128\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\t\n\x01p\x18\x02 \x01(\x03\x12\t\n\x01s\x18\x03 \x01(\x03\"1\n\nDecimal256\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\t\n\x01p\x18\x02 \x01(\x03\x12\t\n\x01s\x18\x03 \x01(\x03\"\x94\x0b\n\tArrowType\x12(\n\x04NONE\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04\x42OOL\x18\x02 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05UINT8\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04INT8\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT16\x18\x05 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT16\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT32\x18\x07 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT32\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06UINT64\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12)\n\x05INT64\x18\n \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT16\x18\x0b \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT32\x18\x0c \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12+\n\x07\x46LOAT64\x18\r \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x04UTF8\x18\x0e \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12.\n\nLARGE_UTF8\x18 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x42INARY\x18\x0f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x1b\n\x11\x46IXED_SIZE_BINARY\x18\x10 \x01(\x05H\x00\x12\x30\n\x0cLARGE_BINARY\x18\x1f \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE32\x18\x11 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12*\n\x06\x44\x41TE64\x18\x12 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12(\n\x08\x44URATION\x18\x13 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12*\n\tTIMESTAMP\x18\x14 \x01(\x0b\x32\x15.datafusion.TimestampH\x00\x12&\n\x06TIME32\x18\x15 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12&\n\x06TIME64\x18\x16 \x01(\x0e\x32\x14.datafusion.TimeUnitH\x00\x12,\n\x08INTERVAL\x18\x17 \x01(\x0e\x32\x18.datafusion.IntervalUnitH\x00\x12&\n\x07\x44\x45\x43IMAL\x18\x18 \x01(\x0b\x32\x13.datafusion.DecimalH\x00\x12 \n\x04LIST\x18\x19 \x01(\x0b\x32\x10.datafusion.ListH\x00\x12&\n\nLARGE_LIST\x18\x1a \x01(\x0b\x32\x10.datafusion.ListH\x00\x12\x34\n\x0f\x46IXED_SIZE_LIST\x18\x1b \x01(\x0b\x32\x19.datafusion.FixedSizeListH\x00\x12$\n\x06STRUCT\x18\x1c \x01(\x0b\x32\x12.datafusion.StructH\x00\x12\"\n\x05UNION\x18\x1d \x01(\x0b\x32\x11.datafusion.UnionH\x00\x12,\n\nDICTIONARY\x18\x1e \x01(\x0b\x32\x16.datafusion.DictionaryH\x00\x12\x1e\n\x03MAP\x18! \x01(\x0b\x32\x0f.datafusion.MapH\x00\x42\x11\n\x0f\x61rrow_type_enum\"\x0e\n\x0c\x45mptyMessage\"0\n\x17\x41nalyzedLogicalPlanType\x12\x15\n\ranalyzer_name\x18\x01 \x01(\t\"2\n\x18OptimizedLogicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"3\n\x19OptimizedPhysicalPlanType\x12\x16\n\x0eoptimizer_name\x18\x01 \x01(\t\"\x8c\x05\n\x08PlanType\x12\x36\n\x12InitialLogicalPlan\x18\x01 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x42\n\x13\x41nalyzedLogicalPlan\x18\x07 \x01(\x0b\x32#.datafusion.AnalyzedLogicalPlanTypeH\x00\x12<\n\x18\x46inalAnalyzedLogicalPlan\x18\x08 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x44\n\x14OptimizedLogicalPlan\x18\x02 \x01(\x0b\x32$.datafusion.OptimizedLogicalPlanTypeH\x00\x12\x34\n\x10\x46inalLogicalPlan\x18\x03 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x37\n\x13InitialPhysicalPlan\x18\x04 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12@\n\x1cInitialPhysicalPlanWithStats\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x46\n\x15OptimizedPhysicalPlan\x18\x05 \x01(\x0b\x32%.datafusion.OptimizedPhysicalPlanTypeH\x00\x12\x35\n\x11\x46inalPhysicalPlan\x18\x06 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12>\n\x1a\x46inalPhysicalPlanWithStats\x18\n \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x42\x10\n\x0eplan_type_enum\"H\n\x0fStringifiedPlan\x12\'\n\tplan_type\x18\x01 \x01(\x0b\x32\x14.datafusion.PlanType\x12\x0c\n\x04plan\x18\x02 \x01(\t\"#\n\x12\x42\x61reTableReference\x12\r\n\x05table\x18\x01 \x01(\t\"6\n\x15PartialTableReference\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\r\n\x05table\x18\x02 \x01(\t\"D\n\x12\x46ullTableReference\x12\x0f\n\x07\x63\x61talog\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\xc3\x01\n\x13OwnedTableReference\x12.\n\x04\x62\x61re\x18\x01 \x01(\x0b\x32\x1e.datafusion.BareTableReferenceH\x00\x12\x34\n\x07partial\x18\x02 \x01(\x0b\x32!.datafusion.PartialTableReferenceH\x00\x12.\n\x04\x66ull\x18\x03 \x01(\x0b\x32\x1e.datafusion.FullTableReferenceH\x00\x42\x16\n\x14table_reference_enum\"\x92\x0c\n\x10PhysicalPlanNode\x12\x37\n\x0cparquet_scan\x18\x01 \x01(\x0b\x32\x1f.datafusion.ParquetScanExecNodeH\x00\x12/\n\x08\x63sv_scan\x18\x02 \x01(\x0b\x32\x1b.datafusion.CsvScanExecNodeH\x00\x12*\n\x05\x65mpty\x18\x03 \x01(\x0b\x32\x19.datafusion.EmptyExecNodeH\x00\x12\x34\n\nprojection\x18\x04 \x01(\x0b\x32\x1e.datafusion.ProjectionExecNodeH\x00\x12\x37\n\x0cglobal_limit\x18\x06 \x01(\x0b\x32\x1f.datafusion.GlobalLimitExecNodeH\x00\x12\x35\n\x0blocal_limit\x18\x07 \x01(\x0b\x32\x1e.datafusion.LocalLimitExecNodeH\x00\x12\x32\n\taggregate\x18\x08 \x01(\x0b\x32\x1d.datafusion.AggregateExecNodeH\x00\x12\x31\n\thash_join\x18\t \x01(\x0b\x32\x1c.datafusion.HashJoinExecNodeH\x00\x12(\n\x04sort\x18\n \x01(\x0b\x32\x18.datafusion.SortExecNodeH\x00\x12?\n\x10\x63oalesce_batches\x18\x0b \x01(\x0b\x32#.datafusion.CoalesceBatchesExecNodeH\x00\x12,\n\x06\x66ilter\x18\x0c \x01(\x0b\x32\x1a.datafusion.FilterExecNodeH\x00\x12\x37\n\x05merge\x18\r \x01(\x0b\x32&.datafusion.CoalescePartitionsExecNodeH\x00\x12\x36\n\x0brepartition\x18\x0e \x01(\x0b\x32\x1f.datafusion.RepartitionExecNodeH\x00\x12/\n\x06window\x18\x0f \x01(\x0b\x32\x1d.datafusion.WindowAggExecNodeH\x00\x12\x33\n\ncross_join\x18\x10 \x01(\x0b\x32\x1d.datafusion.CrossJoinExecNodeH\x00\x12\x31\n\tavro_scan\x18\x11 \x01(\x0b\x32\x1c.datafusion.AvroScanExecNodeH\x00\x12\x36\n\textension\x18\x12 \x01(\x0b\x32!.datafusion.PhysicalExtensionNodeH\x00\x12*\n\x05union\x18\x13 \x01(\x0b\x32\x19.datafusion.UnionExecNodeH\x00\x12.\n\x07\x65xplain\x18\x14 \x01(\x0b\x32\x1b.datafusion.ExplainExecNodeH\x00\x12H\n\x15sort_preserving_merge\x18\x15 \x01(\x0b\x32\'.datafusion.SortPreservingMergeExecNodeH\x00\x12>\n\x10nested_loop_join\x18\x16 \x01(\x0b\x32\".datafusion.NestedLoopJoinExecNodeH\x00\x12.\n\x07\x61nalyze\x18\x17 \x01(\x0b\x32\x1b.datafusion.AnalyzeExecNodeH\x00\x12\x31\n\tjson_sink\x18\x18 \x01(\x0b\x32\x1c.datafusion.JsonSinkExecNodeH\x00\x12\x44\n\x13symmetric_hash_join\x18\x19 \x01(\x0b\x32%.datafusion.SymmetricHashJoinExecNodeH\x00\x12\x34\n\ninterleave\x18\x1a \x01(\x0b\x32\x1e.datafusion.InterleaveExecNodeH\x00\x12=\n\x0fplaceholder_row\x18\x1b \x01(\x0b\x32\".datafusion.PlaceholderRowExecNodeH\x00\x12/\n\x08\x63sv_sink\x18\x1c \x01(\x0b\x32\x1b.datafusion.CsvSinkExecNodeH\x00\x12\x37\n\x0cparquet_sink\x18\x1d \x01(\x0b\x32\x1f.datafusion.ParquetSinkExecNodeH\x00\x42\x12\n\x10PhysicalPlanType\"J\n\x0fPartitionColumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"\x85\x02\n\x15\x46ileTypeWriterOptions\x12\x35\n\x0cjson_options\x18\x01 \x01(\x0b\x32\x1d.datafusion.JsonWriterOptionsH\x00\x12;\n\x0fparquet_options\x18\x02 \x01(\x0b\x32 .datafusion.ParquetWriterOptionsH\x00\x12\x33\n\x0b\x63sv_options\x18\x03 \x01(\x0b\x32\x1c.datafusion.CsvWriterOptionsH\x00\x12\x37\n\rarrow_options\x18\x04 \x01(\x0b\x32\x1e.datafusion.ArrowWriterOptionsH\x00\x42\n\n\x08\x46ileType\"L\n\x11JsonWriterOptions\x12\x37\n\x0b\x63ompression\x18\x01 \x01(\x0e\x32\".datafusion.CompressionTypeVariant\"O\n\x14ParquetWriterOptions\x12\x37\n\x11writer_properties\x18\x01 \x01(\x0b\x32\x1c.datafusion.WriterProperties\"\xe3\x01\n\x10\x43svWriterOptions\x12\x37\n\x0b\x63ompression\x18\x01 \x01(\x0e\x32\".datafusion.CompressionTypeVariant\x12\x11\n\tdelimiter\x18\x02 \x01(\t\x12\x12\n\nhas_header\x18\x03 \x01(\x08\x12\x13\n\x0b\x64\x61te_format\x18\x04 \x01(\t\x12\x17\n\x0f\x64\x61tetime_format\x18\x05 \x01(\t\x12\x18\n\x10timestamp_format\x18\x06 \x01(\t\x12\x13\n\x0btime_format\x18\x07 \x01(\t\x12\x12\n\nnull_value\x18\x08 \x01(\t\"\x14\n\x12\x41rrowWriterOptions\"\xd9\x01\n\x10WriterProperties\x12\x1c\n\x14\x64\x61ta_page_size_limit\x18\x01 \x01(\x04\x12\"\n\x1a\x64ictionary_page_size_limit\x18\x02 \x01(\x04\x12!\n\x19\x64\x61ta_page_row_count_limit\x18\x03 \x01(\x04\x12\x18\n\x10write_batch_size\x18\x04 \x01(\x04\x12\x1a\n\x12max_row_group_size\x18\x05 \x01(\x04\x12\x16\n\x0ewriter_version\x18\x06 \x01(\t\x12\x12\n\ncreated_by\x18\x07 \x01(\t\"\xb5\x02\n\x0e\x46ileSinkConfig\x12\x18\n\x10object_store_url\x18\x01 \x01(\t\x12\x30\n\x0b\x66ile_groups\x18\x02 \x03(\x0b\x32\x1b.datafusion.PartitionedFile\x12\x13\n\x0btable_paths\x18\x03 \x03(\t\x12)\n\routput_schema\x18\x04 \x01(\x0b\x32\x12.datafusion.Schema\x12\x39\n\x14table_partition_cols\x18\x05 \x03(\x0b\x32\x1b.datafusion.PartitionColumn\x12\x11\n\toverwrite\x18\x08 \x01(\x08\x12\x43\n\x18\x66ile_type_writer_options\x18\t \x01(\x0b\x32!.datafusion.FileTypeWriterOptionsJ\x04\x08\x06\x10\x07\"6\n\x08JsonSink\x12*\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x1a.datafusion.FileSinkConfig\"\xcc\x01\n\x10JsonSinkExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\"\n\x04sink\x18\x02 \x01(\x0b\x32\x14.datafusion.JsonSink\x12\'\n\x0bsink_schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12>\n\nsort_order\x18\x04 \x01(\x0b\x32*.datafusion.PhysicalSortExprNodeCollection\"5\n\x07\x43svSink\x12*\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x1a.datafusion.FileSinkConfig\"\xca\x01\n\x0f\x43svSinkExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12!\n\x04sink\x18\x02 \x01(\x0b\x32\x13.datafusion.CsvSink\x12\'\n\x0bsink_schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12>\n\nsort_order\x18\x04 \x01(\x0b\x32*.datafusion.PhysicalSortExprNodeCollection\"9\n\x0bParquetSink\x12*\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x1a.datafusion.FileSinkConfig\"\xd2\x01\n\x13ParquetSinkExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12%\n\x04sink\x18\x02 \x01(\x0b\x32\x17.datafusion.ParquetSink\x12\'\n\x0bsink_schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\x12>\n\nsort_order\x18\x04 \x01(\x0b\x32*.datafusion.PhysicalSortExprNodeCollection\"S\n\x15PhysicalExtensionNode\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12,\n\x06inputs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"\x80\x08\n\x10PhysicalExprNode\x12,\n\x06\x63olumn\x18\x01 \x01(\x0b\x32\x1a.datafusion.PhysicalColumnH\x00\x12*\n\x07literal\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValueH\x00\x12\x39\n\x0b\x62inary_expr\x18\x03 \x01(\x0b\x32\".datafusion.PhysicalBinaryExprNodeH\x00\x12?\n\x0e\x61ggregate_expr\x18\x04 \x01(\x0b\x32%.datafusion.PhysicalAggregateExprNodeH\x00\x12\x32\n\x0cis_null_expr\x18\x05 \x01(\x0b\x32\x1a.datafusion.PhysicalIsNullH\x00\x12\x39\n\x10is_not_null_expr\x18\x06 \x01(\x0b\x32\x1d.datafusion.PhysicalIsNotNullH\x00\x12+\n\x08not_expr\x18\x07 \x01(\x0b\x32\x17.datafusion.PhysicalNotH\x00\x12-\n\x05\x63\x61se_\x18\x08 \x01(\x0b\x32\x1c.datafusion.PhysicalCaseNodeH\x00\x12,\n\x04\x63\x61st\x18\t \x01(\x0b\x32\x1c.datafusion.PhysicalCastNodeH\x00\x12\x30\n\x04sort\x18\n \x01(\x0b\x32 .datafusion.PhysicalSortExprNodeH\x00\x12\x34\n\x08negative\x18\x0b \x01(\x0b\x32 .datafusion.PhysicalNegativeNodeH\x00\x12\x31\n\x07in_list\x18\x0c \x01(\x0b\x32\x1e.datafusion.PhysicalInListNodeH\x00\x12\x41\n\x0fscalar_function\x18\r \x01(\x0b\x32&.datafusion.PhysicalScalarFunctionNodeH\x00\x12\x33\n\x08try_cast\x18\x0e \x01(\x0b\x32\x1f.datafusion.PhysicalTryCastNodeH\x00\x12\x39\n\x0bwindow_expr\x18\x0f \x01(\x0b\x32\".datafusion.PhysicalWindowExprNodeH\x00\x12\x37\n\nscalar_udf\x18\x10 \x01(\x0b\x32!.datafusion.PhysicalScalarUdfNodeH\x00\x12\x35\n\tlike_expr\x18\x12 \x01(\x0b\x32 .datafusion.PhysicalLikeExprNodeH\x00\x12M\n\x16get_indexed_field_expr\x18\x13 \x01(\x0b\x32+.datafusion.PhysicalGetIndexedFieldExprNodeH\x00\x42\n\n\x08\x45xprTypeJ\x04\x08\x11\x10\x12\"}\n\x15PhysicalScalarUdfNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12*\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"\x84\x02\n\x19PhysicalAggregateExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12$\n\x1auser_defined_aggr_function\x18\x04 \x01(\tH\x00\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x36\n\x0cordering_req\x18\x05 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12\x10\n\x08\x64istinct\x18\x03 \x01(\x08\x42\x13\n\x11\x41ggregateFunction\"\xf4\x02\n\x16PhysicalWindowExprNode\x12\x36\n\raggr_function\x18\x01 \x01(\x0e\x32\x1d.datafusion.AggregateFunctionH\x00\x12>\n\x11\x62uilt_in_function\x18\x02 \x01(\x0e\x32!.datafusion.BuiltInWindowFunctionH\x00\x12*\n\x04\x61rgs\x18\x04 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x32\n\x0cpartition_by\x18\x05 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x32\n\x08order_by\x18\x06 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12-\n\x0cwindow_frame\x18\x07 \x01(\x0b\x32\x17.datafusion.WindowFrame\x12\x0c\n\x04name\x18\x08 \x01(\tB\x11\n\x0fwindow_function\"<\n\x0ePhysicalIsNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"?\n\x11PhysicalIsNotNull\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"9\n\x0bPhysicalNot\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"N\n\x11PhysicalAliasNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"v\n\x16PhysicalBinaryExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x80\x01\n PhysicalDateTimeIntervalExprNode\x12\'\n\x01l\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x01r\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\n\n\x02op\x18\x03 \x01(\t\"\x9c\x01\n\x14PhysicalLikeExprNode\x12\x0f\n\x07negated\x18\x01 \x01(\x08\x12\x18\n\x10\x63\x61se_insensitive\x18\x02 \x01(\x08\x12*\n\x04\x65xpr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12-\n\x07pattern\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"d\n\x14PhysicalSortExprNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0b\n\x03\x61sc\x18\x02 \x01(\x08\x12\x13\n\x0bnulls_first\x18\x03 \x01(\x08\"t\n\x10PhysicalWhenThen\x12/\n\twhen_expr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\tthen_expr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"}\n\x12PhysicalInListNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x04list\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0f\n\x07negated\x18\x03 \x01(\x08\"\xa5\x01\n\x10PhysicalCaseNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x34\n\x0ewhen_then_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalWhenThen\x12/\n\telse_expr\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\xab\x01\n\x1aPhysicalScalarFunctionNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\'\n\x03\x66un\x18\x02 \x01(\x0e\x32\x1a.datafusion.ScalarFunction\x12*\n\x04\x61rgs\x18\x03 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x0breturn_type\x18\x04 \x01(\x0b\x32\x15.datafusion.ArrowType\"l\n\x13PhysicalTryCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"i\n\x10PhysicalCastNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12)\n\narrow_type\x18\x02 \x01(\x0b\x32\x15.datafusion.ArrowType\"B\n\x14PhysicalNegativeNode\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\x8d\x01\n\x0e\x46ilterExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\"\n\x1a\x64\x65\x66\x61ult_filter_selectivity\x18\x03 \x01(\r\"7\n\tFileGroup\x12*\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x1b.datafusion.PartitionedFile\"\x1a\n\tScanLimit\x12\r\n\x05limit\x18\x01 \x01(\r\"d\n\x1ePhysicalSortExprNodeCollection\x12\x42\n\x18physical_sort_expr_nodes\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\xcb\x02\n\x10\x46ileScanExecConf\x12*\n\x0b\x66ile_groups\x18\x01 \x03(\x0b\x32\x15.datafusion.FileGroup\x12\"\n\x06schema\x18\x02 \x01(\x0b\x32\x12.datafusion.Schema\x12\x12\n\nprojection\x18\x04 \x03(\r\x12$\n\x05limit\x18\x05 \x01(\x0b\x32\x15.datafusion.ScanLimit\x12*\n\nstatistics\x18\x06 \x01(\x0b\x32\x16.datafusion.Statistics\x12\x1c\n\x14table_partition_cols\x18\x07 \x03(\t\x12\x18\n\x10object_store_url\x18\x08 \x01(\t\x12\x43\n\x0foutput_ordering\x18\t \x03(\x0b\x32*.datafusion.PhysicalSortExprNodeCollectionJ\x04\x08\n\x10\x0b\"}\n\x13ParquetScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12/\n\tpredicate\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNodeJ\x04\x08\x02\x10\x03\"\x9d\x01\n\x0f\x43svScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\x12\x12\n\nhas_header\x18\x02 \x01(\x08\x12\x11\n\tdelimiter\x18\x03 \x01(\t\x12\r\n\x05quote\x18\x04 \x01(\t\x12\x10\n\x06\x65scape\x18\x05 \x01(\tH\x00\x42\x11\n\x0foptional_escape\"C\n\x10\x41vroScanExecNode\x12/\n\tbase_conf\x18\x01 \x01(\x0b\x32\x1c.datafusion.FileScanExecConf\"\xa9\x02\n\x10HashJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x1e\n\x02on\x18\x03 \x03(\x0b\x32\x12.datafusion.JoinOn\x12\'\n\tjoin_type\x18\x04 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x31\n\x0epartition_mode\x18\x06 \x01(\x0e\x32\x19.datafusion.PartitionMode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12&\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x16.datafusion.JoinFilter\"\xaf\x03\n\x19SymmetricHashJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x1e\n\x02on\x18\x03 \x03(\x0b\x32\x12.datafusion.JoinOn\x12\'\n\tjoin_type\x18\x04 \x01(\x0e\x32\x14.datafusion.JoinType\x12\x37\n\x0epartition_mode\x18\x06 \x01(\x0e\x32\x1f.datafusion.StreamPartitionMode\x12\x18\n\x10null_equals_null\x18\x07 \x01(\x08\x12&\n\x06\x66ilter\x18\x08 \x01(\x0b\x32\x16.datafusion.JoinFilter\x12\x39\n\x0fleft_sort_exprs\x18\t \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\x12:\n\x10right_sort_exprs\x18\n \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"B\n\x12InterleaveExecNode\x12,\n\x06inputs\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"=\n\rUnionExecNode\x12,\n\x06inputs\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"~\n\x0f\x45xplainExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\x12\x36\n\x11stringified_plans\x18\x02 \x03(\x0b\x32\x1b.datafusion.StringifiedPlan\x12\x0f\n\x07verbose\x18\x03 \x01(\x08\"\x8c\x01\n\x0f\x41nalyzeExecNode\x12\x0f\n\x07verbose\x18\x01 \x01(\x08\x12\x17\n\x0fshow_statistics\x18\x02 \x01(\x08\x12+\n\x05input\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\"\n\x06schema\x18\x04 \x01(\x0b\x32\x12.datafusion.Schema\"l\n\x11\x43rossJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"-\n\x0ePhysicalColumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\r\"a\n\x06JoinOn\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"3\n\rEmptyExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\"<\n\x16PlaceholderRowExecNode\x12\"\n\x06schema\x18\x01 \x01(\x0b\x32\x12.datafusion.Schema\"\x80\x01\n\x12ProjectionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x11\n\texpr_name\x18\x03 \x03(\t\"0\n\x1dPartiallySortedInputOrderMode\x12\x0f\n\x07\x63olumns\x18\x06 \x03(\x04\"\xe2\x02\n\x11WindowAggExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x37\n\x0bwindow_expr\x18\x02 \x03(\x0b\x32\".datafusion.PhysicalWindowExprNode\x12\x34\n\x0epartition_keys\x18\x05 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x06linear\x18\x07 \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x12\x45\n\x10partially_sorted\x18\x08 \x01(\x0b\x32).datafusion.PartiallySortedInputOrderModeH\x00\x12*\n\x06sorted\x18\t \x01(\x0b\x32\x18.datafusion.EmptyMessageH\x00\x42\x12\n\x10input_order_mode\"9\n\x0bMaybeFilter\x12*\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"M\n\x16MaybePhysicalSortExprs\x12\x33\n\tsort_expr\x18\x01 \x03(\x0b\x32 .datafusion.PhysicalSortExprNode\"\x96\x03\n\x11\x41ggregateExecNode\x12\x30\n\ngroup_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\taggr_expr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\'\n\x04mode\x18\x03 \x01(\x0e\x32\x19.datafusion.AggregateMode\x12+\n\x05input\x18\x04 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x17\n\x0fgroup_expr_name\x18\x05 \x03(\t\x12\x16\n\x0e\x61ggr_expr_name\x18\x06 \x03(\t\x12(\n\x0cinput_schema\x18\x07 \x01(\x0b\x32\x12.datafusion.Schema\x12/\n\tnull_expr\x18\x08 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x0e\n\x06groups\x18\t \x03(\x08\x12,\n\x0b\x66ilter_expr\x18\n \x03(\x0b\x32\x17.datafusion.MaybeFilter\"_\n\x13GlobalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x0c\n\x04skip\x18\x02 \x01(\r\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"P\n\x12LocalLimitExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\r\n\x05\x66\x65tch\x18\x02 \x01(\r\"\x95\x01\n\x0cSortExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\x12\x1d\n\x15preserve_partitioning\x18\x04 \x01(\x08\"\x85\x01\n\x1bSortPreservingMergeExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12*\n\x04\x65xpr\x18\x02 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\r\n\x05\x66\x65tch\x18\x03 \x01(\x03\"\xc2\x01\n\x16NestedLoopJoinExecNode\x12*\n\x04left\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12+\n\x05right\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\'\n\tjoin_type\x18\x03 \x01(\x0e\x32\x14.datafusion.JoinType\x12&\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x16.datafusion.JoinFilter\"a\n\x17\x43oalesceBatchesExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x19\n\x11target_batch_size\x18\x02 \x01(\r\"I\n\x1a\x43oalescePartitionsExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\"c\n\x17PhysicalHashRepartition\x12/\n\thash_expr\x18\x01 \x03(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x17\n\x0fpartition_count\x18\x02 \x01(\x04\"\xb5\x01\n\x13RepartitionExecNode\x12+\n\x05input\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalPlanNode\x12\x15\n\x0bround_robin\x18\x02 \x01(\x04H\x00\x12\x33\n\x04hash\x18\x03 \x01(\x0b\x32#.datafusion.PhysicalHashRepartitionH\x00\x12\x11\n\x07unknown\x18\x04 \x01(\x04H\x00\x42\x12\n\x10partition_method\"\x93\x01\n\nJoinFilter\x12\x30\n\nexpression\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12/\n\x0e\x63olumn_indices\x18\x02 \x03(\x0b\x32\x17.datafusion.ColumnIndex\x12\"\n\x06schema\x18\x03 \x01(\x0b\x32\x12.datafusion.Schema\"@\n\x0b\x43olumnIndex\x12\r\n\x05index\x18\x01 \x01(\r\x12\"\n\x04side\x18\x02 \x01(\x0e\x32\x14.datafusion.JoinSide\"\xa0\x01\n\x0fPartitionedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\x12\x18\n\x10last_modified_ns\x18\x03 \x01(\x04\x12\x31\n\x10partition_values\x18\x04 \x03(\x0b\x32\x17.datafusion.ScalarValue\x12$\n\x05range\x18\x05 \x01(\x0b\x32\x15.datafusion.FileRange\"\'\n\tFileRange\x12\r\n\x05start\x18\x01 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x03\"y\n\x0ePartitionStats\x12\x10\n\x08num_rows\x18\x01 \x01(\x03\x12\x13\n\x0bnum_batches\x18\x02 \x01(\x03\x12\x11\n\tnum_bytes\x18\x03 \x01(\x03\x12-\n\x0c\x63olumn_stats\x18\x04 \x03(\x0b\x32\x17.datafusion.ColumnStats\"d\n\tPrecision\x12\x31\n\x0eprecision_info\x18\x01 \x01(\x0e\x32\x19.datafusion.PrecisionInfo\x12$\n\x03val\x18\x02 \x01(\x0b\x32\x17.datafusion.ScalarValue\"\x94\x01\n\nStatistics\x12\'\n\x08num_rows\x18\x01 \x01(\x0b\x32\x15.datafusion.Precision\x12.\n\x0ftotal_byte_size\x18\x02 \x01(\x0b\x32\x15.datafusion.Precision\x12-\n\x0c\x63olumn_stats\x18\x03 \x03(\x0b\x32\x17.datafusion.ColumnStats\"\xbb\x01\n\x0b\x43olumnStats\x12(\n\tmin_value\x18\x01 \x01(\x0b\x32\x15.datafusion.Precision\x12(\n\tmax_value\x18\x02 \x01(\x0b\x32\x15.datafusion.Precision\x12)\n\nnull_count\x18\x03 \x01(\x0b\x32\x15.datafusion.Precision\x12-\n\x0e\x64istinct_count\x18\x04 \x01(\x0b\x32\x15.datafusion.Precision\"=\n\x14NamedStructFieldExpr\x12%\n\x04name\x18\x01 \x01(\x0b\x32\x17.datafusion.ScalarValue\":\n\rListIndexExpr\x12)\n\x03key\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\x96\x01\n\rListRangeExpr\x12+\n\x05start\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12*\n\x04stop\x18\x02 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12,\n\x06stride\x18\x03 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\"\x86\x02\n\x1fPhysicalGetIndexedFieldExprNode\x12)\n\x03\x61rg\x18\x01 \x01(\x0b\x32\x1c.datafusion.PhysicalExprNode\x12\x43\n\x17named_struct_field_expr\x18\x02 \x01(\x0b\x32 .datafusion.NamedStructFieldExprH\x00\x12\x34\n\x0flist_index_expr\x18\x03 \x01(\x0b\x32\x19.datafusion.ListIndexExprH\x00\x12\x34\n\x0flist_range_expr\x18\x04 \x01(\x0b\x32\x19.datafusion.ListRangeExprH\x00\x42\x07\n\x05\x66ield*n\n\x08JoinType\x12\t\n\x05INNER\x10\x00\x12\x08\n\x04LEFT\x10\x01\x12\t\n\x05RIGHT\x10\x02\x12\x08\n\x04\x46ULL\x10\x03\x12\x0c\n\x08LEFTSEMI\x10\x04\x12\x0c\n\x08LEFTANTI\x10\x05\x12\r\n\tRIGHTSEMI\x10\x06\x12\r\n\tRIGHTANTI\x10\x07*#\n\x0eJoinConstraint\x12\x06\n\x02ON\x10\x00\x12\t\n\x05USING\x10\x01*\xc0\x0e\n\x0eScalarFunction\x12\x07\n\x03\x41\x62s\x10\x00\x12\x08\n\x04\x41\x63os\x10\x01\x12\x08\n\x04\x41sin\x10\x02\x12\x08\n\x04\x41tan\x10\x03\x12\t\n\x05\x41scii\x10\x04\x12\x08\n\x04\x43\x65il\x10\x05\x12\x07\n\x03\x43os\x10\x06\x12\n\n\x06\x44igest\x10\x07\x12\x07\n\x03\x45xp\x10\x08\x12\t\n\x05\x46loor\x10\t\x12\x06\n\x02Ln\x10\n\x12\x07\n\x03Log\x10\x0b\x12\t\n\x05Log10\x10\x0c\x12\x08\n\x04Log2\x10\r\x12\t\n\x05Round\x10\x0e\x12\n\n\x06Signum\x10\x0f\x12\x07\n\x03Sin\x10\x10\x12\x08\n\x04Sqrt\x10\x11\x12\x07\n\x03Tan\x10\x12\x12\t\n\x05Trunc\x10\x13\x12\t\n\x05\x41rray\x10\x14\x12\x0f\n\x0bRegexpMatch\x10\x15\x12\r\n\tBitLength\x10\x16\x12\t\n\x05\x42trim\x10\x17\x12\x13\n\x0f\x43haracterLength\x10\x18\x12\x07\n\x03\x43hr\x10\x19\x12\n\n\x06\x43oncat\x10\x1a\x12\x17\n\x13\x43oncatWithSeparator\x10\x1b\x12\x0c\n\x08\x44\x61tePart\x10\x1c\x12\r\n\tDateTrunc\x10\x1d\x12\x0b\n\x07InitCap\x10\x1e\x12\x08\n\x04Left\x10\x1f\x12\x08\n\x04Lpad\x10 \x12\t\n\x05Lower\x10!\x12\t\n\x05Ltrim\x10\"\x12\x07\n\x03MD5\x10#\x12\x0f\n\x0bOctetLength\x10%\x12\n\n\x06Random\x10&\x12\x11\n\rRegexpReplace\x10\'\x12\n\n\x06Repeat\x10(\x12\x0b\n\x07Replace\x10)\x12\x0b\n\x07Reverse\x10*\x12\t\n\x05Right\x10+\x12\x08\n\x04Rpad\x10,\x12\t\n\x05Rtrim\x10-\x12\n\n\x06SHA224\x10.\x12\n\n\x06SHA256\x10/\x12\n\n\x06SHA384\x10\x30\x12\n\n\x06SHA512\x10\x31\x12\r\n\tSplitPart\x10\x32\x12\x0e\n\nStartsWith\x10\x33\x12\n\n\x06Strpos\x10\x34\x12\n\n\x06Substr\x10\x35\x12\t\n\x05ToHex\x10\x36\x12\x0f\n\x0bToTimestamp\x10\x37\x12\x15\n\x11ToTimestampMillis\x10\x38\x12\x15\n\x11ToTimestampMicros\x10\x39\x12\x16\n\x12ToTimestampSeconds\x10:\x12\x07\n\x03Now\x10;\x12\r\n\tTranslate\x10<\x12\x08\n\x04Trim\x10=\x12\t\n\x05Upper\x10>\x12\x0c\n\x08\x43oalesce\x10?\x12\t\n\x05Power\x10@\x12\r\n\tStructFun\x10\x41\x12\x10\n\x0c\x46romUnixtime\x10\x42\x12\t\n\x05\x41tan2\x10\x43\x12\x0b\n\x07\x44\x61teBin\x10\x44\x12\x0f\n\x0b\x41rrowTypeof\x10\x45\x12\x0f\n\x0b\x43urrentDate\x10\x46\x12\x0f\n\x0b\x43urrentTime\x10G\x12\x08\n\x04Uuid\x10H\x12\x08\n\x04\x43\x62rt\x10I\x12\t\n\x05\x41\x63osh\x10J\x12\t\n\x05\x41sinh\x10K\x12\t\n\x05\x41tanh\x10L\x12\x08\n\x04Sinh\x10M\x12\x08\n\x04\x43osh\x10N\x12\x08\n\x04Tanh\x10O\x12\x06\n\x02Pi\x10P\x12\x0b\n\x07\x44\x65grees\x10Q\x12\x0b\n\x07Radians\x10R\x12\r\n\tFactorial\x10S\x12\x07\n\x03Lcm\x10T\x12\x07\n\x03Gcd\x10U\x12\x0f\n\x0b\x41rrayAppend\x10V\x12\x0f\n\x0b\x41rrayConcat\x10W\x12\r\n\tArrayDims\x10X\x12\x0f\n\x0b\x41rrayRepeat\x10Y\x12\x0f\n\x0b\x41rrayLength\x10Z\x12\x0e\n\nArrayNdims\x10[\x12\x11\n\rArrayPosition\x10\\\x12\x12\n\x0e\x41rrayPositions\x10]\x12\x10\n\x0c\x41rrayPrepend\x10^\x12\x0f\n\x0b\x41rrayRemove\x10_\x12\x10\n\x0c\x41rrayReplace\x10`\x12\x0f\n\x0b\x43\x61rdinality\x10\x62\x12\x10\n\x0c\x41rrayElement\x10\x63\x12\x0e\n\nArraySlice\x10\x64\x12\x07\n\x03\x43ot\x10g\x12\x0c\n\x08\x41rrayHas\x10h\x12\x0f\n\x0b\x41rrayHasAny\x10i\x12\x0f\n\x0b\x41rrayHasAll\x10j\x12\x10\n\x0c\x41rrayRemoveN\x10k\x12\x11\n\rArrayReplaceN\x10l\x12\x12\n\x0e\x41rrayRemoveAll\x10m\x12\x13\n\x0f\x41rrayReplaceAll\x10n\x12\t\n\x05Nanvl\x10o\x12\x0b\n\x07\x46latten\x10p\x12\n\n\x06Iszero\x10r\x12\x0e\n\nArrayEmpty\x10s\x12\x10\n\x0c\x41rrayPopBack\x10t\x12\x11\n\rStringToArray\x10u\x12\x14\n\x10ToTimestampNanos\x10v\x12\x12\n\x0e\x41rrayIntersect\x10w\x12\x0e\n\nArrayUnion\x10x\x12\x0b\n\x07OverLay\x10y\x12\t\n\x05Range\x10z\x12\x0f\n\x0b\x41rrayExcept\x10{\x12\x11\n\rArrayPopFront\x10|\x12\x0f\n\x0bLevenshtein\x10}\x12\x0f\n\x0bSubstrIndex\x10~\x12\r\n\tFindInSet\x10\x7f\x12\x0e\n\tArraySort\x10\x80\x01\x12\x12\n\rArrayDistinct\x10\x81\x01\x12\x10\n\x0b\x41rrayResize\x10\x82\x01\x12\r\n\x08\x45ndsWith\x10\x83\x01\x12\n\n\x05InStr\x10\x84\x01\x12\r\n\x08MakeDate\x10\x85\x01\x12\x11\n\x0c\x41rrayReverse\x10\x86\x01\x12\x0f\n\nRegexpLike\x10\x87\x01\x12\x0b\n\x06ToChar\x10\x88\x01*\xde\x04\n\x11\x41ggregateFunction\x12\x07\n\x03MIN\x10\x00\x12\x07\n\x03MAX\x10\x01\x12\x07\n\x03SUM\x10\x02\x12\x07\n\x03\x41VG\x10\x03\x12\t\n\x05\x43OUNT\x10\x04\x12\x13\n\x0f\x41PPROX_DISTINCT\x10\x05\x12\r\n\tARRAY_AGG\x10\x06\x12\x0c\n\x08VARIANCE\x10\x07\x12\x10\n\x0cVARIANCE_POP\x10\x08\x12\x0e\n\nCOVARIANCE\x10\t\x12\x12\n\x0e\x43OVARIANCE_POP\x10\n\x12\n\n\x06STDDEV\x10\x0b\x12\x0e\n\nSTDDEV_POP\x10\x0c\x12\x0f\n\x0b\x43ORRELATION\x10\r\x12\x1a\n\x16\x41PPROX_PERCENTILE_CONT\x10\x0e\x12\x11\n\rAPPROX_MEDIAN\x10\x0f\x12&\n\"APPROX_PERCENTILE_CONT_WITH_WEIGHT\x10\x10\x12\x0c\n\x08GROUPING\x10\x11\x12\n\n\x06MEDIAN\x10\x12\x12\x0b\n\x07\x42IT_AND\x10\x13\x12\n\n\x06\x42IT_OR\x10\x14\x12\x0b\n\x07\x42IT_XOR\x10\x15\x12\x0c\n\x08\x42OOL_AND\x10\x16\x12\x0b\n\x07\x42OOL_OR\x10\x17\x12\x13\n\x0f\x46IRST_VALUE_AGG\x10\x18\x12\x12\n\x0eLAST_VALUE_AGG\x10\x19\x12\x0e\n\nREGR_SLOPE\x10\x1a\x12\x12\n\x0eREGR_INTERCEPT\x10\x1b\x12\x0e\n\nREGR_COUNT\x10\x1c\x12\x0b\n\x07REGR_R2\x10\x1d\x12\r\n\tREGR_AVGX\x10\x1e\x12\r\n\tREGR_AVGY\x10\x1f\x12\x0c\n\x08REGR_SXX\x10 \x12\x0c\n\x08REGR_SYY\x10!\x12\x0c\n\x08REGR_SXY\x10\"\x12\x0e\n\nSTRING_AGG\x10#\x12\x11\n\rNTH_VALUE_AGG\x10$*\xb0\x01\n\x15\x42uiltInWindowFunction\x12\x0e\n\nROW_NUMBER\x10\x00\x12\x08\n\x04RANK\x10\x01\x12\x0e\n\nDENSE_RANK\x10\x02\x12\x10\n\x0cPERCENT_RANK\x10\x03\x12\r\n\tCUME_DIST\x10\x04\x12\t\n\x05NTILE\x10\x05\x12\x07\n\x03LAG\x10\x06\x12\x08\n\x04LEAD\x10\x07\x12\x0f\n\x0b\x46IRST_VALUE\x10\x08\x12\x0e\n\nLAST_VALUE\x10\t\x12\r\n\tNTH_VALUE\x10\n*3\n\x10WindowFrameUnits\x12\x08\n\x04ROWS\x10\x00\x12\t\n\x05RANGE\x10\x01\x12\n\n\x06GROUPS\x10\x02*E\n\x14WindowFrameBoundType\x12\x0f\n\x0b\x43URRENT_ROW\x10\x00\x12\r\n\tPRECEDING\x10\x01\x12\r\n\tFOLLOWING\x10\x02*(\n\x08\x44\x61teUnit\x12\x07\n\x03\x44\x61y\x10\x00\x12\x13\n\x0f\x44\x61teMillisecond\x10\x01*H\n\x08TimeUnit\x12\n\n\x06Second\x10\x00\x12\x0f\n\x0bMillisecond\x10\x01\x12\x0f\n\x0bMicrosecond\x10\x02\x12\x0e\n\nNanosecond\x10\x03*<\n\x0cIntervalUnit\x12\r\n\tYearMonth\x10\x00\x12\x0b\n\x07\x44\x61yTime\x10\x01\x12\x10\n\x0cMonthDayNano\x10\x02*\"\n\tUnionMode\x12\n\n\x06sparse\x10\x00\x12\t\n\x05\x64\x65nse\x10\x01*Q\n\x16\x43ompressionTypeVariant\x12\x08\n\x04GZIP\x10\x00\x12\t\n\x05\x42ZIP2\x10\x01\x12\x06\n\x02XZ\x10\x02\x12\x08\n\x04ZSTD\x10\x03\x12\x10\n\x0cUNCOMPRESSED\x10\x04*<\n\rPartitionMode\x12\x10\n\x0c\x43OLLECT_LEFT\x10\x00\x12\x0f\n\x0bPARTITIONED\x10\x01\x12\x08\n\x04\x41UTO\x10\x02*A\n\x13StreamPartitionMode\x12\x14\n\x10SINGLE_PARTITION\x10\x00\x12\x14\n\x10PARTITIONED_EXEC\x10\x01*b\n\rAggregateMode\x12\x0b\n\x07PARTIAL\x10\x00\x12\t\n\x05\x46INAL\x10\x01\x12\x15\n\x11\x46INAL_PARTITIONED\x10\x02\x12\n\n\x06SINGLE\x10\x03\x12\x16\n\x12SINGLE_PARTITIONED\x10\x04*)\n\x08JoinSide\x12\r\n\tLEFT_SIDE\x10\x00\x12\x0e\n\nRIGHT_SIDE\x10\x01*3\n\rPrecisionInfo\x12\t\n\x05\x45XACT\x10\x00\x12\x0b\n\x07INEXACT\x10\x01\x12\n\n\x06\x41\x42SENT\x10\x02\x42\x39\n$org.apache.arrow.datafusion.protobufB\x0f\x44\x61tafusionProtoP\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -26,38 +26,46 @@ _DFSCHEMA_METADATAENTRY._serialized_options = b'8\001' _CREATEEXTERNALTABLENODE_OPTIONSENTRY._options = None _CREATEEXTERNALTABLENODE_OPTIONSENTRY._serialized_options = b'8\001' + _CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY._options = None + _CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY._serialized_options = b'8\001' _SCHEMA_METADATAENTRY._options = None _SCHEMA_METADATAENTRY._serialized_options = b'8\001' _FIELD_METADATAENTRY._options = None _FIELD_METADATAENTRY._serialized_options = b'8\001' - _globals['_JOINTYPE']._serialized_start=26361 - _globals['_JOINTYPE']._serialized_end=26471 - _globals['_JOINCONSTRAINT']._serialized_start=26473 - _globals['_JOINCONSTRAINT']._serialized_end=26508 - _globals['_SCALARFUNCTION']._serialized_start=26511 - _globals['_SCALARFUNCTION']._serialized_end=27893 - _globals['_AGGREGATEFUNCTION']._serialized_start=27896 - _globals['_AGGREGATEFUNCTION']._serialized_end=28330 - _globals['_BUILTINWINDOWFUNCTION']._serialized_start=28333 - _globals['_BUILTINWINDOWFUNCTION']._serialized_end=28509 - _globals['_WINDOWFRAMEUNITS']._serialized_start=28511 - _globals['_WINDOWFRAMEUNITS']._serialized_end=28562 - _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_start=28564 - _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_end=28633 - _globals['_DATEUNIT']._serialized_start=28635 - _globals['_DATEUNIT']._serialized_end=28675 - _globals['_TIMEUNIT']._serialized_start=28677 - _globals['_TIMEUNIT']._serialized_end=28749 - _globals['_INTERVALUNIT']._serialized_start=28751 - _globals['_INTERVALUNIT']._serialized_end=28811 - _globals['_UNIONMODE']._serialized_start=28813 - _globals['_UNIONMODE']._serialized_end=28847 - _globals['_PARTITIONMODE']._serialized_start=28849 - _globals['_PARTITIONMODE']._serialized_end=28909 - _globals['_AGGREGATEMODE']._serialized_start=28911 - _globals['_AGGREGATEMODE']._serialized_end=29009 - _globals['_JOINSIDE']._serialized_start=29011 - _globals['_JOINSIDE']._serialized_end=29052 + _globals['_JOINTYPE']._serialized_start=32527 + _globals['_JOINTYPE']._serialized_end=32637 + _globals['_JOINCONSTRAINT']._serialized_start=32639 + _globals['_JOINCONSTRAINT']._serialized_end=32674 + _globals['_SCALARFUNCTION']._serialized_start=32677 + _globals['_SCALARFUNCTION']._serialized_end=34533 + _globals['_AGGREGATEFUNCTION']._serialized_start=34536 + _globals['_AGGREGATEFUNCTION']._serialized_end=35142 + _globals['_BUILTINWINDOWFUNCTION']._serialized_start=35145 + _globals['_BUILTINWINDOWFUNCTION']._serialized_end=35321 + _globals['_WINDOWFRAMEUNITS']._serialized_start=35323 + _globals['_WINDOWFRAMEUNITS']._serialized_end=35374 + _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_start=35376 + _globals['_WINDOWFRAMEBOUNDTYPE']._serialized_end=35445 + _globals['_DATEUNIT']._serialized_start=35447 + _globals['_DATEUNIT']._serialized_end=35487 + _globals['_TIMEUNIT']._serialized_start=35489 + _globals['_TIMEUNIT']._serialized_end=35561 + _globals['_INTERVALUNIT']._serialized_start=35563 + _globals['_INTERVALUNIT']._serialized_end=35623 + _globals['_UNIONMODE']._serialized_start=35625 + _globals['_UNIONMODE']._serialized_end=35659 + _globals['_COMPRESSIONTYPEVARIANT']._serialized_start=35661 + _globals['_COMPRESSIONTYPEVARIANT']._serialized_end=35742 + _globals['_PARTITIONMODE']._serialized_start=35744 + _globals['_PARTITIONMODE']._serialized_end=35804 + _globals['_STREAMPARTITIONMODE']._serialized_start=35806 + _globals['_STREAMPARTITIONMODE']._serialized_end=35871 + _globals['_AGGREGATEMODE']._serialized_start=35873 + _globals['_AGGREGATEMODE']._serialized_end=35971 + _globals['_JOINSIDE']._serialized_start=35973 + _globals['_JOINSIDE']._serialized_end=36014 + _globals['_PRECISIONINFO']._serialized_start=36016 + _globals['_PRECISIONINFO']._serialized_end=36067 _globals['_COLUMNRELATION']._serialized_start=38 _globals['_COLUMNRELATION']._serialized_end=72 _globals['_COLUMN']._serialized_start=74 @@ -69,329 +77,403 @@ _globals['_DFSCHEMA_METADATAENTRY']._serialized_start=341 _globals['_DFSCHEMA_METADATAENTRY']._serialized_end=388 _globals['_LOGICALPLANNODE']._serialized_start=391 - _globals['_LOGICALPLANNODE']._serialized_end=1722 - _globals['_LOGICALEXTENSIONNODE']._serialized_start=1724 - _globals['_LOGICALEXTENSIONNODE']._serialized_end=1805 - _globals['_PROJECTIONCOLUMNS']._serialized_start=1807 - _globals['_PROJECTIONCOLUMNS']._serialized_end=1843 - _globals['_CSVFORMAT']._serialized_start=1845 - _globals['_CSVFORMAT']._serialized_end=1895 - _globals['_PARQUETFORMAT']._serialized_start=1897 - _globals['_PARQUETFORMAT']._serialized_end=1918 - _globals['_AVROFORMAT']._serialized_start=1920 - _globals['_AVROFORMAT']._serialized_end=1932 - _globals['_LOGICALEXPRNODECOLLECTION']._serialized_start=1934 - _globals['_LOGICALEXPRNODECOLLECTION']._serialized_end=2018 - _globals['_LISTINGTABLESCANNODE']._serialized_start=2021 - _globals['_LISTINGTABLESCANNODE']._serialized_end=2559 - _globals['_VIEWTABLESCANNODE']._serialized_start=2562 - _globals['_VIEWTABLESCANNODE']._serialized_end=2791 - _globals['_CUSTOMTABLESCANNODE']._serialized_start=2794 - _globals['_CUSTOMTABLESCANNODE']._serialized_end=3034 - _globals['_PROJECTIONNODE']._serialized_start=3037 - _globals['_PROJECTIONNODE']._serialized_end=3175 - _globals['_SELECTIONNODE']._serialized_start=3177 - _globals['_SELECTIONNODE']._serialized_end=3279 - _globals['_SORTNODE']._serialized_start=3281 - _globals['_SORTNODE']._serialized_end=3393 - _globals['_REPARTITIONNODE']._serialized_start=3396 - _globals['_REPARTITIONNODE']._serialized_end=3545 - _globals['_HASHREPARTITION']._serialized_start=3547 - _globals['_HASHREPARTITION']._serialized_end=3637 - _globals['_EMPTYRELATIONNODE']._serialized_start=3639 - _globals['_EMPTYRELATIONNODE']._serialized_end=3683 - _globals['_CREATEEXTERNALTABLENODE']._serialized_start=3686 - _globals['_CREATEEXTERNALTABLENODE']._serialized_end=4176 - _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_start=4124 - _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_end=4170 - _globals['_PREPARENODE']._serialized_start=4178 - _globals['_PREPARENODE']._serialized_end=4292 - _globals['_CREATECATALOGSCHEMANODE']._serialized_start=4294 - _globals['_CREATECATALOGSCHEMANODE']._serialized_end=4401 - _globals['_CREATECATALOGNODE']._serialized_start=4403 - _globals['_CREATECATALOGNODE']._serialized_end=4505 - _globals['_DROPVIEWNODE']._serialized_start=4507 - _globals['_DROPVIEWNODE']._serialized_end=4625 - _globals['_CREATEVIEWNODE']._serialized_start=4628 - _globals['_CREATEVIEWNODE']._serialized_end=4781 - _globals['_VALUESNODE']._serialized_start=4783 - _globals['_VALUESNODE']._serialized_end=4861 - _globals['_ANALYZENODE']._serialized_start=4863 - _globals['_ANALYZENODE']._serialized_end=4937 - _globals['_EXPLAINNODE']._serialized_start=4939 - _globals['_EXPLAINNODE']._serialized_end=5013 - _globals['_AGGREGATENODE']._serialized_start=5016 - _globals['_AGGREGATENODE']._serialized_end=5172 - _globals['_WINDOWNODE']._serialized_start=5174 - _globals['_WINDOWNODE']._serialized_end=5280 - _globals['_JOINNODE']._serialized_start=5283 - _globals['_JOINNODE']._serialized_end=5650 - _globals['_DISTINCTNODE']._serialized_start=5652 - _globals['_DISTINCTNODE']._serialized_end=5710 - _globals['_UNIONNODE']._serialized_start=5712 - _globals['_UNIONNODE']._serialized_end=5768 - _globals['_CROSSJOINNODE']._serialized_start=5770 - _globals['_CROSSJOINNODE']._serialized_end=5872 - _globals['_LIMITNODE']._serialized_start=5874 - _globals['_LIMITNODE']._serialized_end=5958 - _globals['_SELECTIONEXECNODE']._serialized_start=5960 - _globals['_SELECTIONEXECNODE']._serialized_end=6022 - _globals['_SUBQUERYALIASNODE']._serialized_start=6024 - _globals['_SUBQUERYALIASNODE']._serialized_end=6141 - _globals['_LOGICALEXPRNODE']._serialized_start=6144 - _globals['_LOGICALEXPRNODE']._serialized_end=7725 - _globals['_PLACEHOLDERNODE']._serialized_start=7727 - _globals['_PLACEHOLDERNODE']._serialized_end=7798 - _globals['_LOGICALEXPRLIST']._serialized_start=7800 - _globals['_LOGICALEXPRLIST']._serialized_end=7860 - _globals['_GROUPINGSETNODE']._serialized_start=7862 - _globals['_GROUPINGSETNODE']._serialized_end=7922 - _globals['_CUBENODE']._serialized_start=7924 - _globals['_CUBENODE']._serialized_end=7977 - _globals['_ROLLUPNODE']._serialized_start=7979 - _globals['_ROLLUPNODE']._serialized_end=8034 - _globals['_GETINDEXEDFIELD']._serialized_start=8036 - _globals['_GETINDEXEDFIELD']._serialized_end=8134 - _globals['_ISNULL']._serialized_start=8136 - _globals['_ISNULL']._serialized_end=8187 - _globals['_ISNOTNULL']._serialized_start=8189 - _globals['_ISNOTNULL']._serialized_end=8243 - _globals['_ISTRUE']._serialized_start=8245 - _globals['_ISTRUE']._serialized_end=8296 - _globals['_ISFALSE']._serialized_start=8298 - _globals['_ISFALSE']._serialized_end=8350 - _globals['_ISUNKNOWN']._serialized_start=8352 - _globals['_ISUNKNOWN']._serialized_end=8406 - _globals['_ISNOTTRUE']._serialized_start=8408 - _globals['_ISNOTTRUE']._serialized_end=8462 - _globals['_ISNOTFALSE']._serialized_start=8464 - _globals['_ISNOTFALSE']._serialized_end=8519 - _globals['_ISNOTUNKNOWN']._serialized_start=8521 - _globals['_ISNOTUNKNOWN']._serialized_end=8578 - _globals['_NOT']._serialized_start=8580 - _globals['_NOT']._serialized_end=8628 - _globals['_ALIASNODE']._serialized_start=8630 - _globals['_ALIASNODE']._serialized_end=8699 - _globals['_BINARYEXPRNODE']._serialized_start=8701 - _globals['_BINARYEXPRNODE']._serialized_end=8776 - _globals['_NEGATIVENODE']._serialized_start=8778 - _globals['_NEGATIVENODE']._serialized_end=8835 - _globals['_INLISTNODE']._serialized_start=8837 - _globals['_INLISTNODE']._serialized_end=8952 - _globals['_SCALARFUNCTIONNODE']._serialized_start=8954 - _globals['_SCALARFUNCTIONNODE']._serialized_end=9058 - _globals['_AGGREGATEEXPRNODE']._serialized_start=9061 - _globals['_AGGREGATEEXPRNODE']._serialized_end=9287 - _globals['_AGGREGATEUDFEXPRNODE']._serialized_start=9290 - _globals['_AGGREGATEUDFEXPRNODE']._serialized_end=9465 - _globals['_SCALARUDFEXPRNODE']._serialized_start=9467 - _globals['_SCALARUDFEXPRNODE']._serialized_end=9547 - _globals['_WINDOWEXPRNODE']._serialized_start=9550 - _globals['_WINDOWEXPRNODE']._serialized_end=9925 - _globals['_BETWEENNODE']._serialized_start=9928 - _globals['_BETWEENNODE']._serialized_end=10086 - _globals['_LIKENODE']._serialized_start=10089 - _globals['_LIKENODE']._serialized_end=10226 - _globals['_ILIKENODE']._serialized_start=10229 - _globals['_ILIKENODE']._serialized_end=10367 - _globals['_SIMILARTONODE']._serialized_start=10370 - _globals['_SIMILARTONODE']._serialized_end=10512 - _globals['_CASENODE']._serialized_start=10515 - _globals['_CASENODE']._serialized_end=10662 - _globals['_WHENTHEN']._serialized_start=10664 - _globals['_WHENTHEN']._serialized_end=10770 - _globals['_CASTNODE']._serialized_start=10772 - _globals['_CASTNODE']._serialized_end=10868 - _globals['_TRYCASTNODE']._serialized_start=10870 - _globals['_TRYCASTNODE']._serialized_end=10969 - _globals['_SORTEXPRNODE']._serialized_start=10971 - _globals['_SORTEXPRNODE']._serialized_end=11062 - _globals['_WINDOWFRAME']._serialized_start=11065 - _globals['_WINDOWFRAME']._serialized_end=11247 - _globals['_WINDOWFRAMEBOUND']._serialized_start=11250 - _globals['_WINDOWFRAMEBOUND']._serialized_end=11381 - _globals['_SCHEMA']._serialized_start=11384 - _globals['_SCHEMA']._serialized_end=11529 + _globals['_LOGICALPLANNODE']._serialized_end=1816 + _globals['_LOGICALEXTENSIONNODE']._serialized_start=1818 + _globals['_LOGICALEXTENSIONNODE']._serialized_end=1899 + _globals['_PROJECTIONCOLUMNS']._serialized_start=1901 + _globals['_PROJECTIONCOLUMNS']._serialized_end=1937 + _globals['_CSVFORMAT']._serialized_start=1939 + _globals['_CSVFORMAT']._serialized_end=2041 + _globals['_PARQUETFORMAT']._serialized_start=2043 + _globals['_PARQUETFORMAT']._serialized_end=2064 + _globals['_AVROFORMAT']._serialized_start=2066 + _globals['_AVROFORMAT']._serialized_end=2078 + _globals['_LOGICALEXPRNODECOLLECTION']._serialized_start=2080 + _globals['_LOGICALEXPRNODECOLLECTION']._serialized_end=2164 + _globals['_LISTINGTABLESCANNODE']._serialized_start=2167 + _globals['_LISTINGTABLESCANNODE']._serialized_end=2705 + _globals['_VIEWTABLESCANNODE']._serialized_start=2708 + _globals['_VIEWTABLESCANNODE']._serialized_end=2937 + _globals['_CUSTOMTABLESCANNODE']._serialized_start=2940 + _globals['_CUSTOMTABLESCANNODE']._serialized_end=3180 + _globals['_PROJECTIONNODE']._serialized_start=3183 + _globals['_PROJECTIONNODE']._serialized_end=3321 + _globals['_SELECTIONNODE']._serialized_start=3323 + _globals['_SELECTIONNODE']._serialized_end=3425 + _globals['_SORTNODE']._serialized_start=3427 + _globals['_SORTNODE']._serialized_end=3539 + _globals['_REPARTITIONNODE']._serialized_start=3542 + _globals['_REPARTITIONNODE']._serialized_end=3691 + _globals['_HASHREPARTITION']._serialized_start=3693 + _globals['_HASHREPARTITION']._serialized_end=3783 + _globals['_EMPTYRELATIONNODE']._serialized_start=3785 + _globals['_EMPTYRELATIONNODE']._serialized_end=3829 + _globals['_PRIMARYKEYCONSTRAINT']._serialized_start=3831 + _globals['_PRIMARYKEYCONSTRAINT']._serialized_end=3870 + _globals['_UNIQUECONSTRAINT']._serialized_start=3872 + _globals['_UNIQUECONSTRAINT']._serialized_end=3907 + _globals['_CONSTRAINT']._serialized_start=3910 + _globals['_CONSTRAINT']._serialized_end=4046 + _globals['_CONSTRAINTS']._serialized_start=4048 + _globals['_CONSTRAINTS']._serialized_end=4106 + _globals['_CREATEEXTERNALTABLENODE']._serialized_start=4109 + _globals['_CREATEEXTERNALTABLENODE']._serialized_end=4853 + _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_start=4711 + _globals['_CREATEEXTERNALTABLENODE_OPTIONSENTRY']._serialized_end=4757 + _globals['_CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY']._serialized_start=4759 + _globals['_CREATEEXTERNALTABLENODE_COLUMNDEFAULTSENTRY']._serialized_end=4841 + _globals['_PREPARENODE']._serialized_start=4855 + _globals['_PREPARENODE']._serialized_end=4969 + _globals['_CREATECATALOGSCHEMANODE']._serialized_start=4971 + _globals['_CREATECATALOGSCHEMANODE']._serialized_end=5078 + _globals['_CREATECATALOGNODE']._serialized_start=5080 + _globals['_CREATECATALOGNODE']._serialized_end=5182 + _globals['_DROPVIEWNODE']._serialized_start=5184 + _globals['_DROPVIEWNODE']._serialized_end=5302 + _globals['_CREATEVIEWNODE']._serialized_start=5305 + _globals['_CREATEVIEWNODE']._serialized_end=5458 + _globals['_VALUESNODE']._serialized_start=5460 + _globals['_VALUESNODE']._serialized_end=5538 + _globals['_ANALYZENODE']._serialized_start=5540 + _globals['_ANALYZENODE']._serialized_end=5614 + _globals['_EXPLAINNODE']._serialized_start=5616 + _globals['_EXPLAINNODE']._serialized_end=5690 + _globals['_AGGREGATENODE']._serialized_start=5693 + _globals['_AGGREGATENODE']._serialized_end=5849 + _globals['_WINDOWNODE']._serialized_start=5851 + _globals['_WINDOWNODE']._serialized_end=5957 + _globals['_JOINNODE']._serialized_start=5960 + _globals['_JOINNODE']._serialized_end=6327 + _globals['_DISTINCTNODE']._serialized_start=6329 + _globals['_DISTINCTNODE']._serialized_end=6387 + _globals['_DISTINCTONNODE']._serialized_start=6390 + _globals['_DISTINCTONNODE']._serialized_end=6594 + _globals['_COPYTONODE']._serialized_start=6597 + _globals['_COPYTONODE']._serialized_end=6815 + _globals['_SQLOPTIONS']._serialized_start=6817 + _globals['_SQLOPTIONS']._serialized_end=6868 + _globals['_SQLOPTION']._serialized_start=6870 + _globals['_SQLOPTION']._serialized_end=6909 + _globals['_UNIONNODE']._serialized_start=6911 + _globals['_UNIONNODE']._serialized_end=6967 + _globals['_CROSSJOINNODE']._serialized_start=6969 + _globals['_CROSSJOINNODE']._serialized_end=7071 + _globals['_LIMITNODE']._serialized_start=7073 + _globals['_LIMITNODE']._serialized_end=7157 + _globals['_SELECTIONEXECNODE']._serialized_start=7159 + _globals['_SELECTIONEXECNODE']._serialized_end=7221 + _globals['_SUBQUERYALIASNODE']._serialized_start=7223 + _globals['_SUBQUERYALIASNODE']._serialized_end=7340 + _globals['_LOGICALEXPRNODE']._serialized_start=7343 + _globals['_LOGICALEXPRNODE']._serialized_end=8984 + _globals['_WILDCARD']._serialized_start=8986 + _globals['_WILDCARD']._serialized_end=9015 + _globals['_PLACEHOLDERNODE']._serialized_start=9017 + _globals['_PLACEHOLDERNODE']._serialized_end=9088 + _globals['_LOGICALEXPRLIST']._serialized_start=9090 + _globals['_LOGICALEXPRLIST']._serialized_end=9150 + _globals['_GROUPINGSETNODE']._serialized_start=9152 + _globals['_GROUPINGSETNODE']._serialized_end=9212 + _globals['_CUBENODE']._serialized_start=9214 + _globals['_CUBENODE']._serialized_end=9267 + _globals['_ROLLUPNODE']._serialized_start=9269 + _globals['_ROLLUPNODE']._serialized_end=9324 + _globals['_NAMEDSTRUCTFIELD']._serialized_start=9326 + _globals['_NAMEDSTRUCTFIELD']._serialized_end=9383 + _globals['_LISTINDEX']._serialized_start=9385 + _globals['_LISTINDEX']._serialized_end=9438 + _globals['_LISTRANGE']._serialized_start=9441 + _globals['_LISTRANGE']._serialized_end=9584 + _globals['_GETINDEXEDFIELD']._serialized_start=9587 + _globals['_GETINDEXEDFIELD']._serialized_end=9806 + _globals['_ISNULL']._serialized_start=9808 + _globals['_ISNULL']._serialized_end=9859 + _globals['_ISNOTNULL']._serialized_start=9861 + _globals['_ISNOTNULL']._serialized_end=9915 + _globals['_ISTRUE']._serialized_start=9917 + _globals['_ISTRUE']._serialized_end=9968 + _globals['_ISFALSE']._serialized_start=9970 + _globals['_ISFALSE']._serialized_end=10022 + _globals['_ISUNKNOWN']._serialized_start=10024 + _globals['_ISUNKNOWN']._serialized_end=10078 + _globals['_ISNOTTRUE']._serialized_start=10080 + _globals['_ISNOTTRUE']._serialized_end=10134 + _globals['_ISNOTFALSE']._serialized_start=10136 + _globals['_ISNOTFALSE']._serialized_end=10191 + _globals['_ISNOTUNKNOWN']._serialized_start=10193 + _globals['_ISNOTUNKNOWN']._serialized_end=10250 + _globals['_NOT']._serialized_start=10252 + _globals['_NOT']._serialized_end=10300 + _globals['_ALIASNODE']._serialized_start=10302 + _globals['_ALIASNODE']._serialized_end=10422 + _globals['_BINARYEXPRNODE']._serialized_start=10424 + _globals['_BINARYEXPRNODE']._serialized_end=10499 + _globals['_NEGATIVENODE']._serialized_start=10501 + _globals['_NEGATIVENODE']._serialized_end=10558 + _globals['_UNNEST']._serialized_start=10560 + _globals['_UNNEST']._serialized_end=10612 + _globals['_INLISTNODE']._serialized_start=10614 + _globals['_INLISTNODE']._serialized_end=10729 + _globals['_SCALARFUNCTIONNODE']._serialized_start=10731 + _globals['_SCALARFUNCTIONNODE']._serialized_end=10835 + _globals['_AGGREGATEEXPRNODE']._serialized_start=10838 + _globals['_AGGREGATEEXPRNODE']._serialized_end=11064 + _globals['_AGGREGATEUDFEXPRNODE']._serialized_start=11067 + _globals['_AGGREGATEUDFEXPRNODE']._serialized_end=11242 + _globals['_SCALARUDFEXPRNODE']._serialized_start=11244 + _globals['_SCALARUDFEXPRNODE']._serialized_end=11324 + _globals['_WINDOWEXPRNODE']._serialized_start=11327 + _globals['_WINDOWEXPRNODE']._serialized_end=11702 + _globals['_BETWEENNODE']._serialized_start=11705 + _globals['_BETWEENNODE']._serialized_end=11863 + _globals['_LIKENODE']._serialized_start=11866 + _globals['_LIKENODE']._serialized_end=12003 + _globals['_ILIKENODE']._serialized_start=12006 + _globals['_ILIKENODE']._serialized_end=12144 + _globals['_SIMILARTONODE']._serialized_start=12147 + _globals['_SIMILARTONODE']._serialized_end=12289 + _globals['_CASENODE']._serialized_start=12292 + _globals['_CASENODE']._serialized_end=12439 + _globals['_WHENTHEN']._serialized_start=12441 + _globals['_WHENTHEN']._serialized_end=12547 + _globals['_CASTNODE']._serialized_start=12549 + _globals['_CASTNODE']._serialized_end=12645 + _globals['_TRYCASTNODE']._serialized_start=12647 + _globals['_TRYCASTNODE']._serialized_end=12746 + _globals['_SORTEXPRNODE']._serialized_start=12748 + _globals['_SORTEXPRNODE']._serialized_end=12839 + _globals['_WINDOWFRAME']._serialized_start=12842 + _globals['_WINDOWFRAME']._serialized_end=13024 + _globals['_WINDOWFRAMEBOUND']._serialized_start=13027 + _globals['_WINDOWFRAMEBOUND']._serialized_end=13158 + _globals['_SCHEMA']._serialized_start=13161 + _globals['_SCHEMA']._serialized_end=13306 _globals['_SCHEMA_METADATAENTRY']._serialized_start=341 _globals['_SCHEMA_METADATAENTRY']._serialized_end=388 - _globals['_FIELD']._serialized_start=11532 - _globals['_FIELD']._serialized_end=11751 + _globals['_FIELD']._serialized_start=13309 + _globals['_FIELD']._serialized_end=13567 _globals['_FIELD_METADATAENTRY']._serialized_start=341 _globals['_FIELD_METADATAENTRY']._serialized_end=388 - _globals['_FIXEDSIZEBINARY']._serialized_start=11753 - _globals['_FIXEDSIZEBINARY']._serialized_end=11786 - _globals['_TIMESTAMP']._serialized_start=11788 - _globals['_TIMESTAMP']._serialized_end=11858 - _globals['_DECIMAL']._serialized_start=11860 - _globals['_DECIMAL']._serialized_end=11915 - _globals['_LIST']._serialized_start=11917 - _globals['_LIST']._serialized_end=11962 - _globals['_FIXEDSIZELIST']._serialized_start=11964 - _globals['_FIXEDSIZELIST']._serialized_end=12037 - _globals['_DICTIONARY']._serialized_start=12039 - _globals['_DICTIONARY']._serialized_end=12125 - _globals['_STRUCT']._serialized_start=12127 - _globals['_STRUCT']._serialized_end=12179 - _globals['_MAP']._serialized_start=12181 - _globals['_MAP']._serialized_end=12246 - _globals['_UNION']._serialized_start=12248 - _globals['_UNION']._serialized_end=12356 - _globals['_SCALARLISTVALUE']._serialized_start=12358 - _globals['_SCALARLISTVALUE']._serialized_end=12467 - _globals['_SCALARTIME32VALUE']._serialized_start=12469 - _globals['_SCALARTIME32VALUE']._serialized_end=12564 - _globals['_SCALARTIME64VALUE']._serialized_start=12566 - _globals['_SCALARTIME64VALUE']._serialized_end=12665 - _globals['_SCALARTIMESTAMPVALUE']._serialized_start=12668 - _globals['_SCALARTIMESTAMPVALUE']._serialized_end=12847 - _globals['_SCALARDICTIONARYVALUE']._serialized_start=12849 - _globals['_SCALARDICTIONARYVALUE']._serialized_end=12955 - _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_start=12957 - _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_end=13029 - _globals['_STRUCTVALUE']._serialized_start=13031 - _globals['_STRUCTVALUE']._serialized_end=13126 - _globals['_SCALARFIXEDSIZEBINARY']._serialized_start=13128 - _globals['_SCALARFIXEDSIZEBINARY']._serialized_end=13183 - _globals['_SCALARVALUE']._serialized_start=13186 - _globals['_SCALARVALUE']._serialized_end=14417 - _globals['_DECIMAL128']._serialized_start=14419 - _globals['_DECIMAL128']._serialized_end=14468 - _globals['_ARROWTYPE']._serialized_start=14471 - _globals['_ARROWTYPE']._serialized_end=15899 - _globals['_EMPTYMESSAGE']._serialized_start=15901 - _globals['_EMPTYMESSAGE']._serialized_end=15915 - _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_start=15917 - _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_end=15965 - _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_start=15967 - _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_end=16017 - _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_start=16019 - _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_end=16070 - _globals['_PLANTYPE']._serialized_start=16073 - _globals['_PLANTYPE']._serialized_end=16595 - _globals['_STRINGIFIEDPLAN']._serialized_start=16597 - _globals['_STRINGIFIEDPLAN']._serialized_end=16669 - _globals['_BARETABLEREFERENCE']._serialized_start=16671 - _globals['_BARETABLEREFERENCE']._serialized_end=16706 - _globals['_PARTIALTABLEREFERENCE']._serialized_start=16708 - _globals['_PARTIALTABLEREFERENCE']._serialized_end=16762 - _globals['_FULLTABLEREFERENCE']._serialized_start=16764 - _globals['_FULLTABLEREFERENCE']._serialized_end=16832 - _globals['_OWNEDTABLEREFERENCE']._serialized_start=16835 - _globals['_OWNEDTABLEREFERENCE']._serialized_end=17030 - _globals['_PHYSICALPLANNODE']._serialized_start=17033 - _globals['_PHYSICALPLANNODE']._serialized_end=18195 - _globals['_PHYSICALEXTENSIONNODE']._serialized_start=18197 - _globals['_PHYSICALEXTENSIONNODE']._serialized_end=18280 - _globals['_PHYSICALEXPRNODE']._serialized_start=18283 - _globals['_PHYSICALEXPRNODE']._serialized_end=19382 - _globals['_PHYSICALSCALARUDFNODE']._serialized_start=19384 - _globals['_PHYSICALSCALARUDFNODE']._serialized_end=19509 - _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_start=19512 - _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_end=19772 - _globals['_PHYSICALWINDOWEXPRNODE']._serialized_start=19775 - _globals['_PHYSICALWINDOWEXPRNODE']._serialized_end=19982 - _globals['_PHYSICALISNULL']._serialized_start=19984 - _globals['_PHYSICALISNULL']._serialized_end=20044 - _globals['_PHYSICALISNOTNULL']._serialized_start=20046 - _globals['_PHYSICALISNOTNULL']._serialized_end=20109 - _globals['_PHYSICALNOT']._serialized_start=20111 - _globals['_PHYSICALNOT']._serialized_end=20168 - _globals['_PHYSICALALIASNODE']._serialized_start=20170 - _globals['_PHYSICALALIASNODE']._serialized_end=20248 - _globals['_PHYSICALBINARYEXPRNODE']._serialized_start=20250 - _globals['_PHYSICALBINARYEXPRNODE']._serialized_end=20368 - _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_start=20371 - _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_end=20499 - _globals['_PHYSICALLIKEEXPRNODE']._serialized_start=20502 - _globals['_PHYSICALLIKEEXPRNODE']._serialized_end=20658 - _globals['_PHYSICALSORTEXPRNODE']._serialized_start=20660 - _globals['_PHYSICALSORTEXPRNODE']._serialized_end=20760 - _globals['_PHYSICALWHENTHEN']._serialized_start=20762 - _globals['_PHYSICALWHENTHEN']._serialized_end=20878 - _globals['_PHYSICALINLISTNODE']._serialized_start=20880 - _globals['_PHYSICALINLISTNODE']._serialized_end=21005 - _globals['_PHYSICALCASENODE']._serialized_start=21008 - _globals['_PHYSICALCASENODE']._serialized_end=21173 - _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_start=21176 - _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_end=21347 - _globals['_PHYSICALTRYCASTNODE']._serialized_start=21349 - _globals['_PHYSICALTRYCASTNODE']._serialized_end=21457 - _globals['_PHYSICALCASTNODE']._serialized_start=21459 - _globals['_PHYSICALCASTNODE']._serialized_end=21564 - _globals['_PHYSICALNEGATIVENODE']._serialized_start=21566 - _globals['_PHYSICALNEGATIVENODE']._serialized_end=21632 - _globals['_FILTEREXECNODE']._serialized_start=21634 - _globals['_FILTEREXECNODE']._serialized_end=21739 - _globals['_FILEGROUP']._serialized_start=21741 - _globals['_FILEGROUP']._serialized_end=21796 - _globals['_SCANLIMIT']._serialized_start=21798 - _globals['_SCANLIMIT']._serialized_end=21824 - _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_start=21826 - _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_end=21926 - _globals['_FILESCANEXECCONF']._serialized_start=21929 - _globals['_FILESCANEXECCONF']._serialized_end=22260 - _globals['_PARQUETSCANEXECNODE']._serialized_start=22262 - _globals['_PARQUETSCANEXECNODE']._serialized_end=22387 - _globals['_CSVSCANEXECNODE']._serialized_start=22389 - _globals['_CSVSCANEXECNODE']._serialized_end=22494 - _globals['_AVROSCANEXECNODE']._serialized_start=22496 - _globals['_AVROSCANEXECNODE']._serialized_end=22563 - _globals['_HASHJOINEXECNODE']._serialized_start=22566 - _globals['_HASHJOINEXECNODE']._serialized_end=22863 - _globals['_UNIONEXECNODE']._serialized_start=22865 - _globals['_UNIONEXECNODE']._serialized_end=22926 - _globals['_EXPLAINEXECNODE']._serialized_start=22928 - _globals['_EXPLAINEXECNODE']._serialized_end=23054 - _globals['_CROSSJOINEXECNODE']._serialized_start=23056 - _globals['_CROSSJOINEXECNODE']._serialized_end=23164 - _globals['_PHYSICALCOLUMN']._serialized_start=23166 - _globals['_PHYSICALCOLUMN']._serialized_end=23211 - _globals['_JOINON']._serialized_start=23213 - _globals['_JOINON']._serialized_end=23306 - _globals['_EMPTYEXECNODE']._serialized_start=23308 - _globals['_EMPTYEXECNODE']._serialized_end=23384 - _globals['_PROJECTIONEXECNODE']._serialized_start=23387 - _globals['_PROJECTIONEXECNODE']._serialized_end=23515 - _globals['_WINDOWAGGEXECNODE']._serialized_start=23518 - _globals['_WINDOWAGGEXECNODE']._serialized_end=23701 - _globals['_MAYBEFILTER']._serialized_start=23703 - _globals['_MAYBEFILTER']._serialized_end=23760 - _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_start=23762 - _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_end=23839 - _globals['_AGGREGATEEXECNODE']._serialized_start=23842 - _globals['_AGGREGATEEXECNODE']._serialized_end=24307 - _globals['_GLOBALLIMITEXECNODE']._serialized_start=24309 - _globals['_GLOBALLIMITEXECNODE']._serialized_end=24404 - _globals['_LOCALLIMITEXECNODE']._serialized_start=24406 - _globals['_LOCALLIMITEXECNODE']._serialized_end=24486 - _globals['_SORTEXECNODE']._serialized_start=24489 - _globals['_SORTEXECNODE']._serialized_end=24638 - _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_start=24641 - _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_end=24774 - _globals['_NESTEDLOOPJOINEXECNODE']._serialized_start=24777 - _globals['_NESTEDLOOPJOINEXECNODE']._serialized_end=24971 - _globals['_COALESCEBATCHESEXECNODE']._serialized_start=24973 - _globals['_COALESCEBATCHESEXECNODE']._serialized_end=25070 - _globals['_COALESCEPARTITIONSEXECNODE']._serialized_start=25072 - _globals['_COALESCEPARTITIONSEXECNODE']._serialized_end=25145 - _globals['_PHYSICALHASHREPARTITION']._serialized_start=25147 - _globals['_PHYSICALHASHREPARTITION']._serialized_end=25246 - _globals['_REPARTITIONEXECNODE']._serialized_start=25249 - _globals['_REPARTITIONEXECNODE']._serialized_end=25430 - _globals['_JOINFILTER']._serialized_start=25433 - _globals['_JOINFILTER']._serialized_end=25580 - _globals['_COLUMNINDEX']._serialized_start=25582 - _globals['_COLUMNINDEX']._serialized_end=25646 - _globals['_PARTITIONEDFILE']._serialized_start=25649 - _globals['_PARTITIONEDFILE']._serialized_end=25809 - _globals['_FILERANGE']._serialized_start=25811 - _globals['_FILERANGE']._serialized_end=25850 - _globals['_PARTITIONSTATS']._serialized_start=25852 - _globals['_PARTITIONSTATS']._serialized_end=25973 - _globals['_STATISTICS']._serialized_start=25975 - _globals['_STATISTICS']._serialized_end=26095 - _globals['_COLUMNSTATS']._serialized_start=26098 - _globals['_COLUMNSTATS']._serialized_end=26243 - _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_start=26245 - _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_end=26359 + _globals['_FIXEDSIZEBINARY']._serialized_start=13569 + _globals['_FIXEDSIZEBINARY']._serialized_end=13602 + _globals['_TIMESTAMP']._serialized_start=13604 + _globals['_TIMESTAMP']._serialized_end=13674 + _globals['_DECIMAL']._serialized_start=13676 + _globals['_DECIMAL']._serialized_end=13731 + _globals['_LIST']._serialized_start=13733 + _globals['_LIST']._serialized_end=13778 + _globals['_FIXEDSIZELIST']._serialized_start=13780 + _globals['_FIXEDSIZELIST']._serialized_end=13853 + _globals['_DICTIONARY']._serialized_start=13855 + _globals['_DICTIONARY']._serialized_end=13941 + _globals['_STRUCT']._serialized_start=13943 + _globals['_STRUCT']._serialized_end=13995 + _globals['_MAP']._serialized_start=13997 + _globals['_MAP']._serialized_end=14062 + _globals['_UNION']._serialized_start=14064 + _globals['_UNION']._serialized_end=14172 + _globals['_SCALARNESTEDVALUE']._serialized_start=14174 + _globals['_SCALARNESTEDVALUE']._serialized_end=14270 + _globals['_SCALARTIME32VALUE']._serialized_start=14272 + _globals['_SCALARTIME32VALUE']._serialized_end=14367 + _globals['_SCALARTIME64VALUE']._serialized_start=14369 + _globals['_SCALARTIME64VALUE']._serialized_end=14468 + _globals['_SCALARTIMESTAMPVALUE']._serialized_start=14471 + _globals['_SCALARTIMESTAMPVALUE']._serialized_end=14650 + _globals['_SCALARDICTIONARYVALUE']._serialized_start=14652 + _globals['_SCALARDICTIONARYVALUE']._serialized_end=14758 + _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_start=14760 + _globals['_INTERVALMONTHDAYNANOVALUE']._serialized_end=14832 + _globals['_SCALARFIXEDSIZEBINARY']._serialized_start=14834 + _globals['_SCALARFIXEDSIZEBINARY']._serialized_end=14889 + _globals['_SCALARVALUE']._serialized_start=14892 + _globals['_SCALARVALUE']._serialized_end=16306 + _globals['_DECIMAL128']._serialized_start=16308 + _globals['_DECIMAL128']._serialized_end=16357 + _globals['_DECIMAL256']._serialized_start=16359 + _globals['_DECIMAL256']._serialized_end=16408 + _globals['_ARROWTYPE']._serialized_start=16411 + _globals['_ARROWTYPE']._serialized_end=17839 + _globals['_EMPTYMESSAGE']._serialized_start=17841 + _globals['_EMPTYMESSAGE']._serialized_end=17855 + _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_start=17857 + _globals['_ANALYZEDLOGICALPLANTYPE']._serialized_end=17905 + _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_start=17907 + _globals['_OPTIMIZEDLOGICALPLANTYPE']._serialized_end=17957 + _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_start=17959 + _globals['_OPTIMIZEDPHYSICALPLANTYPE']._serialized_end=18010 + _globals['_PLANTYPE']._serialized_start=18013 + _globals['_PLANTYPE']._serialized_end=18665 + _globals['_STRINGIFIEDPLAN']._serialized_start=18667 + _globals['_STRINGIFIEDPLAN']._serialized_end=18739 + _globals['_BARETABLEREFERENCE']._serialized_start=18741 + _globals['_BARETABLEREFERENCE']._serialized_end=18776 + _globals['_PARTIALTABLEREFERENCE']._serialized_start=18778 + _globals['_PARTIALTABLEREFERENCE']._serialized_end=18832 + _globals['_FULLTABLEREFERENCE']._serialized_start=18834 + _globals['_FULLTABLEREFERENCE']._serialized_end=18902 + _globals['_OWNEDTABLEREFERENCE']._serialized_start=18905 + _globals['_OWNEDTABLEREFERENCE']._serialized_end=19100 + _globals['_PHYSICALPLANNODE']._serialized_start=19103 + _globals['_PHYSICALPLANNODE']._serialized_end=20657 + _globals['_PARTITIONCOLUMN']._serialized_start=20659 + _globals['_PARTITIONCOLUMN']._serialized_end=20733 + _globals['_FILETYPEWRITEROPTIONS']._serialized_start=20736 + _globals['_FILETYPEWRITEROPTIONS']._serialized_end=20997 + _globals['_JSONWRITEROPTIONS']._serialized_start=20999 + _globals['_JSONWRITEROPTIONS']._serialized_end=21075 + _globals['_PARQUETWRITEROPTIONS']._serialized_start=21077 + _globals['_PARQUETWRITEROPTIONS']._serialized_end=21156 + _globals['_CSVWRITEROPTIONS']._serialized_start=21159 + _globals['_CSVWRITEROPTIONS']._serialized_end=21386 + _globals['_ARROWWRITEROPTIONS']._serialized_start=21388 + _globals['_ARROWWRITEROPTIONS']._serialized_end=21408 + _globals['_WRITERPROPERTIES']._serialized_start=21411 + _globals['_WRITERPROPERTIES']._serialized_end=21628 + _globals['_FILESINKCONFIG']._serialized_start=21631 + _globals['_FILESINKCONFIG']._serialized_end=21940 + _globals['_JSONSINK']._serialized_start=21942 + _globals['_JSONSINK']._serialized_end=21996 + _globals['_JSONSINKEXECNODE']._serialized_start=21999 + _globals['_JSONSINKEXECNODE']._serialized_end=22203 + _globals['_CSVSINK']._serialized_start=22205 + _globals['_CSVSINK']._serialized_end=22258 + _globals['_CSVSINKEXECNODE']._serialized_start=22261 + _globals['_CSVSINKEXECNODE']._serialized_end=22463 + _globals['_PARQUETSINK']._serialized_start=22465 + _globals['_PARQUETSINK']._serialized_end=22522 + _globals['_PARQUETSINKEXECNODE']._serialized_start=22525 + _globals['_PARQUETSINKEXECNODE']._serialized_end=22735 + _globals['_PHYSICALEXTENSIONNODE']._serialized_start=22737 + _globals['_PHYSICALEXTENSIONNODE']._serialized_end=22820 + _globals['_PHYSICALEXPRNODE']._serialized_start=22823 + _globals['_PHYSICALEXPRNODE']._serialized_end=23847 + _globals['_PHYSICALSCALARUDFNODE']._serialized_start=23849 + _globals['_PHYSICALSCALARUDFNODE']._serialized_end=23974 + _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_start=23977 + _globals['_PHYSICALAGGREGATEEXPRNODE']._serialized_end=24237 + _globals['_PHYSICALWINDOWEXPRNODE']._serialized_start=24240 + _globals['_PHYSICALWINDOWEXPRNODE']._serialized_end=24612 + _globals['_PHYSICALISNULL']._serialized_start=24614 + _globals['_PHYSICALISNULL']._serialized_end=24674 + _globals['_PHYSICALISNOTNULL']._serialized_start=24676 + _globals['_PHYSICALISNOTNULL']._serialized_end=24739 + _globals['_PHYSICALNOT']._serialized_start=24741 + _globals['_PHYSICALNOT']._serialized_end=24798 + _globals['_PHYSICALALIASNODE']._serialized_start=24800 + _globals['_PHYSICALALIASNODE']._serialized_end=24878 + _globals['_PHYSICALBINARYEXPRNODE']._serialized_start=24880 + _globals['_PHYSICALBINARYEXPRNODE']._serialized_end=24998 + _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_start=25001 + _globals['_PHYSICALDATETIMEINTERVALEXPRNODE']._serialized_end=25129 + _globals['_PHYSICALLIKEEXPRNODE']._serialized_start=25132 + _globals['_PHYSICALLIKEEXPRNODE']._serialized_end=25288 + _globals['_PHYSICALSORTEXPRNODE']._serialized_start=25290 + _globals['_PHYSICALSORTEXPRNODE']._serialized_end=25390 + _globals['_PHYSICALWHENTHEN']._serialized_start=25392 + _globals['_PHYSICALWHENTHEN']._serialized_end=25508 + _globals['_PHYSICALINLISTNODE']._serialized_start=25510 + _globals['_PHYSICALINLISTNODE']._serialized_end=25635 + _globals['_PHYSICALCASENODE']._serialized_start=25638 + _globals['_PHYSICALCASENODE']._serialized_end=25803 + _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_start=25806 + _globals['_PHYSICALSCALARFUNCTIONNODE']._serialized_end=25977 + _globals['_PHYSICALTRYCASTNODE']._serialized_start=25979 + _globals['_PHYSICALTRYCASTNODE']._serialized_end=26087 + _globals['_PHYSICALCASTNODE']._serialized_start=26089 + _globals['_PHYSICALCASTNODE']._serialized_end=26194 + _globals['_PHYSICALNEGATIVENODE']._serialized_start=26196 + _globals['_PHYSICALNEGATIVENODE']._serialized_end=26262 + _globals['_FILTEREXECNODE']._serialized_start=26265 + _globals['_FILTEREXECNODE']._serialized_end=26406 + _globals['_FILEGROUP']._serialized_start=26408 + _globals['_FILEGROUP']._serialized_end=26463 + _globals['_SCANLIMIT']._serialized_start=26465 + _globals['_SCANLIMIT']._serialized_end=26491 + _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_start=26493 + _globals['_PHYSICALSORTEXPRNODECOLLECTION']._serialized_end=26593 + _globals['_FILESCANEXECCONF']._serialized_start=26596 + _globals['_FILESCANEXECCONF']._serialized_end=26927 + _globals['_PARQUETSCANEXECNODE']._serialized_start=26929 + _globals['_PARQUETSCANEXECNODE']._serialized_end=27054 + _globals['_CSVSCANEXECNODE']._serialized_start=27057 + _globals['_CSVSCANEXECNODE']._serialized_end=27214 + _globals['_AVROSCANEXECNODE']._serialized_start=27216 + _globals['_AVROSCANEXECNODE']._serialized_end=27283 + _globals['_HASHJOINEXECNODE']._serialized_start=27286 + _globals['_HASHJOINEXECNODE']._serialized_end=27583 + _globals['_SYMMETRICHASHJOINEXECNODE']._serialized_start=27586 + _globals['_SYMMETRICHASHJOINEXECNODE']._serialized_end=28017 + _globals['_INTERLEAVEEXECNODE']._serialized_start=28019 + _globals['_INTERLEAVEEXECNODE']._serialized_end=28085 + _globals['_UNIONEXECNODE']._serialized_start=28087 + _globals['_UNIONEXECNODE']._serialized_end=28148 + _globals['_EXPLAINEXECNODE']._serialized_start=28150 + _globals['_EXPLAINEXECNODE']._serialized_end=28276 + _globals['_ANALYZEEXECNODE']._serialized_start=28279 + _globals['_ANALYZEEXECNODE']._serialized_end=28419 + _globals['_CROSSJOINEXECNODE']._serialized_start=28421 + _globals['_CROSSJOINEXECNODE']._serialized_end=28529 + _globals['_PHYSICALCOLUMN']._serialized_start=28531 + _globals['_PHYSICALCOLUMN']._serialized_end=28576 + _globals['_JOINON']._serialized_start=28578 + _globals['_JOINON']._serialized_end=28675 + _globals['_EMPTYEXECNODE']._serialized_start=28677 + _globals['_EMPTYEXECNODE']._serialized_end=28728 + _globals['_PLACEHOLDERROWEXECNODE']._serialized_start=28730 + _globals['_PLACEHOLDERROWEXECNODE']._serialized_end=28790 + _globals['_PROJECTIONEXECNODE']._serialized_start=28793 + _globals['_PROJECTIONEXECNODE']._serialized_end=28921 + _globals['_PARTIALLYSORTEDINPUTORDERMODE']._serialized_start=28923 + _globals['_PARTIALLYSORTEDINPUTORDERMODE']._serialized_end=28971 + _globals['_WINDOWAGGEXECNODE']._serialized_start=28974 + _globals['_WINDOWAGGEXECNODE']._serialized_end=29328 + _globals['_MAYBEFILTER']._serialized_start=29330 + _globals['_MAYBEFILTER']._serialized_end=29387 + _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_start=29389 + _globals['_MAYBEPHYSICALSORTEXPRS']._serialized_end=29466 + _globals['_AGGREGATEEXECNODE']._serialized_start=29469 + _globals['_AGGREGATEEXECNODE']._serialized_end=29875 + _globals['_GLOBALLIMITEXECNODE']._serialized_start=29877 + _globals['_GLOBALLIMITEXECNODE']._serialized_end=29972 + _globals['_LOCALLIMITEXECNODE']._serialized_start=29974 + _globals['_LOCALLIMITEXECNODE']._serialized_end=30054 + _globals['_SORTEXECNODE']._serialized_start=30057 + _globals['_SORTEXECNODE']._serialized_end=30206 + _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_start=30209 + _globals['_SORTPRESERVINGMERGEEXECNODE']._serialized_end=30342 + _globals['_NESTEDLOOPJOINEXECNODE']._serialized_start=30345 + _globals['_NESTEDLOOPJOINEXECNODE']._serialized_end=30539 + _globals['_COALESCEBATCHESEXECNODE']._serialized_start=30541 + _globals['_COALESCEBATCHESEXECNODE']._serialized_end=30638 + _globals['_COALESCEPARTITIONSEXECNODE']._serialized_start=30640 + _globals['_COALESCEPARTITIONSEXECNODE']._serialized_end=30713 + _globals['_PHYSICALHASHREPARTITION']._serialized_start=30715 + _globals['_PHYSICALHASHREPARTITION']._serialized_end=30814 + _globals['_REPARTITIONEXECNODE']._serialized_start=30817 + _globals['_REPARTITIONEXECNODE']._serialized_end=30998 + _globals['_JOINFILTER']._serialized_start=31001 + _globals['_JOINFILTER']._serialized_end=31148 + _globals['_COLUMNINDEX']._serialized_start=31150 + _globals['_COLUMNINDEX']._serialized_end=31214 + _globals['_PARTITIONEDFILE']._serialized_start=31217 + _globals['_PARTITIONEDFILE']._serialized_end=31377 + _globals['_FILERANGE']._serialized_start=31379 + _globals['_FILERANGE']._serialized_end=31418 + _globals['_PARTITIONSTATS']._serialized_start=31420 + _globals['_PARTITIONSTATS']._serialized_end=31541 + _globals['_PRECISION']._serialized_start=31543 + _globals['_PRECISION']._serialized_end=31643 + _globals['_STATISTICS']._serialized_start=31646 + _globals['_STATISTICS']._serialized_end=31794 + _globals['_COLUMNSTATS']._serialized_start=31797 + _globals['_COLUMNSTATS']._serialized_end=31984 + _globals['_NAMEDSTRUCTFIELDEXPR']._serialized_start=31986 + _globals['_NAMEDSTRUCTFIELDEXPR']._serialized_end=32047 + _globals['_LISTINDEXEXPR']._serialized_start=32049 + _globals['_LISTINDEXEXPR']._serialized_end=32107 + _globals['_LISTRANGEEXPR']._serialized_start=32110 + _globals['_LISTRANGEEXPR']._serialized_end=32260 + _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_start=32263 + _globals['_PHYSICALGETINDEXEDFIELDEXPRNODE']._serialized_end=32525 # @@protoc_insertion_point(module_scope) diff --git a/vegafusion-common/Cargo.toml b/vegafusion-common/Cargo.toml index 15ed770a1..d302a747c 100644 --- a/vegafusion-common/Cargo.toml +++ b/vegafusion-common/Cargo.toml @@ -7,7 +7,7 @@ license = "BSD-3-Clause" [features] pyarrow = [ "pyo3", "arrow/pyarrow",] -json = [ "serde_json", "arrow/json", "chrono",] +json = [ "serde_json/preserve_order", "arrow/json", "chrono",] prettyprint = [ "arrow/prettyprint",] [dependencies] @@ -32,7 +32,6 @@ features = [ "ipc",] [dependencies.datafusion-common] workspace = true -default_features = false [dependencies.datafusion-expr] workspace = true diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index 0372800f4..6138e29eb 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -429,7 +429,7 @@ fn set_column_for_json_rows( let inner_objs = struct_array_to_jsonmap_array(as_struct_array(array), row_count)?; rows.iter_mut() .take(row_count) - .zip(inner_objs.into_iter()) + .zip(inner_objs) .for_each(|(row, obj)| { row.insert(col_name.to_string(), Value::Object(obj)); }); @@ -931,7 +931,7 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"duration_sec":"PT120S","duration_msec":"PT0.120S","duration_usec":"PT0.000120S","duration_nsec":"PT0.000000120S","name":"a"} + r#"{"duration_sec":"PT120S","duration_msec":"PT0.12S","duration_usec":"PT0.00012S","duration_nsec":"PT0.00000012S","name":"a"} {"duration_sec":null,"duration_msec":null,"duration_usec":null,"duration_nsec":null,"name":"b"} "# ); diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 63c28f4ec..d0ffb6d13 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -1,10 +1,13 @@ use crate::error::{Result, VegaFusionError}; +use arrow::array::{new_empty_array, Array, ArrayRef, ListArray}; +use datafusion_common::DataFusionError; +use arrow::datatypes::DataType; +use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; #[cfg(feature = "json")] use { - arrow::datatypes::{DataType, Field}, serde_json::{Map, Value}, std::ops::Deref, std::sync::Arc, @@ -63,18 +66,18 @@ impl ScalarValueHelpers for ScalarValue { } } Value::Array(elements) => { - let (elements, dtype) = if elements.is_empty() { - (Vec::new(), DataType::Float64) + let array: ListArray = if elements.is_empty() { + array_into_list_array(Arc::new(new_empty_array(&DataType::Float64))) } else { let elements: Vec<_> = elements .iter() .map(ScalarValue::from_json) .collect::>>()?; - let dtype = elements[0].data_type(); - (elements, dtype) + + array_into_list_array(ScalarValue::iter_to_array(elements)?) }; - ScalarValue::List(Some(elements), Arc::new(Field::new("item", dtype, true))) + ScalarValue::List(Arc::new(array)) } }; Ok(scalar_value) @@ -133,23 +136,29 @@ impl ScalarValueHelpers for ScalarValue { ScalarValue::IntervalDayTime(Some(_v)) => { unimplemented!() } - ScalarValue::List(Some(v), _) => Value::Array( - v.clone() + ScalarValue::List(a) => { + let values = a + .value(0) + .to_scalar_vec()? .into_iter() .map(|v| v.to_json()) - .collect::>>()?, - ), - ScalarValue::List(None, _) => Value::Array(Vec::new()), - ScalarValue::Struct(Some(v), fields) => { + .collect::>>()?; + + Value::Array(values) + } + ScalarValue::Struct(sa) => { let mut pairs: Map = Default::default(); - for (val, field) in v.iter().zip(fields.deref()) { - pairs.insert(field.name().clone(), val.to_json()?); + for (col_ind, field) in sa.fields().deref().iter().enumerate() { + let column = sa.column(col_ind); + pairs.insert( + field.name().clone(), + ScalarValue::try_from_array(column, 0)?.to_json()?, + ); } Value::Object(pairs) } _ => Value::Null, }; - Ok(res) } @@ -174,7 +183,8 @@ impl ScalarValueHelpers for ScalarValue { } fn to_f64x2(&self) -> Result<[f64; 2]> { - if let ScalarValue::List(Some(elements), _) = self { + if let ScalarValue::List(array) = self { + let elements = array.value(0).to_scalar_vec()?; if let [v0, v1] = elements.as_slice() { return Ok([v0.to_f64()?, v1.to_f64()?]); } @@ -212,3 +222,55 @@ impl ScalarValueHelpers for ScalarValue { } } } + +pub trait ArrayRefHelpers { + fn to_scalar_vec(&self) -> std::result::Result, DataFusionError>; + + fn list_el_to_scalar_vec(&self) -> std::result::Result, DataFusionError>; + + fn list_el_len(&self) -> std::result::Result; + + fn list_el_dtype(&self) -> std::result::Result; +} + +impl ArrayRefHelpers for ArrayRef { + /// Convert ArrayRef into vector of ScalarValues + fn to_scalar_vec(&self) -> std::result::Result, DataFusionError> { + (0..self.len()) + .map(|i| ScalarValue::try_from_array(self, i)) + .collect::, DataFusionError>>() + } + + /// Extract Vec for single element ListArray (as is stored inside ScalarValue::List(arr)) + fn list_el_to_scalar_vec(&self) -> std::result::Result, DataFusionError> { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_to_scalar_vec called on non-List type".to_string(), + ))?; + a.value(0).to_scalar_vec() + } + + /// Extract length of single element ListArray + fn list_el_len(&self) -> std::result::Result { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_len called on non-List type".to_string(), + ))?; + Ok(a.value(0).len()) + } + + /// Extract data type of single element ListArray + fn list_el_dtype(&self) -> std::result::Result { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_len called on non-List type".to_string(), + ))?; + Ok(a.value(0).data_type().clone()) + } +} diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index 37b896b02..50568364b 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -13,6 +13,7 @@ use crate::{ error::{Result, ResultWithContext, VegaFusionError}, }; +use arrow::array::new_empty_array; #[cfg(feature = "prettyprint")] use arrow::util::pretty::pretty_format_batches; use std::{ @@ -41,6 +42,7 @@ use { #[cfg(feature = "base64")] use base64::{engine::general_purpose, Engine as _}; +use datafusion_common::utils::array_into_list_array; #[derive(Clone, Debug)] pub struct VegaFusionTable { @@ -172,7 +174,7 @@ impl VegaFusionTable { pub fn to_record_batch(&self) -> Result { let mut schema = self.schema.clone(); - if let Some(batch) = self.batches.get(0) { + if let Some(batch) = self.batches.first() { schema = batch.schema() } concat_batches(&schema, &self.batches) @@ -182,30 +184,11 @@ impl VegaFusionTable { pub fn to_scalar_value(&self) -> Result { if self.num_rows() == 0 { // Return empty list with (arbitrary) Float64 type - let dtype = DataType::Float64; - return Ok(ScalarValue::List( - Some(Vec::new()), - Arc::new(Field::new("item", dtype, true)), - )); + let array = Arc::new(new_empty_array(&DataType::Float64)); + return Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))); } - - let mut elements: Vec = Vec::new(); - for batch in &self.batches { - let array = Arc::new(StructArray::from(batch.clone())) as ArrayRef; - - for i in 0..array.len() { - let scalar = ScalarValue::try_from_array(&array, i).with_context(|| { - "Failed to convert record batch row to ScalarValue".to_string() - })?; - elements.push(scalar) - } - } - - let dtype = elements[0].data_type(); - Ok(ScalarValue::List( - Some(elements), - Arc::new(Field::new("item", dtype, true)), - )) + let array = Arc::new(StructArray::from(self.to_record_batch()?)) as ArrayRef; + Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))) } #[cfg(feature = "json")] @@ -222,7 +205,7 @@ impl VegaFusionTable { if let serde_json::Value::Array(values) = value { // Handle special case where array elements are non-object scalars let mut values = Cow::Borrowed(values); - if let Some(first) = values.get(0) { + if let Some(first) = values.first() { if let Value::Object(props) = first { // Handle odd special case where vega will interpret // [{}, {}] as [{"datum": {}}, {"datum": {}}] @@ -251,7 +234,7 @@ impl VegaFusionTable { "__dummy", ScalarValue::try_from(&DataType::Float64).unwrap(), )]); - let array = empty_scalar.to_array_of_size(values.len()); + let array = empty_scalar.to_array_of_size(values.len())?; let struct_array = array.as_any().downcast_ref::().unwrap(); let record_batch = RecordBatch::from(struct_array); Self::try_new(record_batch.schema(), vec![record_batch]) @@ -315,7 +298,7 @@ impl VegaFusionTable { let batches_list = PyList::new(py, batch_objects); // Convert table's schema into pyarrow schema - let schema = if let Some(batch) = self.batches.get(0) { + let schema = if let Some(batch) = self.batches.first() { // Get schema from first batch if present batch.schema() } else { diff --git a/vegafusion-common/src/datatypes.rs b/vegafusion-common/src/datatypes.rs index debeb7846..784e343f2 100644 --- a/vegafusion-common/src/datatypes.rs +++ b/vegafusion-common/src/datatypes.rs @@ -1,7 +1,10 @@ use crate::error::{Result, ResultWithContext}; use arrow::datatypes::DataType; use datafusion_common::DFSchema; -use datafusion_expr::{coalesce, expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, TryCast}; +use datafusion_expr::{ + coalesce, expr, lit, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, + TryCast, +}; pub fn is_numeric_datatype(dtype: &DataType) -> bool { matches!( @@ -87,7 +90,9 @@ pub fn to_numeric(value: Expr, schema: &DFSchema) -> Result { // Convert to milliseconds Expr::TryCast(TryCast { expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::ToTimestampMillis, + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::ToTimestampMillis, + ), args: vec![value], })), data_type: DataType::Int64, diff --git a/vegafusion-core/Cargo.toml b/vegafusion-core/Cargo.toml index 220d3a0b4..5363b1333 100644 --- a/vegafusion-core/Cargo.toml +++ b/vegafusion-core/Cargo.toml @@ -44,7 +44,6 @@ version = "1.6.4" [dependencies.datafusion-common] workspace = true -default_features = false [dependencies.pyo3] workspace = true diff --git a/vegafusion-core/src/expression/visitors.rs b/vegafusion-core/src/expression/visitors.rs index 2c5100101..330f35e47 100644 --- a/vegafusion-core/src/expression/visitors.rs +++ b/vegafusion-core/src/expression/visitors.rs @@ -105,7 +105,7 @@ impl ExpressionVisitor for GetInputVariablesVisitor { /// Collect data and scale identifiers. These show up as a literal string as the first /// argument to a Data or Scale callable. fn visit_called_identifier(&mut self, node: &Identifier, args: &[Expression]) { - if let Some(arg0) = args.get(0) { + if let Some(arg0) = args.first() { if let Ok(arg0) = arg0.as_literal() { if let Value::String(arg0) = arg0.value() { // Check data callable @@ -157,7 +157,7 @@ impl UpdateVariablesExprVisitor { impl ExpressionVisitor for UpdateVariablesExprVisitor { fn visit_called_identifier(&mut self, node: &Identifier, args: &[Expression]) { if node.name == "modify" { - if let Some(arg0) = args.get(0) { + if let Some(arg0) = args.first() { if let Ok(arg0) = arg0.as_literal() { if let Value::String(arg0) = arg0.value() { // First arg is a string, which holds the name of the output dataset @@ -372,7 +372,7 @@ impl<'a> ExpressionVisitor for DatasetsColumnUsageVisitor<'a> { .. })), .. - }) = node.arguments.get(0) + }) = node.arguments.first() { // Resolve data variable let reference_data_var = Variable::new_data(reference_data_name); diff --git a/vegafusion-core/src/planning/lift_facet_aggregations.rs b/vegafusion-core/src/planning/lift_facet_aggregations.rs index 86d43f959..92d741e55 100644 --- a/vegafusion-core/src/planning/lift_facet_aggregations.rs +++ b/vegafusion-core/src/planning/lift_facet_aggregations.rs @@ -66,7 +66,7 @@ impl ExtractFacetAggregationsVisitor { } fn num_children_of_dataset(&self, name: &str, scope: &[u32]) -> usize { - let facet_dataset_var: ScopedVariable = (Variable::new_data(&name), Vec::from(scope)); + let facet_dataset_var: ScopedVariable = (Variable::new_data(name), Vec::from(scope)); let Some(facet_dataset_idx) = self.node_indexes.get(&facet_dataset_var) else { return 0; }; @@ -107,7 +107,7 @@ impl MutChartVisitor for ExtractFacetAggregationsVisitor { let mut lifted_transforms: Vec = Vec::new(); let agg = loop { - match child_dataset.transform.get(0).cloned() { + match child_dataset.transform.first().cloned() { None => { // End of transforms for this dataset, advance to child dataset if possible if self.num_children_of_dataset(&child_dataset.name, scope) != 1 { @@ -235,11 +235,9 @@ impl MutChartVisitor for ExtractFacetAggregationsVisitor { // Add lifted aggregate transform, potentially after the joinaggregate transform lifted_transforms.push(TransformSpec::Aggregate(agg)); - } else { - if lifted_transforms.is_empty() { - // No supported transforms found - return Ok(()); - } + } else if lifted_transforms.is_empty() { + // No supported transforms found + return Ok(()); } // Create facet dataset name and increment counter to keep names unique even if the same diff --git a/vegafusion-core/src/planning/projection_pushdown.rs b/vegafusion-core/src/planning/projection_pushdown.rs index d1c61cb99..75a414e00 100644 --- a/vegafusion-core/src/planning/projection_pushdown.rs +++ b/vegafusion-core/src/planning/projection_pushdown.rs @@ -701,8 +701,8 @@ impl<'a> MutChartVisitor for InsertProjectionVisitor<'a> { // so we can append a projection transform to limit the columns that are produced // Note: empty strings here seem to break vega, filter them out let proj_fields: Vec<_> = sorted(columns) + .filter(|&f| !f.is_empty()) .cloned() - .filter(|f| !f.is_empty()) .map(|f| escape_field(&f)) .collect(); diff --git a/vegafusion-core/src/planning/stringify_local_datetimes.rs b/vegafusion-core/src/planning/stringify_local_datetimes.rs index a9c8b769d..693b654fe 100644 --- a/vegafusion-core/src/planning/stringify_local_datetimes.rs +++ b/vegafusion-core/src/planning/stringify_local_datetimes.rs @@ -42,8 +42,8 @@ pub fn stringify_local_datetimes( let server_to_client_datasets: HashSet<_> = comm_plan .server_to_client .iter() + .filter(|&var| var.0.namespace == VariableNamespace::Data as i32) .cloned() - .filter(|var| var.0.namespace == VariableNamespace::Data as i32) .collect(); let mut visitor = CollectCandidateDatasetMapping::new( @@ -407,7 +407,7 @@ impl<'a> MutChartVisitor for StringifyLocalDatetimeFieldsVisitor<'a> { let source_resolved_var = (source_resolved.var, source_resolved.scope); if let Some(fields) = self.local_datetime_fields.get(&source_resolved_var) { for field in sorted(fields) { - let field = unescape_field(&field); + let field = unescape_field(field); let expr_str = format!("toDate(datum['{field}'], 'local')"); let transforms = &mut data.transform; let transform = FormulaTransformSpec { diff --git a/vegafusion-core/src/spec/data.rs b/vegafusion-core/src/spec/data.rs index 6202ab670..c3fa0f81c 100644 --- a/vegafusion-core/src/spec/data.rs +++ b/vegafusion-core/src/spec/data.rs @@ -85,7 +85,7 @@ impl DataSpec { } else if self.url.is_some() { DependencyNodeSupported::PartiallySupported } else { - match self.transform.get(0) { + match self.transform.first() { Some(tx) if tx.supported_and_allowed(planner_config, task_scope, scope) => { DependencyNodeSupported::PartiallySupported } diff --git a/vegafusion-core/src/spec/transform/bin.rs b/vegafusion-core/src/spec/transform/bin.rs index ec169165d..2524c5502 100644 --- a/vegafusion-core/src/spec/transform/bin.rs +++ b/vegafusion-core/src/spec/transform/bin.rs @@ -125,7 +125,7 @@ impl TransformSpecTrait for BinTransformSpec { let bin_start = self .as_ .as_ref() - .and_then(|as_| as_.get(0).cloned()) + .and_then(|as_| as_.first().cloned()) .unwrap_or_else(|| "bin0".to_string()); let bin_end = self .as_ diff --git a/vegafusion-core/src/spec/transform/fold.rs b/vegafusion-core/src/spec/transform/fold.rs index 6cac92cb2..a925773ef 100644 --- a/vegafusion-core/src/spec/transform/fold.rs +++ b/vegafusion-core/src/spec/transform/fold.rs @@ -24,7 +24,7 @@ impl FoldTransformSpec { pub fn as_(&self) -> Vec { let as_ = self.as_.clone().unwrap_or_default(); vec![ - as_.get(0).cloned().unwrap_or_else(|| "key".to_string()), + as_.first().cloned().unwrap_or_else(|| "key".to_string()), as_.get(1).cloned().unwrap_or_else(|| "value".to_string()), ] } diff --git a/vegafusion-core/src/spec/transform/pivot.rs b/vegafusion-core/src/spec/transform/pivot.rs index 7ba7c7a03..1ae11bd1e 100644 --- a/vegafusion-core/src/spec/transform/pivot.rs +++ b/vegafusion-core/src/spec/transform/pivot.rs @@ -67,7 +67,7 @@ impl TransformSpecTrait for PivotTransformSpec { .unwrap_or_default() .iter() .filter_map(|groupby_field| { - let unescaped = unescape_field(&groupby_field); + let unescaped = unescape_field(groupby_field); if input_local_datetime_columns.contains(&unescaped) { Some(unescaped) } else { diff --git a/vegafusion-core/src/spec/transform/timeunit.rs b/vegafusion-core/src/spec/transform/timeunit.rs index 6f9f30d2f..e9df4aa0d 100644 --- a/vegafusion-core/src/spec/transform/timeunit.rs +++ b/vegafusion-core/src/spec/transform/timeunit.rs @@ -67,7 +67,7 @@ impl TimeUnitTransformSpec { let as0 = self .as_ .clone() - .and_then(|as_| as_.get(0).cloned()) + .and_then(|as_| as_.first().cloned()) .unwrap_or_else(|| "unit0".to_string()); let as1 = self .as_ diff --git a/vegafusion-core/src/task_graph/memory.rs b/vegafusion-core/src/task_graph/memory.rs index 68676b51b..7d1702ab7 100644 --- a/vegafusion-core/src/task_graph/memory.rs +++ b/vegafusion-core/src/task_graph/memory.rs @@ -1,3 +1,4 @@ +use crate::arrow::array::ListArray; use datafusion_common::ScalarValue; use std::mem::{size_of, size_of_val}; use vegafusion_common::arrow::array::ArrayRef; @@ -47,27 +48,16 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { ScalarValue::LargeUtf8(Some(s)) => size_of_val(s.as_bytes()) + size_of::(), ScalarValue::Binary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), ScalarValue::LargeBinary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), - ScalarValue::List(Some(values), field) => { - let values_bytes: usize = size_of::>() - + values - .iter() - .map(|v| size_of::() + inner_size_of_scalar(v)) - .sum::(); - - let dtype_bytes = size_of::() + inner_size_of_dtype(field.data_type()); - - values_bytes + dtype_bytes - } - ScalarValue::Struct(Some(values), fields) => { - let values_bytes: usize = size_of::>() - + values - .iter() - .map(|v| size_of::() + inner_size_of_scalar(v)) - .sum::(); - + ScalarValue::List(array) => size_of::>() + size_of_list_array(array), + ScalarValue::Struct(sa) => { + let fields = sa.fields(); let fields_bytes: usize = size_of::>() + fields.iter().map(size_of_field).sum::(); - + let values_bytes: usize = sa + .columns() + .iter() + .map(|col| col.get_array_memory_size()) + .sum(); values_bytes + fields_bytes } _ => { @@ -77,6 +67,13 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { } } +pub fn size_of_list_array(array: &ListArray) -> usize { + array + .iter() + .map(|el| el.map(|el| size_of_array_ref(&el)).unwrap_or(0)) + .sum() +} + pub fn size_of_array_ref(array: &ArrayRef) -> usize { array.get_array_memory_size() + inner_size_of_dtype(array.data_type()) + size_of::() } diff --git a/vegafusion-core/src/task_graph/task_value.rs b/vegafusion-core/src/task_graph/task_value.rs index 734ca1eb4..f84d32328 100644 --- a/vegafusion-core/src/task_graph/task_value.rs +++ b/vegafusion-core/src/task_graph/task_value.rs @@ -70,7 +70,7 @@ impl TryFrom<&TaskValue> for ProtoTaskValue { fn try_from(value: &TaskValue) -> std::result::Result { match value { TaskValue::Scalar(scalar) => { - let scalar_array = scalar.to_array(); + let scalar_array = scalar.to_array()?; let scalar_rb = RecordBatch::try_from_iter(vec![("value", scalar_array)])?; let ipc_bytes = VegaFusionTable::from(scalar_rb).to_ipc_bytes()?; Ok(Self { diff --git a/vegafusion-core/src/transform/bin.rs b/vegafusion-core/src/transform/bin.rs index 58359d5ae..6fe72d378 100644 --- a/vegafusion-core/src/transform/bin.rs +++ b/vegafusion-core/src/transform/bin.rs @@ -39,7 +39,7 @@ impl Bin { Ok(Self { field, extent: Some(extent_expr), - alias_0: as_.get(0).cloned(), + alias_0: as_.first().cloned(), alias_1: as_.get(1).cloned(), anchor: config.anchor, maxbins: config.maxbins, diff --git a/vegafusion-core/src/transform/stack.rs b/vegafusion-core/src/transform/stack.rs index a6b836745..6b9bd9a5c 100644 --- a/vegafusion-core/src/transform/stack.rs +++ b/vegafusion-core/src/transform/stack.rs @@ -47,7 +47,7 @@ impl Stack { // Extract aliases let alias0 = spec .as_() - .get(0) + .first() .cloned() .unwrap_or_else(|| "y0".to_string()); let alias1 = spec diff --git a/vegafusion-core/src/transform/timeunit.rs b/vegafusion-core/src/transform/timeunit.rs index 555b59330..a84bef850 100644 --- a/vegafusion-core/src/transform/timeunit.rs +++ b/vegafusion-core/src/transform/timeunit.rs @@ -16,7 +16,7 @@ impl TimeUnit { .collect(); let signal = transform.signal.clone(); - let alias_0 = transform.as_.as_ref().and_then(|v| v.get(0).cloned()); + let alias_0 = transform.as_.as_ref().and_then(|v| v.first().cloned()); let alias_1 = transform.as_.as_ref().and_then(|v| v.get(1).cloned()); let timezone = match &transform.timezone { diff --git a/vegafusion-dataframe/src/dataframe.rs b/vegafusion-dataframe/src/dataframe.rs index 7e2f39b45..4a72ac6b0 100644 --- a/vegafusion-dataframe/src/dataframe.rs +++ b/vegafusion-dataframe/src/dataframe.rs @@ -4,10 +4,7 @@ use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::{ - expr, window_function, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, - WindowFrameUnits, -}; +use datafusion_expr::{expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFunctionDefinition}; use std::any::Any; use std::fmt::{Display, Formatter}; use std::sync::Arc; @@ -33,7 +30,7 @@ pub trait DataFrame: Send + Sync + 'static { async fn collect_flat(&self) -> Result { let mut arrow_schema = Arc::new(self.schema()) as SchemaRef; let table = self.collect().await?; - if let Some(batch) = table.batches.get(0) { + if let Some(batch) = table.batches.first() { arrow_schema = batch.schema() } concat_batches(&arrow_schema, table.batches.as_slice()) @@ -112,24 +109,20 @@ pub trait DataFrame: Send + Sync + 'static { async fn with_index(&self, index_name: &str) -> Result> { if self.schema().column_with_name(index_name).is_some() { // Column is already present, don't overwrite - self.select(vec![Expr::Wildcard]).await + self.select(vec![Expr::Wildcard { qualifier: None }]).await } else { let selections = vec![ Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], partition_by: vec![], order_by: vec![], - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(index_name), - Expr::Wildcard, + Expr::Wildcard { qualifier: None }, ]; self.select(selections).await } diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index 7e6fbc27a..8afefd60a 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use vegafusion_common::arrow::array::{Array, ArrayRef, UInt32Array}; use vegafusion_common::arrow::compute::sort_to_indices; use vegafusion_common::arrow::datatypes::{DataType, Field, FieldRef}; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{create_udaf, Accumulator, AggregateUDF, Volatility}; @@ -18,9 +18,9 @@ pub(crate) struct PercentileContAccumulator { } impl Accumulator for PercentileContAccumulator { - fn state(&self) -> Result, DataFusionError> { - let state = ScalarValue::new_list(Some(self.all_values.clone()), self.data_type.clone()); - Ok(vec![state]) + fn state(&mut self) -> Result, DataFusionError> { + let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type); + Ok(vec![ScalarValue::List(state)]) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<(), DataFusionError> { @@ -43,14 +43,13 @@ impl Accumulator for PercentileContAccumulator { assert!(matches!(array.data_type(), DataType::List(_))); for index in 0..array.len() { match ScalarValue::try_from_array(array, index)? { - ScalarValue::List(Some(values), _) => { - for scalar in values { + ScalarValue::List(array) => { + for scalar in array.value(0).to_scalar_vec()? { if !scalar_is_non_finite(&scalar) { self.all_values.push(scalar); } } } - ScalarValue::List(None, _) => {} // skip empty state v => { return Err(DataFusionError::Internal(format!( "unexpected state in percentile_cont. Expected DataType::List, got {v:?}" @@ -61,7 +60,7 @@ impl Accumulator for PercentileContAccumulator { Ok(()) } - fn evaluate(&self) -> Result { + fn evaluate(&mut self) -> Result { if !self.all_values.iter().any(|v| !v.is_null()) { return ScalarValue::try_from(&self.data_type); } diff --git a/vegafusion-datafusion-udfs/src/udfs/array/constructor.rs b/vegafusion-datafusion-udfs/src/udfs/array/constructor.rs deleted file mode 100644 index cd413a95c..000000000 --- a/vegafusion-datafusion-udfs/src/udfs/array/constructor.rs +++ /dev/null @@ -1,87 +0,0 @@ -use datafusion_physical_expr::functions::make_scalar_function; -use std::sync::Arc; -use std::vec; -use vegafusion_common::arrow::array::{ - Array, ArrayDataBuilder, ArrayRef, BooleanBufferBuilder, Int32Array, ListArray, -}; -use vegafusion_common::arrow::datatypes::{DataType, Field, Float64Type}; -use vegafusion_common::datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}; - -pub fn make_array_constructor_udf() -> ScalarUDF { - let array_constructor = |args: &[ArrayRef]| { - // Signature ensures arguments have the same type and length - let num_args = args.len(); - - // Return empty array as single row, empty array with Float64 data type - if num_args == 0 || num_args == 1 && args[0].data_type() == &DataType::Null { - let empty_row = vec![Some(Vec::>::new())]; - let empty = ListArray::from_iter_primitive::(empty_row); - return Ok(Arc::new(empty) as ArrayRef); - } - - let num_rows = args[0].len(); - let element_dtype = args[0].data_type(); - let array_dtype = DataType::List(Arc::new(Field::new("item", element_dtype.clone(), true))); - - // Concatenate arrays into single flat array (in column-major ordering) - let arrays: Vec<&dyn Array> = args.iter().map(|a| a.as_ref()).collect(); - let concatted = vegafusion_common::arrow::compute::concat(&arrays).unwrap(); - - // Transpose array into row-major ordering and compute offset indices - let mut indices_builder = Int32Array::builder(num_rows * num_args); - let mut flat_valid_builder = BooleanBufferBuilder::new(num_rows * num_args); - let mut offsets_builder = Int32Array::builder(num_rows + 1); - - offsets_builder.append_value(0); - - for r in 0..num_rows { - for (a, arg) in args.iter().enumerate() { - let col_major_idx = (a * num_rows + r) as i32; - indices_builder.append_value(col_major_idx); - flat_valid_builder.append(arg.is_valid(r)); - } - offsets_builder.append_value((num_args * (r + 1)) as i32); - } - - let flat_values = vegafusion_common::arrow::compute::take( - concatted.as_ref(), - &indices_builder.finish(), - Default::default(), - ) - .unwrap(); - let offsets = offsets_builder.finish(); - - // Build ListArray data - let list_array_data = ArrayDataBuilder::new(array_dtype) - .len(num_rows) - .null_bit_buffer(Some(flat_valid_builder.finish().into_inner())) - .add_buffer(offsets.to_data().buffers()[0].clone()) - .add_child_data(flat_values.to_data()) - .build()?; - - Ok(Arc::new(ListArray::from(list_array_data)) as ArrayRef) - }; - let array_constructor = make_scalar_function(array_constructor); - - let return_type: ReturnTypeFunction = Arc::new(move |args| { - Ok(Arc::new(DataType::List(Arc::new(Field::new( - "item", - if args.is_empty() { - DataType::Float64 - } else { - args[0].clone() - }, - true, - ))))) - }); - ScalarUDF::new( - "make_list", - &Signature::variadic_equal(Volatility::Immutable), - &return_type, - &array_constructor, - ) -} - -lazy_static! { - pub static ref ARRAY_CONSTRUCTOR_UDF: ScalarUDF = make_array_constructor_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs index 9145ea710..4e8e6cbf5 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs @@ -1,4 +1,5 @@ use ordered_float::NotNan; +use std::any::Any; use std::collections::HashMap; use std::sync::Arc; use vegafusion_common::arrow::array::{ @@ -6,12 +7,9 @@ use vegafusion_common::arrow::array::{ }; use vegafusion_common::arrow::compute::cast; use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, -}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; use vegafusion_common::datatypes::{is_numeric_datatype, is_string_datatype}; /// `indexof(array, value)` @@ -20,12 +18,47 @@ use vegafusion_common::datatypes::{is_numeric_datatype, is_string_datatype}; /// /// See https://vega.github.io/vega/docs/expressions/#indexof /// and https://vega.github.io/vega/docs/expressions/#string_indexof -fn make_indexof_udf() -> ScalarUDF { - let indexof_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct IndexOfUDF { + signature: Signature, +} + +impl Default for IndexOfUDF { + fn default() -> Self { + Self::new() + } +} + +impl IndexOfUDF { + pub fn new() -> Self { + let signature = Signature::any(2, Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for IndexOfUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "indexof" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int32) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // Signature ensures there is a single argument let (array, array_dtype) = match &args[0] { - ColumnarValue::Scalar(ScalarValue::List(Some(scalar_array), field)) => { - (scalar_array.clone(), field.data_type().clone()) + ColumnarValue::Scalar(ScalarValue::List(array)) => { + let scalar_array = array.value(0).to_scalar_vec()?; + (scalar_array, array.value(0).data_type().clone()) } _ => { return Err(DataFusionError::Internal( @@ -111,16 +144,7 @@ fn make_indexof_udf() -> ScalarUDF { } } }) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int32))); - - ScalarUDF::new( - "indexof", - &Signature::any(2, Volatility::Immutable), - &return_type, - &indexof_fn, - ) + } } fn build_notnan_index_map(array: &[ScalarValue]) -> HashMap, i32> { @@ -137,7 +161,3 @@ fn build_notnan_index_map(array: &[ScalarValue]) -> HashMap, i32> { }) .collect::>() } - -lazy_static! { - pub static ref INDEXOF_UDF: ScalarUDF = make_indexof_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/array/length.rs b/vegafusion-datafusion-udfs/src/udfs/array/length.rs deleted file mode 100644 index 3f166cab2..000000000 --- a/vegafusion-datafusion-udfs/src/udfs/array/length.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::convert::TryFrom; -use std::sync::Arc; -use vegafusion_common::arrow::array::{new_null_array, Array, Float64Array, ListArray}; -use vegafusion_common::arrow::compute::kernels; -use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::datafusion_common::ScalarValue; -use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, -}; - -/// `length(array)` -/// -/// Returns the length of the input array or string -/// -/// See https://vega.github.io/vega/docs/expressions/#length -fn make_length_udf() -> ScalarUDF { - let length_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { - // Signature ensures there is a single argument - let arg = &args[0]; - Ok(match arg { - ColumnarValue::Scalar(value) => { - match value { - ScalarValue::List(Some(arr), _) => { - ColumnarValue::Scalar(ScalarValue::from(arr.len() as f64)) - } - ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => { - ColumnarValue::Scalar(ScalarValue::from(s.len() as f64)) - } - _ => { - // Scalar f64 null - ColumnarValue::Scalar(ScalarValue::try_from(&DataType::Float64).unwrap()) - } - } - } - ColumnarValue::Array(array) => { - match array.data_type() { - DataType::Utf8 | DataType::LargeUtf8 => { - // String length - ColumnarValue::Array( - kernels::cast( - &kernels::length::length(array.as_ref()).unwrap(), - &DataType::Float64, - ) - .unwrap(), - ) - } - DataType::FixedSizeList(_, n) => { - // Use scalar length - ColumnarValue::Scalar(ScalarValue::from(*n as f64)) - } - DataType::List(_) => { - let array = array.as_any().downcast_ref::().unwrap(); - let offsets = array.value_offsets(); - let mut length_builder = Float64Array::builder(array.len()); - - for i in 0..array.len() { - length_builder.append_value((offsets[i + 1] - offsets[i]) as f64); - } - - ColumnarValue::Array(Arc::new(length_builder.finish())) - } - _ => { - // Array of f64 - ColumnarValue::Array(new_null_array(&DataType::Float64, array.len())) - } - } - } - }) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64))); - ScalarUDF::new( - "len", - &Signature::any(1, Volatility::Immutable), - &return_type, - &length_fn, - ) -} - -lazy_static! { - pub static ref LENGTH_UDF: ScalarUDF = make_length_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/array/mod.rs b/vegafusion-datafusion-udfs/src/udfs/array/mod.rs index f0273cc8b..72e68adb0 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/mod.rs @@ -1,4 +1,2 @@ -pub mod constructor; pub mod indexof; -pub mod length; pub mod span; diff --git a/vegafusion-datafusion-udfs/src/udfs/array/span.rs b/vegafusion-datafusion-udfs/src/udfs/array/span.rs index 925818e47..5a6760ea2 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/span.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/span.rs @@ -1,11 +1,11 @@ +use std::any::Any; use std::sync::Arc; +use vegafusion_common::arrow::array::Array; +use vegafusion_common::arrow::array::ListArray; use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, -}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; /// `span(array)` /// @@ -13,8 +13,50 @@ use vegafusion_common::datafusion_expr::{ /// or array[array.length-1] - array[0]. /// /// See https://vega.github.io/vega/docs/expressions/#span -fn make_span_udf() -> ScalarUDF { - let span_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct SpanUDF { + signature: Signature, +} + +impl Default for SpanUDF { + fn default() -> Self { + Self::new() + } +} + +impl SpanUDF { + pub fn new() -> Self { + let signature = Signature::uniform( + 1, + vec![ + DataType::Float64, // For null + DataType::Null, // For null + DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for SpanUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "span" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // Signature ensures there is a single argument let arg = &args[0]; Ok(match arg { @@ -25,22 +67,31 @@ fn make_span_udf() -> ScalarUDF { // Span of scalar (including null) is 0 ColumnarValue::Scalar(ScalarValue::from(0.0)) } - ScalarValue::List(Some(arr), element_type) => { - match element_type.data_type() { - DataType::Float64 => { - if arr.is_empty() { - // Span of empty array is 0 - ColumnarValue::Scalar(ScalarValue::from(0.0)) - } else { - let first = arr.first().unwrap().to_f64().unwrap(); - let last = arr.last().unwrap().to_f64().unwrap(); + ScalarValue::List(arr) => { + // Unwrap single element ListArray + let arr = arr.as_any().downcast_ref::().unwrap(); + let arr = arr.value(0); + if arr.is_empty() { + ColumnarValue::Scalar(ScalarValue::from(0.0)) + } else { + match arr.data_type() { + DataType::Float64 => { + let first = ScalarValue::try_from_array(&arr, 0) + .unwrap() + .to_f64() + .unwrap(); + let last = ScalarValue::try_from_array(&arr, arr.len() - 1) + .unwrap() + .to_f64() + .unwrap(); ColumnarValue::Scalar(ScalarValue::from(last - first)) } - } - _ => { - return Err(DataFusionError::Internal(format!( - "Unexpected element type for span function: {element_type}" - ))) + _ => { + return Err(DataFusionError::Internal(format!( + "Unexpected element type for span function: {}", + arr.data_type() + ))) + } } } } @@ -55,25 +106,5 @@ fn make_span_udf() -> ScalarUDF { todo!("Span on column not yet implemented") } }) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64))); - ScalarUDF::new( - "span", - &Signature::uniform( - 1, - vec![ - DataType::Float64, // For null - DataType::Null, // For null - DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), - ], - Volatility::Immutable, - ), - &return_type, - &span_fn, - ) -} - -lazy_static! { - pub static ref SPAN_UDF: ScalarUDF = make_span_udf(); + } } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs index 390c292c0..971f29705 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_add_tz.rs @@ -1,52 +1,84 @@ -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use std::any::Any; +use vegafusion_common::datafusion_common::DataFusionError; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_add_tz_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DateAddTzUDF { + signature: Signature, +} + +impl Default for DateAddTzUDF { + fn default() -> Self { + Self::new() + } +} + +impl DateAddTzUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Date32, + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Date64, + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, + DataType::Int32, + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DateAddTzUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_add_tz" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + _args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { unimplemented!("date_add_tz function is not implemented by DataFusion") - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Date32, - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Date64, - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, - DataType::Int32, - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("date_add_tz", &signature, &return_type, &scalar_fn) + } +} + +fn make_date_add_tz_udf() -> ScalarUDF { + ScalarUDF::from(DateAddTzUDF::new()) } lazy_static! { diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs index d358ae35a..e322b92ad 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs @@ -1,24 +1,79 @@ use crate::udfs::datetime::from_utc_timestamp::from_utc_timestamp; use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; use datafusion_physical_expr::datetime_expressions; +use std::any::Any; use std::str::FromStr; -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, datafusion_common::DataFusionError, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_part_tz_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DatePartTzUDF { + signature: Signature, +} + +impl Default for DatePartTzUDF { + fn default() -> Self { + Self::new() + } +} + +impl DatePartTzUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date32, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date64, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, // timezone + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DatePartTzUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_part_tz" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // [1] data array let timestamp_array = match &args[1] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = to_timestamp_ms(×tamp_array)?; @@ -48,39 +103,9 @@ fn make_date_part_tz_udf() -> ScalarUDF { args[0].clone(), // Part timestamp_in_tz, // Timestamp converted to timezone ]) - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Float64))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date32, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date64, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, // timezone - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("date_part_tz", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref DATE_PART_TZ_UDF: ScalarUDF = make_date_part_tz_udf(); + pub static ref DATE_PART_TZ_UDF: ScalarUDF = ScalarUDF::from(DatePartTzUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs index f375023d8..ff00ecb9d 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs @@ -1,6 +1,8 @@ use chrono::{NaiveDateTime, TimeZone}; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, Date32Array, TimestampMillisecondArray}, @@ -8,18 +10,52 @@ use vegafusion_common::{ datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_to_utc_timestamp() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DateToUtcTimestampUDF { + signature: Signature, +} + +impl Default for DateToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl DateToUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::exact( + vec![DataType::Date32, DataType::Utf8], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DateToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { // [0] data array let date_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string @@ -61,24 +97,10 @@ fn make_date_to_utc_timestamp() -> ScalarUDF { } else { ScalarValue::try_from_array(×tamp_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = Signature::exact( - vec![DataType::Date32, DataType::Utf8], - Volatility::Immutable, - ); - - ScalarUDF::new( - "date_to_utc_timestamp", - &signature, - &return_type, - &scalar_fn, - ) + } } lazy_static! { - pub static ref DATE_TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_date_to_utc_timestamp(); + pub static ref DATE_TO_UTC_TIMESTAMP_UDF: ScalarUDF = + ScalarUDF::from(DateToUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs index 3314ce270..6035210bb 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_trunc_tz.rs @@ -1,50 +1,80 @@ -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use std::any::Any; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_date_trunc_tz_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct DateTruncTzUDF { + signature: Signature, +} + +impl Default for DateTruncTzUDF { + fn default() -> Self { + Self::new() + } +} + +impl DateTruncTzUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date32, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Date64, + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, // timezone + ]), + TypeSignature::Exact(vec![ + DataType::Utf8, // part + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, // timezone + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for DateTruncTzUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_trunc_tz" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + _args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { unimplemented!("date_trunc_tz function is not implemented by DataFusion") - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date32, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Date64, - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, // timezone - ]), - TypeSignature::Exact(vec![ - DataType::Utf8, // part - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, // timezone - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("date_trunc_tz", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref DATE_TRUNC_TZ_UDF: ScalarUDF = make_date_trunc_tz_udf(); + pub static ref DATE_TRUNC_TZ_UDF: ScalarUDF = ScalarUDF::from(DateTruncTzUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs index 7a8be8d31..57fc3903c 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs @@ -1,22 +1,60 @@ -use std::sync::Arc; +use std::any::Any; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ compute::cast, datatypes::{DataType, TimeUnit}, }, datafusion_common::ScalarValue, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_epoch_to_utc_timestamp() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct EpochMsToUtcTimestampUDF { + signature: Signature, +} + +impl Default for EpochMsToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl EpochMsToUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for EpochMsToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "epoch_ms_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = cast( @@ -30,21 +68,10 @@ fn make_epoch_to_utc_timestamp() -> ScalarUDF { } else { ScalarValue::try_from_array(×tamp_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); - - ScalarUDF::new( - "epoch_ms_to_utc_timestamp", - &signature, - &return_type, - &scalar_fn, - ) + } } lazy_static! { - pub static ref EPOCH_MS_TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_epoch_to_utc_timestamp(); + pub static ref EPOCH_MS_TO_UTC_TIMESTAMP_UDF: ScalarUDF = + ScalarUDF::from(EpochMsToUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs index 35228436b..e020f521b 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs @@ -1,25 +1,78 @@ use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; use chrono::NaiveDateTime; +use std::any::Any; use std::sync::Arc; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, StringArray, TimestampMillisecondArray}, datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - TypeSignature, Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, TypeSignature, Volatility}, }; -fn make_format_timestamp_udf() -> ScalarUDF { - let time_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct FormatTimestampUDF { + signature: Signature, +} + +impl Default for FormatTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl FormatTimestampUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for FormatTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "format_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Utf8) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // Argument order // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] time format string @@ -67,27 +120,7 @@ fn make_format_timestamp_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&formatted, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Utf8))); - - let signature: Signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), - TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("format_timestamp", &signature, &return_type, &time_fn) + } } fn convert_d3_format_string(d3_format_str: &str) -> String { @@ -99,5 +132,5 @@ fn convert_d3_format_string(d3_format_str: &str) -> String { } lazy_static! { - pub static ref FORMAT_TIMESTAMP_UDF: ScalarUDF = make_format_timestamp_udf(); + pub static ref FORMAT_TIMESTAMP_UDF: ScalarUDF = ScalarUDF::from(FormatTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs index 190abe336..974e19020 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs @@ -1,29 +1,82 @@ use chrono::NaiveDateTime; use chrono::TimeZone; use chrono_tz::Tz; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::Array; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, TimestampMillisecondArray}, datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - TypeSignature, Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, TypeSignature, Volatility}, }; use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; -fn make_from_utc_timestamp() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct FromUtcTimestampUDF { + signature: Signature, +} + +impl Default for FromUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl FromUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for FromUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "from_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string @@ -46,28 +99,7 @@ fn make_from_utc_timestamp() -> ScalarUDF { } else { ScalarValue::try_from_array(&result_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), - TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("from_utc_timestamp", &signature, &return_type, &scalar_fn) + } } pub fn from_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result { @@ -104,5 +136,5 @@ pub fn from_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result ScalarUDF { - let datetime_components: ScalarFunctionImplementation = - Arc::new(move |args: &[ColumnarValue]| { - let tz_str = if let ColumnarValue::Scalar(tz_scalar) = &args[7] { - tz_scalar.to_string() - } else { - return Err(DataFusionError::Internal( - "Expected timezone to be a scalar".to_string(), - )); - }; - - let input_tz = chrono_tz::Tz::from_str(&tz_str).map_err(|_err| { - DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) - })?; - - // first, identify if any of the arguments is an Array. If yes, store its `len`, - // as any scalar will need to be converted to an array of len `len`. - let len = args[..7] - .iter() - .fold(Option::::None, |acc, arg| match arg { - ColumnarValue::Scalar(_) => acc, - ColumnarValue::Array(a) => Some(a.len()), - }); - - // to arrays - let args = if let Some(len) = len { - args.iter() - .map(|arg| arg.clone().into_array(len)) - .collect::>() +#[derive(Debug, Clone)] +pub struct MakeUtcTimestampUDF { + signature: Signature, +} + +impl Default for MakeUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl MakeUtcTimestampUDF { + pub fn new() -> Self { + let signature = Signature::exact( + vec![ + DataType::Float64, // year + DataType::Float64, // month + DataType::Float64, // date + DataType::Float64, // hour + DataType::Float64, // minute + DataType::Float64, // second + DataType::Float64, // millisecond + DataType::Utf8, // time zone + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for MakeUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "make_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { + let tz_str = if let ColumnarValue::Scalar(tz_scalar) = &args[7] { + tz_scalar.to_string() + } else { + return Err(DataFusionError::Internal( + "Expected timezone to be a scalar".to_string(), + )); + }; + + let input_tz = chrono_tz::Tz::from_str(&tz_str).map_err(|_err| { + DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) + })?; + + // first, identify if any of the arguments is an Array. If yes, store its `len`, + // as any scalar will need to be converted to an array of len `len`. + let len = args[..7] + .iter() + .fold(Option::::None, |acc, arg| match arg { + ColumnarValue::Scalar(_) => acc, + ColumnarValue::Array(a) => Some(a.len()), + }); + + // to arrays + let args = if let Some(len) = len { + args.iter() + .map(|arg| arg.clone().into_array(len)) + .collect::, DataFusionError>>()? + } else { + args.iter() + .map(|arg| arg.clone().into_array(1)) + .collect::, DataFusionError>>()? + }; + + // To int64 arrays + let years = cast(&args[0], &DataType::Int64).unwrap(); + let years = years.as_any().downcast_ref::().unwrap(); + + let months = cast(&args[1], &DataType::Int64).unwrap(); + let months = months.as_any().downcast_ref::().unwrap(); + + let days = cast(&args[2], &DataType::Int64).unwrap(); + let days = days.as_any().downcast_ref::().unwrap(); + + let hours = cast(&args[3], &DataType::Int64).unwrap(); + let hours = hours.as_any().downcast_ref::().unwrap(); + + let minutes = cast(&args[4], &DataType::Int64).unwrap(); + let minutes = minutes.as_any().downcast_ref::().unwrap(); + + let seconds = cast(&args[5], &DataType::Int64).unwrap(); + let seconds = seconds.as_any().downcast_ref::().unwrap(); + + let millis = cast(&args[6], &DataType::Int64).unwrap(); + let millis = millis.as_any().downcast_ref::().unwrap(); + + let num_rows = years.len(); + let mut datetime_builder = TimestampMillisecondBuilder::new(); + + for i in 0..num_rows { + if years.is_null(i) + || months.is_null(i) + || days.is_null(i) + || hours.is_null(i) + || minutes.is_null(i) + || seconds.is_null(i) + || millis.is_null(i) + { + // If any component is null, propagate null + datetime_builder.append_null(); } else { - args.iter() - .map(|arg| arg.clone().into_array(1)) - .collect::>() - }; - - // To int64 arrays - let years = cast(&args[0], &DataType::Int64).unwrap(); - let years = years.as_any().downcast_ref::().unwrap(); - - let months = cast(&args[1], &DataType::Int64).unwrap(); - let months = months.as_any().downcast_ref::().unwrap(); - - let days = cast(&args[2], &DataType::Int64).unwrap(); - let days = days.as_any().downcast_ref::().unwrap(); - - let hours = cast(&args[3], &DataType::Int64).unwrap(); - let hours = hours.as_any().downcast_ref::().unwrap(); - - let minutes = cast(&args[4], &DataType::Int64).unwrap(); - let minutes = minutes.as_any().downcast_ref::().unwrap(); - - let seconds = cast(&args[5], &DataType::Int64).unwrap(); - let seconds = seconds.as_any().downcast_ref::().unwrap(); - - let millis = cast(&args[6], &DataType::Int64).unwrap(); - let millis = millis.as_any().downcast_ref::().unwrap(); - - let num_rows = years.len(); - let mut datetime_builder = TimestampMillisecondBuilder::new(); - - for i in 0..num_rows { - if years.is_null(i) - || months.is_null(i) - || days.is_null(i) - || hours.is_null(i) - || minutes.is_null(i) - || seconds.is_null(i) - || millis.is_null(i) - { - // If any component is null, propagate null - datetime_builder.append_null(); + let year = years.value(i); + let month = months.value(i); + let day = days.value(i); + let hour = hours.value(i); + let minute = minutes.value(i); + let second = seconds.value(i); + let milli = millis.value(i); + + // Treat 00-99 as 1900 to 1999 + let mut year = year; + if (0..100).contains(&year) { + year += 1900 + } + + let datetime: Option> = input_tz + .with_ymd_and_hms( + year as i32, + month as u32 + 1, + day as u32, + hour as u32, + minute as u32, + second as u32, + ) + .single() + .and_then(|date: DateTime<_>| date.with_nanosecond((milli * 1_000_000) as u32)); + + if let Some(datetime) = datetime { + // Always convert to UTC + let datetime = datetime.with_timezone(&chrono::Utc); + datetime_builder.append_value(datetime.timestamp_millis()); } else { - let year = years.value(i); - let month = months.value(i); - let day = days.value(i); - let hour = hours.value(i); - let minute = minutes.value(i); - let second = seconds.value(i); - let milli = millis.value(i); - - // Treat 00-99 as 1900 to 1999 - let mut year = year; - if (0..100).contains(&year) { - year += 1900 - } - - let datetime: Option> = input_tz - .with_ymd_and_hms( - year as i32, - month as u32 + 1, - day as u32, - hour as u32, - minute as u32, - second as u32, - ) - .single() - .and_then(|date: DateTime<_>| { - date.with_nanosecond((milli * 1_000_000) as u32) - }); - - if let Some(datetime) = datetime { - // Always convert to UTC - let datetime = datetime.with_timezone(&chrono::Utc); - datetime_builder.append_value(datetime.timestamp_millis()); - } else { - // Invalid date - datetime_builder.append_null(); - } + // Invalid date + datetime_builder.append_null(); } } + } - let result = Arc::new(datetime_builder.finish()) as ArrayRef; + let result = Arc::new(datetime_builder.finish()) as ArrayRef; - // maybe back to scalar - if len.is_some() { - Ok(ColumnarValue::Array(result)) - } else { - ScalarValue::try_from_array(&result, 0).map(ColumnarValue::Scalar) - } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature = Signature::exact( - vec![ - DataType::Float64, // year - DataType::Float64, // month - DataType::Float64, // date - DataType::Float64, // hour - DataType::Float64, // minute - DataType::Float64, // second - DataType::Float64, // millisecond - DataType::Utf8, // time zone - ], - Volatility::Immutable, - ); - ScalarUDF::new( - "make_utc_timestamp", - &signature, - &return_type, - &datetime_components, - ) + // maybe back to scalar + if len.is_some() { + Ok(ColumnarValue::Array(result)) + } else { + ScalarValue::try_from_array(&result, 0).map(ColumnarValue::Scalar) + } + } } lazy_static! { - pub static ref MAKE_UTC_TIMESTAMP: ScalarUDF = make_utc_timestamp_udf(); + pub static ref MAKE_UTC_TIMESTAMP: ScalarUDF = ScalarUDF::from(MakeUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs index 13571862f..3c1a3d757 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/mod.rs @@ -49,10 +49,7 @@ pub fn process_input_datetime( let array: Int64Array = unary(array, |v| (v as i64) * ms_per_day); Arc::new(array) as ArrayRef as _ } - DataType::Date64 => { - let int_array = cast(arg, &DataType::Int64).unwrap(); - int_array - } + DataType::Date64 => (cast(arg, &DataType::Int64).unwrap()) as _, DataType::Int64 => arg.clone(), DataType::Float64 => cast(arg, &DataType::Int64).expect("Failed to cast float to int"), _ => { diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs index 151cbd103..bb8d633ee 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs @@ -3,13 +3,13 @@ use chrono::{ {DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, Utc}, }; use regex::Regex; +use std::any::Any; use std::{str::FromStr, sync::Arc}; use vegafusion_common::arrow::array::{ArrayRef, StringArray, TimestampMillisecondArray}; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; lazy_static! { @@ -329,12 +329,53 @@ pub fn datetime_strs_to_timestamp_millis( Arc::new(millis_array) as ArrayRef } -fn make_str_to_utc_timestamp_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct StrToUtcTimestampUDF { + signature: Signature, +} + +impl Default for StrToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl StrToUtcTimestampUDF { + pub fn new() -> Self { + let signature = + Signature::exact(vec![DataType::Utf8, DataType::Utf8], Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for StrToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "str_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let str_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string @@ -359,19 +400,12 @@ fn make_str_to_utc_timestamp_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(×tamp_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let signature: Signature = - Signature::exact(vec![DataType::Utf8, DataType::Utf8], Volatility::Immutable); - - ScalarUDF::new("str_to_utc_timestamp", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref STR_TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_str_to_utc_timestamp_udf(); + pub static ref STR_TO_UTC_TIMESTAMP_UDF: ScalarUDF = + ScalarUDF::from(StrToUtcTimestampUDF::new()); } #[test] diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs index b2eccd41d..517d83ade 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs @@ -1,5 +1,6 @@ use crate::udfs::datetime::process_input_datetime; use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, TimeZone, Timelike, Utc, Weekday}; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, Int64Array, TimestampMillisecondArray}; @@ -8,8 +9,7 @@ use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::arrow::temporal_conversions::date64_to_datetime; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - TypeSignature, Volatility, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; fn extract_bool(value: &ColumnarValue) -> std::result::Result { @@ -39,7 +39,7 @@ fn unpack_timeunit_udf_args( DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) })?; - let timestamp = columns[0].clone().into_array(1); + let timestamp = columns[0].clone().into_array(1)?; let timestamp = process_input_datetime(×tamp, &tz)?; Ok(( @@ -223,9 +223,77 @@ fn perform_timeunit_start_from_utc( dt_value } -fn make_timeunit_start_udf() -> ScalarUDF { - let timeunit_start: ScalarFunctionImplementation = Arc::new(|columns: &[ColumnarValue]| { - let (timestamp, tz, units_mask) = unpack_timeunit_udf_args(columns)?; +#[derive(Debug, Clone)] +pub struct TimeunitStartUDF { + signature: Signature, +} + +impl Default for TimeunitStartUDF { + fn default() -> Self { + Self::new() + } +} + +impl TimeunitStartUDF { + pub fn new() -> Self { + let make_sig = |timestamp_dtype: DataType| -> TypeSignature { + TypeSignature::Exact(vec![ + timestamp_dtype, // [0] timestamp + DataType::Utf8, // [1] timezone + DataType::Boolean, // [2] Year + DataType::Boolean, // [3] Quarter + DataType::Boolean, // [4] Month + DataType::Boolean, // [5] Date + DataType::Boolean, // [6] Week + DataType::Boolean, // [7] Day + DataType::Boolean, // [8] DayOfYear + DataType::Boolean, // [9] Hours + DataType::Boolean, // [10] Minutes + DataType::Boolean, // [11] Seconds + DataType::Boolean, // [12] Milliseconds + ]) + }; + + let signature = Signature::one_of( + vec![ + make_sig(DataType::Int64), + make_sig(DataType::Date64), + make_sig(DataType::Date32), + make_sig(DataType::Timestamp(TimeUnit::Millisecond, None)), + make_sig(DataType::Timestamp(TimeUnit::Nanosecond, None)), + ], + Volatility::Immutable, + ); + + Self { signature } + } +} + +impl ScalarUDFImpl for TimeunitStartUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "vega_timeunit" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { + let (timestamp, tz, units_mask) = unpack_timeunit_udf_args(args)?; let array = timestamp.as_any().downcast_ref::().unwrap(); let result_array: TimestampMillisecondArray = unary(array, |value| { @@ -233,43 +301,9 @@ fn make_timeunit_start_udf() -> ScalarUDF { }); Ok(ColumnarValue::Array(Arc::new(result_array) as ArrayRef)) - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_datatypes| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - let make_sig = |timestamp_dtype: DataType| -> TypeSignature { - TypeSignature::Exact(vec![ - timestamp_dtype, // [0] timestamp - DataType::Utf8, // [1] timezone - DataType::Boolean, // [2] Year - DataType::Boolean, // [3] Quarter - DataType::Boolean, // [4] Month - DataType::Boolean, // [5] Date - DataType::Boolean, // [6] Week - DataType::Boolean, // [7] Day - DataType::Boolean, // [8] DayOfYear - DataType::Boolean, // [9] Hours - DataType::Boolean, // [10] Minutes - DataType::Boolean, // [11] Seconds - DataType::Boolean, // [12] Milliseconds - ]) - }; - - let signature: Signature = Signature::one_of( - vec![ - make_sig(DataType::Int64), - make_sig(DataType::Date64), - make_sig(DataType::Date32), - make_sig(DataType::Timestamp(TimeUnit::Millisecond, None)), - make_sig(DataType::Timestamp(TimeUnit::Nanosecond, None)), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("vega_timeunit", &signature, &return_type, &timeunit_start) + } } lazy_static! { - pub static ref TIMEUNIT_START_UDF: ScalarUDF = make_timeunit_start_udf(); + pub static ref TIMEUNIT_START_UDF: ScalarUDF = ScalarUDF::from(TimeunitStartUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs index b35f93024..de4c8e8f4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs @@ -1,9 +1,11 @@ use chrono::TimeZone; use chrono::{NaiveDateTime, Timelike}; use chrono_tz::Tz; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::Array; +use vegafusion_common::datafusion_expr::ScalarUDFImpl; use vegafusion_common::{ arrow::{ array::{ArrayRef, TimestampMillisecondArray}, @@ -11,18 +13,58 @@ use vegafusion_common::{ datatypes::{DataType, TimeUnit}, }, datafusion_common::{DataFusionError, ScalarValue}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_to_utc_timestamp_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct ToUtcTimestampUDF { + signature: Signature, +} + +impl Default for ToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl ToUtcTimestampUDF { + pub fn new() -> Self { + // Signature should be (Timestamp, UTF8), but specifying Timestamp in the signature + // requires specifying the timezone explicitly, and DataFusion doesn't currently + // coerce between timezones. + let signature: Signature = Signature::any(2, Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for ToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string @@ -45,17 +87,7 @@ fn make_to_utc_timestamp_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&result_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - - // Signature should be (Timestamp, UTF8), but specifying Timestamp in the signature - // requires specifying the timezone explicitly, and DataFusion doesn't currently - // coerce between timezones. - let signature: Signature = Signature::any(2, Volatility::Immutable); - - ScalarUDF::new("to_utc_timestamp", &signature, &return_type, &scalar_fn) + } } pub fn to_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result { @@ -127,5 +159,5 @@ pub fn to_timestamp_ms(array: &ArrayRef) -> Result { } lazy_static! { - pub static ref TO_UTC_TIMESTAMP_UDF: ScalarUDF = make_to_utc_timestamp_udf(); + pub static ref TO_UTC_TIMESTAMP_UDF: ScalarUDF = ScalarUDF::from(ToUtcTimestampUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs index 01ef27608..3a19ff217 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs @@ -1,24 +1,69 @@ use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; -use std::sync::Arc; -use vegafusion_common::datafusion_expr::TypeSignature; +use std::any::Any; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::{ compute::cast, datatypes::{DataType, TimeUnit}, }, datafusion_common::ScalarValue, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_utc_timestamp_to_epoch_ms_udf() -> ScalarUDF { - let time_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct UtcTimestampToEpochUDF { + signature: Signature, +} + +impl Default for UtcTimestampToEpochUDF { + fn default() -> Self { + Self::new() + } +} + +impl UtcTimestampToEpochUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32]), + TypeSignature::Exact(vec![DataType::Date64]), + TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Millisecond, None)]), + TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Nanosecond, None)]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for UtcTimestampToEpochUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "utc_timestamp_to_epoch_ms" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Int64) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let data_array = to_timestamp_ms(&data_array)?; @@ -31,27 +76,10 @@ fn make_utc_timestamp_to_epoch_ms_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&result_array, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32]), - TypeSignature::Exact(vec![DataType::Date64]), - TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Millisecond, None)]), - TypeSignature::Exact(vec![DataType::Timestamp(TimeUnit::Nanosecond, None)]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new( - "utc_timestamp_to_epoch_ms", - &signature, - &return_type, - &time_fn, - ) + } } lazy_static! { - pub static ref UTC_TIMESTAMP_TO_EPOCH_MS: ScalarUDF = make_utc_timestamp_to_epoch_ms_udf(); + pub static ref UTC_TIMESTAMP_TO_EPOCH_MS: ScalarUDF = + ScalarUDF::from(UtcTimestampToEpochUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs index 12c4f6ec3..11fdaf907 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs @@ -1,26 +1,78 @@ use crate::udfs::datetime::from_utc_timestamp::from_utc_timestamp; use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; use chrono::NaiveDateTime; +use std::any::Any; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, StringArray, TimestampMillisecondArray}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; -use vegafusion_common::datafusion_expr::TypeSignature; +use vegafusion_common::datafusion_expr::{ScalarUDFImpl, TypeSignature}; use vegafusion_common::{ arrow::datatypes::{DataType, TimeUnit}, - datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, - }, + datafusion_expr::{ColumnarValue, ScalarUDF, Signature, Volatility}, }; -fn make_utc_timestamp_to_str_udf() -> ScalarUDF { - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +#[derive(Debug, Clone)] +pub struct UtcTimestampToStrUDF { + signature: Signature, +} + +impl Default for UtcTimestampToStrUDF { + fn default() -> Self { + Self::new() + } +} + +impl UtcTimestampToStrUDF { + pub fn new() -> Self { + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + ]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], + Volatility::Immutable, + ); + Self { signature } + } +} + +impl ScalarUDFImpl for UtcTimestampToStrUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "utc_timestamp_to_str" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> vegafusion_common::datafusion_common::Result { + Ok(DataType::Utf8) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> vegafusion_common::datafusion_common::Result { // Argument order // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = to_timestamp_ms(×tamp_array)?; @@ -74,29 +126,10 @@ fn make_utc_timestamp_to_str_udf() -> ScalarUDF { } else { ScalarValue::try_from_array(&formatted, 0).map(ColumnarValue::Scalar) } - }); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Utf8))); - - let signature = Signature::one_of( - vec![ - TypeSignature::Exact(vec![DataType::Date32, DataType::Utf8]), - TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Utf8, - ]), - TypeSignature::Exact(vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Utf8, - ]), - ], - Volatility::Immutable, - ); - - ScalarUDF::new("utc_timestamp_to_str", &signature, &return_type, &scalar_fn) + } } lazy_static! { - pub static ref UTC_TIMESTAMP_TO_STR_UDF: ScalarUDF = make_utc_timestamp_to_str_udf(); + pub static ref UTC_TIMESTAMP_TO_STR_UDF: ScalarUDF = + ScalarUDF::from(UtcTimestampToStrUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs index 631a90220..2769c9105 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/isfinite.rs @@ -1,46 +1,83 @@ -use datafusion_physical_expr::functions::make_scalar_function; +use datafusion_physical_expr::udf::ScalarUDF; +use std::any::Any; use std::sync::Arc; -use vegafusion_common::arrow::array::{ArrayRef, BooleanArray, Float32Array, Float64Array}; +use vegafusion_common::arrow::array::{BooleanArray, Float32Array, Float64Array}; use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}; +use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; /// `isFinite(value)` /// /// Returns true if value is a finite number. /// /// See: https://vega.github.io/vega/docs/expressions/#isFinite -fn make_is_finite_udf() -> ScalarUDF { - let is_finite = |args: &[ArrayRef]| { - // Signature ensures there is a single argument - let arg = &args[0]; +#[derive(Debug, Clone)] +pub struct IsFiniteUDF { + signature: Signature, +} - let is_finite_array = match arg.data_type() { - DataType::Float32 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_finite()) - } - DataType::Float64 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_finite()) +impl Default for IsFiniteUDF { + fn default() -> Self { + Self::new() + } +} + +impl IsFiniteUDF { + pub fn new() -> Self { + let signature = Signature::any(1, Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for IsFiniteUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "isfinite" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Boolean) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { + // Signature ensures there is a single argument + match &args[0] { + ColumnarValue::Scalar(arg) => { + let res = match arg { + ScalarValue::Float32(Some(v)) => v.is_finite(), + ScalarValue::Float64(Some(v)) => v.is_finite(), + _ => true, + }; + Ok(ColumnarValue::Scalar(ScalarValue::from(res))) } - _ => { - // No other type can be non-finite - BooleanArray::from(vec![true; arg.len()]) + ColumnarValue::Array(arg) => { + let is_finite_array = match arg.data_type() { + DataType::Float32 => { + let array = arg.as_any().downcast_ref::().unwrap(); + BooleanArray::from_unary(array, |a| a.is_finite()) + } + DataType::Float64 => { + let array = arg.as_any().downcast_ref::().unwrap(); + BooleanArray::from_unary(array, |a| a.is_finite()) + } + _ => { + // No other type can be non-finite + BooleanArray::from(vec![true; arg.len()]) + } + }; + Ok(ColumnarValue::Array(Arc::new(is_finite_array))) } - }; - Ok(Arc::new(is_finite_array) as ArrayRef) - }; - let is_finite = make_scalar_function(is_finite); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Boolean))); - ScalarUDF::new( - "isfinite", - &Signature::any(1, Volatility::Immutable), - &return_type, - &is_finite, - ) + } + } } lazy_static! { - pub static ref ISFINITE_UDF: ScalarUDF = make_is_finite_udf(); + pub static ref ISFINITE_UDF: ScalarUDF = ScalarUDF::from(IsFiniteUDF::new()); } diff --git a/vegafusion-datafusion-udfs/src/udfs/math/isnan.rs b/vegafusion-datafusion-udfs/src/udfs/math/isnan.rs deleted file mode 100644 index ed782977c..000000000 --- a/vegafusion-datafusion-udfs/src/udfs/math/isnan.rs +++ /dev/null @@ -1,50 +0,0 @@ -use datafusion_physical_expr::functions::make_scalar_function; -use std::sync::Arc; -use vegafusion_common::{ - arrow::{ - array::{ArrayRef, BooleanArray, Float32Array, Float64Array}, - datatypes::DataType, - }, - datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}, -}; - -/// isNaN(value) -/// -/// Returns true if value is not a number. Same as JavaScript’s Number.isNaN. -/// -/// See: https://vega.github.io/vega/docs/expressions/#isNaN -fn make_is_nan_udf() -> ScalarUDF { - let is_nan = |args: &[ArrayRef]| { - // Signature ensures there is a single argument - let arg = &args[0]; - - let is_nan_array = match arg.data_type() { - DataType::Float32 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_nan()) - } - DataType::Float64 => { - let array = arg.as_any().downcast_ref::().unwrap(); - BooleanArray::from_unary(array, |a| a.is_nan()) - } - _ => { - // No other type can be NaN - BooleanArray::from(vec![false; arg.len()]) - } - }; - Ok(Arc::new(is_nan_array) as ArrayRef) - }; - let is_nan = make_scalar_function(is_nan); - - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Boolean))); - ScalarUDF::new( - "isnan", - &Signature::any(1, Volatility::Immutable), - &return_type, - &is_nan, - ) -} - -lazy_static! { - pub static ref ISNAN_UDF: ScalarUDF = make_is_nan_udf(); -} diff --git a/vegafusion-datafusion-udfs/src/udfs/math/mod.rs b/vegafusion-datafusion-udfs/src/udfs/math/mod.rs index 4dfcdc32f..cbe4d8772 100644 --- a/vegafusion-datafusion-udfs/src/udfs/math/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/math/mod.rs @@ -1,2 +1 @@ pub mod isfinite; -pub mod isnan; diff --git a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs index 2e7921a94..24f9f47c9 100644 --- a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs @@ -1,78 +1,175 @@ -use datafusion_physical_expr::functions::make_scalar_function; -use std::sync::Arc; -use vegafusion_common::arrow::array::{ - new_null_array, Array, ArrayRef, Int32Array, ListArray, StructArray, -}; +use std::any::Any; +use vegafusion_common::arrow::array::{new_null_array, Array, Int32Array, ListArray, StructArray}; use vegafusion_common::arrow::compute::kernels; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ - ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, - Volatility, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; use vegafusion_common::error::{Result, VegaFusionError}; +#[derive(Debug, Clone)] +pub struct GetObjectMemberUDF { + field_type: DataType, + field_index: usize, + signature: Signature, + name: String, +} + +impl GetObjectMemberUDF { + pub fn new(property_name: String, object_type: DataType) -> Result { + let signature = Signature::exact(vec![object_type.clone()], Volatility::Immutable); + let name = format!("get[{property_name}]"); + + let (field_index, field_type) = if let DataType::Struct(fields) = &object_type { + match fields + .iter() + .enumerate() + .find(|(_i, f)| f.name() == &property_name) + { + Some((field_index, field)) => (field_index, field.data_type().clone()), + None => { + return Err(VegaFusionError::compilation(format!( + "No object property named {property_name}" + ))) + } + } + } else { + return Err(VegaFusionError::compilation( + "Target of object property access is not a Struct type", + )); + }; + + Ok(Self { + field_type, + field_index, + signature, + name, + }) + } +} + +impl ScalarUDFImpl for GetObjectMemberUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[DataType], + ) -> std::result::Result { + Ok(self.field_type.clone()) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> std::result::Result { + // Signature ensures there is a single argument + let arg = &args[0]; + + match arg { + ColumnarValue::Scalar(ScalarValue::Struct(arg)) => { + let c = arg.column(self.field_index); + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(c, 0)?)) + } + ColumnarValue::Array(object) => { + let struct_array = object.as_any().downcast_ref::().unwrap(); + Ok(ColumnarValue::Array( + struct_array.column(self.field_index).clone(), + )) + } + _ => Err(DataFusionError::Internal(format!( + "Unexpected object type for member access: {:?}", + arg.data_type() + ))), + } + } +} + pub fn make_get_object_member_udf( object_type: &DataType, property_name: &str, ) -> Result { - let (field_index, field_type) = if let DataType::Struct(fields) = object_type { - match fields - .iter() - .enumerate() - .find(|(_i, f)| f.name() == property_name) - { - Some((field_index, field)) => (field_index, field.data_type().clone()), - None => { - return Err(VegaFusionError::compilation(format!( - "No object property named {property_name}" - ))) - } - } - } else { - return Err(VegaFusionError::compilation( - "Target of object property access is not a Struct type", - )); - }; + Ok(ScalarUDF::from(GetObjectMemberUDF::new( + property_name.to_string(), + object_type.clone(), + )?)) +} - let get = move |args: &[ArrayRef]| { - // Signature ensures there is a single argument - let object = &args[0]; - - let struct_array = object.as_any().downcast_ref::().unwrap(); - Ok(struct_array.column(field_index).clone()) - }; - let get = make_scalar_function(get); - - let return_type: ReturnTypeFunction = - Arc::new(move |_dtype: &[DataType]| Ok(Arc::new(field_type.clone()))); - - Ok(ScalarUDF::new( - &format!("get[{property_name}]"), - &Signature::exact(vec![object_type.clone()], Volatility::Immutable), - &return_type, - &get, - )) +#[derive(Debug, Clone)] +pub struct ArrayElementUDF { + name: String, + signature: Signature, + index: i32, } -pub fn make_get_element_udf(index: i32) -> ScalarUDF { - let get_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { +impl ArrayElementUDF { + pub fn new(index: i32) -> Self { + let signature = Signature::any(1, Volatility::Immutable); + Self { + name: format!("get[{index}]"), + signature, + index, + } + } +} + +impl ScalarUDFImpl for ArrayElementUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + arg_types: &[DataType], + ) -> std::result::Result { + let new_dtype = match &arg_types[0] { + DataType::List(field) => field.data_type().clone(), + dtype => { + return Err(DataFusionError::Internal(format!( + "Unsupported datatype for get index {dtype:?}" + ))) + } + }; + Ok(new_dtype) + } + + fn invoke( + &self, + args: &[ColumnarValue], + ) -> std::result::Result { // Signature ensures there is a single argument let arg = &args[0]; Ok(match arg { ColumnarValue::Scalar(value) => { match value { - ScalarValue::List(Some(arr), element_dtype) => { - match arr.get(index as usize) { - Some(element) => { + ScalarValue::List(arr) => { + let arr = arr.as_any().downcast_ref::().unwrap(); + match ScalarValue::try_from_array(&arr.value(0), self.index as usize) { + Ok(element) => { // Scalar element of list ColumnarValue::Scalar(element.clone()) } - None => { + _ => { // out of bounds, null - ColumnarValue::Scalar( - ScalarValue::try_from(element_dtype.data_type()).unwrap(), - ) + ColumnarValue::Scalar(ScalarValue::try_from(arr.data_type())?) } } } @@ -95,8 +192,8 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { for i in 0..array.len() { let el_start = offsets[i]; let el_stop = offsets[i + 1]; - if el_start + index < el_stop { - take_index_builder.append_value(el_start + index); + if el_start + self.index < el_stop { + take_index_builder.append_value(el_start + self.index); } else { take_index_builder.append_null(); } @@ -115,23 +212,9 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { } } }) - }); + } +} - let return_type: ReturnTypeFunction = Arc::new(move |dtype| { - let new_dtype = match &dtype[0] { - DataType::List(field) => field.data_type().clone(), - dtype => { - return Err(DataFusionError::Internal(format!( - "Unsupported datatype for get index {dtype:?}" - ))) - } - }; - Ok(Arc::new(new_dtype)) - }); - ScalarUDF::new( - &format!("get[{index}]"), - &Signature::any(1, Volatility::Immutable), - &return_type, - &get_fn, - ) +pub fn make_get_element_udf(index: i32) -> ScalarUDF { + ScalarUDF::from(ArrayElementUDF::new(index)) } diff --git a/vegafusion-datafusion-udfs/src/udfs/mod.rs b/vegafusion-datafusion-udfs/src/udfs/mod.rs index 6c7bfe80d..40f35087d 100644 --- a/vegafusion-datafusion-udfs/src/udfs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/mod.rs @@ -3,3 +3,4 @@ pub mod datetime; pub mod math; pub mod member; pub mod object; +pub mod util; diff --git a/vegafusion-datafusion-udfs/src/udfs/object/mod.rs b/vegafusion-datafusion-udfs/src/udfs/object/mod.rs index 8924d568b..458464813 100644 --- a/vegafusion-datafusion-udfs/src/udfs/object/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/object/mod.rs @@ -1,42 +1,79 @@ -use datafusion_physical_expr::functions::make_scalar_function; +use crate::udfs::util::make_scalar_function; +use std::any::Any; use std::sync::Arc; use vegafusion_common::arrow::array::{ArrayRef, StructArray}; use vegafusion_common::arrow::datatypes::{DataType, Field, FieldRef, Fields}; -use vegafusion_common::datafusion_expr::{ReturnTypeFunction, ScalarUDF, Signature, Volatility}; +use vegafusion_common::datafusion_common::DataFusionError; +use vegafusion_common::datafusion_expr::{ + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, Volatility, +}; -pub fn make_object_constructor_udf(keys: &[String], value_types: &[DataType]) -> ScalarUDF { - // Build fields vector - let fields: Vec<_> = keys - .iter() - .zip(value_types.iter()) - .map(|(k, dtype)| Field::new(k, dtype.clone(), true)) - .collect(); +#[derive(Debug, Clone)] +struct ObjectConstructorUDF { + signature: Signature, + fields: Vec, + struct_dtype: DataType, + name: String, +} - let struct_dtype = DataType::Struct(Fields::from(fields.clone())); +impl ObjectConstructorUDF { + pub fn new(keys: &[String], value_types: &[DataType]) -> Self { + // Build fields vector + let fields: Vec<_> = keys + .iter() + .zip(value_types.iter()) + .map(|(k, dtype)| Field::new(k, dtype.clone(), true)) + .collect(); + let struct_dtype = DataType::Struct(Fields::from(fields.clone())); - let object_constructor = move |args: &[ArrayRef]| { - let pairs: Vec<_> = fields + // Build name + let name_csv: Vec<_> = keys .iter() - .zip(args.iter()) - .map(|(f, v)| (FieldRef::new(f.clone()), v.clone())) + .zip(value_types) + .map(|(k, dtype)| format!("{k}: {dtype}")) .collect(); - Ok(Arc::new(StructArray::from(pairs)) as ArrayRef) - }; + let name = format!("object{{{}}}", name_csv.join(",")); + Self { + signature: Signature::any(keys.len(), Volatility::Immutable), + fields, + struct_dtype, + name, + } + } +} - let object_constructor = make_scalar_function(object_constructor); +impl ScalarUDFImpl for ObjectConstructorUDF { + fn as_any(&self) -> &dyn Any { + self + } - let return_type: ReturnTypeFunction = Arc::new(move |_args| Ok(Arc::new(struct_dtype.clone()))); + fn name(&self) -> &str { + self.name.as_str() + } - let name_csv: Vec<_> = keys - .iter() - .zip(value_types) - .map(|(k, dtype)| format!("{k}: {dtype}")) - .collect(); + fn signature(&self) -> &Signature { + &self.signature + } - ScalarUDF::new( - &format!("object{{{}}}", name_csv.join(",")), - &Signature::any(keys.len(), Volatility::Immutable), - &return_type, - &object_constructor, - ) + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(self.struct_dtype.clone()) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { + let fields = self.fields.clone(); + let object_constructor = move |args: &[ArrayRef]| { + let pairs: Vec<_> = fields + .iter() + .zip(args.iter()) + .map(|(f, v)| (FieldRef::new(f.clone()), v.clone())) + .collect(); + Ok(Arc::new(StructArray::from(pairs)) as ArrayRef) + }; + let object_constructor = make_scalar_function(object_constructor); + object_constructor(args) + } +} + +pub fn make_object_constructor_udf(keys: &[String], value_types: &[DataType]) -> ScalarUDF { + ScalarUDF::from(ObjectConstructorUDF::new(keys, value_types)) } diff --git a/vegafusion-datafusion-udfs/src/udfs/util.rs b/vegafusion-datafusion-udfs/src/udfs/util.rs new file mode 100644 index 000000000..9d3a745ea --- /dev/null +++ b/vegafusion-datafusion-udfs/src/udfs/util.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; +use vegafusion_common::arrow::array::ArrayRef; +use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; +use vegafusion_common::datafusion_expr::{ColumnarValue, ScalarFunctionImplementation}; + +pub fn make_scalar_function(inner: F) -> ScalarFunctionImplementation +where + F: Fn(&[ArrayRef]) -> Result + Sync + Send + 'static, +{ + Arc::new(move |args: &[ColumnarValue]| { + // first, identify if any of the arguments is an Array. If yes, store its `len`, + // as any scalar will need to be converted to an array of len `len`. + let len = args + .iter() + .fold(Option::::None, |acc, arg| match arg { + ColumnarValue::Scalar(_) => acc, + ColumnarValue::Array(a) => Some(a.len()), + }); + + let is_scalar = len.is_none(); + + let inferred_length = len.unwrap_or(1); + let args = args + .iter() + .map(|arg| arg.clone().into_array(inferred_length)) + .collect::, DataFusionError>>()?; + + let result = (inner)(&args); + + if is_scalar { + // If all inputs are scalar, keeps output as scalar + let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); + result.map(ColumnarValue::Scalar) + } else { + result.map(ColumnarValue::Array) + } + }) +} diff --git a/vegafusion-python-embed/Cargo.toml b/vegafusion-python-embed/Cargo.toml index d3024d1dc..ce8a24094 100644 --- a/vegafusion-python-embed/Cargo.toml +++ b/vegafusion-python-embed/Cargo.toml @@ -15,7 +15,7 @@ protobuf-src = [ "vegafusion-core/protobuf-src",] [dependencies] log = "0.4.17" env_logger = "0.10.0" -pythonize = "0.19.0" +pythonize = { workspace=true } async-trait = "0.1.73" uuid = "1.3.0" diff --git a/vegafusion-python-embed/src/connection.rs b/vegafusion-python-embed/src/connection.rs index 2a74d5f74..b14fd19cc 100644 --- a/vegafusion-python-embed/src/connection.rs +++ b/vegafusion-python-embed/src/connection.rs @@ -96,7 +96,7 @@ impl Connection for PySqlConnection { let mut tables: HashMap = HashMap::new(); for key in tables_dict.keys() { - let value = tables_dict.get_item(key).unwrap(); + let value = tables_dict.get_item(key)?.unwrap(); let key_string = key.extract::()?; let value_schema = Schema::from_pyarrow(value)?; tables.insert(key_string, value_schema); diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index e527b079b..6d0fe00f6 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -42,7 +42,7 @@ futures-util = "0.3.21" rstest = "0.18.2" test-case = "3.1.0" base64 = "0.21.0" -dssim = "3.1.0" +pixelmatch = "0.1.0" rgb = "0.8.32" lodepng = "3.6.1" @@ -100,6 +100,9 @@ workspace = true [dependencies.datafusion-optimizer] workspace = true +[dependencies.datafusion-functions] +workspace = true + [dependencies.tokio] workspace = true features = [ "macros", "rt-multi-thread", "fs",] diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index 36b7ce263..ff7343ded 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -5,7 +5,7 @@ use crate::task_graph::task::TaskCall; use async_trait::async_trait; -use datafusion_expr::{expr, lit, Expr}; +use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::collections::{HashMap, HashSet}; use std::path::Path; @@ -336,16 +336,20 @@ async fn process_datetimes( .map(|tz_config| tz_config.default_input_tz.to_string()) .unwrap_or_else(|| "UTC".to_string()); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![flat_col(&spec.name), lit(default_input_tz_str)], }) } else if is_integer_datatype(dtype) { // Assume Year was parsed numerically, use local time let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*MAKE_UTC_TIMESTAMP).clone(), + )), args: vec![ flat_col(&spec.name), // year lit(0), // month @@ -395,8 +399,10 @@ async fn process_datetimes( DataType::Timestamp(_, tz) => match tz { Some(tz) => { // Timestamp has explicit timezone - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![flat_col(field.name()), lit(tz.as_ref())], }) } @@ -405,8 +411,10 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -418,8 +426,10 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -430,8 +440,10 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*DATE_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*DATE_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![flat_col(field.name()), lit(tz_config.local_tz.to_string())], }) } @@ -471,7 +483,7 @@ impl TaskCall for DataValuesTask { // (which generates its own data) if values_table.num_rows() == 0 { if let Some(pipeline) = &self.pipeline { - if let Some(first_tx) = pipeline.transforms.get(0) { + if let Some(first_tx) = pipeline.transforms.first() { if !matches!(first_tx.transform_kind(), TransformKind::Sequence(_)) { return Ok((TaskValue::Table(values_table), Default::default())); } diff --git a/vegafusion-runtime/src/expression/compiler/array.rs b/vegafusion-runtime/src/expression/compiler/array.rs index 9b95869f2..9732b2ec9 100644 --- a/vegafusion-runtime/src/expression/compiler/array.rs +++ b/vegafusion-runtime/src/expression/compiler/array.rs @@ -1,11 +1,8 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{expr, Expr}; -use std::ops::Deref; -use std::sync::Arc; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_core::error::Result; use vegafusion_core::proto::gen::expression::ArrayExpression; -use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; pub fn compile_array( node: &ArrayExpression, @@ -17,8 +14,8 @@ pub fn compile_array( let phys_expr = compile(el, config, Some(schema))?; elements.push(phys_expr); } - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: elements, })) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs new file mode 100644 index 000000000..6bb710190 --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs @@ -0,0 +1,44 @@ +use datafusion_common::DFSchema; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition}; +use vegafusion_common::arrow::datatypes::DataType; +use vegafusion_common::error::{ResultWithContext, VegaFusionError}; + +pub fn length_transform( + args: &[Expr], + schema: &DFSchema, +) -> vegafusion_common::error::Result { + if args.len() == 1 { + let arg = args[0].clone(); + let dtype = arg + .get_type(schema) + .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; + + let len_expr = match dtype { + DataType::Utf8 | DataType::LargeUtf8 => Ok(Expr::Cast(expr::Cast { + expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::CharacterLength), + args: vec![arg], + })), + data_type: DataType::Float64 + })), + DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => Ok(Expr::Cast(expr::Cast { + expr: Box::new(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), + args: vec![arg], + })), + data_type: DataType::Float64 + })), + _ => Err(VegaFusionError::parse(format!( + "length function support array and string arguments. Received argument with type {:?}", + dtype + ))), + }?; + + Ok(len_expr.cast_to(&DataType::Float64, schema)?) + } else { + Err(VegaFusionError::parse(format!( + "length requires a single argument. Received {} arguments", + args.len() + ))) + } +} diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs new file mode 100644 index 000000000..687393174 --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/mod.rs @@ -0,0 +1,4 @@ +pub mod length; + +// // Span transform not in use yet +// pub mod span; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs new file mode 100644 index 000000000..22fca919a --- /dev/null +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/span.rs @@ -0,0 +1,51 @@ +use std::ops::Sub; +use datafusion_common::DFSchema; +use datafusion_expr::{BuiltinScalarFunction, Expr, expr, ExprSchemable, lit, ScalarFunctionDefinition, when}; +use vegafusion_common::arrow::datatypes::DataType; +use vegafusion_common::error::{ResultWithContext, VegaFusionError}; + +// Note: I believe this implementation of span, using built-in DataFusion functions, is correct. +// But the DataFusion simplifier doesn't seem to know how to simplify it, which is what we use for +// scalar evaluation, so we can't use it yet. +pub fn span_transform(args: &[Expr], schema: &DFSchema) -> vegafusion_common::error::Result { + if args.len() == 1 { + let arg = args[0].clone(); + let dtype = arg + .get_type(schema) + .with_context(|| format!("Failed to infer type of expression: {arg:?}"))?; + + match dtype { + DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) => { + if field.data_type().is_numeric() { + let len = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayLength), + args: vec![arg.clone()], + }).cast_to(&DataType::Int32, schema)?; + + let first_el = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayElement), + args: vec![arg.clone(), lit(1)], + }); + + let last_el = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::ArrayElement), + args: vec![arg.clone(), len.clone()], + }); + + Ok(when(len.eq(lit(0)), lit(0.0)).otherwise(last_el.sub(first_el))?) + } else { + Ok(lit(0.0)) + } + }, + _ => { + // Span of non-array is zero + Ok(lit(0.0)) + }, + } + } else { + Err(VegaFusionError::parse(format!( + "span requires a single argument. Received {} arguments", + args.len() + ))) + } +} \ No newline at end of file diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs index 9a63d94b3..cdfb61435 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs @@ -1,12 +1,11 @@ +use datafusion_common::utils::array_into_list_array; use datafusion_expr::{lit, Expr}; use itertools::Itertools; use std::collections::HashMap; use std::convert::TryFrom; use std::str::FromStr; use std::sync::Arc; - -use vegafusion_common::arrow::datatypes::{DataType, Field}; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; use vegafusion_common::error::{Result, VegaFusionError}; @@ -63,8 +62,8 @@ pub fn vl_selection_resolve_fn( let _op = parse_args(args)?; // Extract vector of rows for selection dataset - let rows = if let ScalarValue::List(Some(elements), _) = table.to_scalar_value()? { - elements + let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { + array.value(0).to_scalar_vec()? } else { unreachable!() }; @@ -79,16 +78,17 @@ pub fn vl_selection_resolve_fn( for (field, value) in row_spec.fields.iter().zip(&row_spec.values) { let value = match field.typ { SelectionType::Enum => { - if let ScalarValue::List(Some(elements), _) = value { - elements.clone() + if let ScalarValue::List(array) = value { + array.value(0).to_scalar_vec()? } else { vec![value.clone()] } } _ => { match &value { - ScalarValue::List(Some(elements), _) if elements.len() == 2 => { + ScalarValue::List(array) if array.value(0).len() == 2 => { // Don't assume elements are in ascending order + let elements = array.value(0).to_scalar_vec()?; let first = elements[0].to_f64()?; let second = elements[1].to_f64()?; @@ -98,9 +98,6 @@ pub fn vl_selection_resolve_fn( (second, first) }; vec![ScalarValue::from(low), ScalarValue::from(high)] - // ScalarValue::List(Some(Box::new(vec![ - // ScalarValue::from(low), ScalarValue::from(high) - // ])), Box::new(DataType::Float64)) } v => { return Err(VegaFusionError::internal(format!( @@ -111,25 +108,25 @@ pub fn vl_selection_resolve_fn( } }; - let values = props.entry(field.field.clone()).or_insert_with(Vec::new); + let values = props.entry(field.field.clone()).or_default(); values.extend(value.clone()); } } - let props: Vec<_> = props + let props = props .into_iter() .map(|(name, values)| { // Turn values into a scalar list - let dtype = values - .get(0) - .map(|s| s.data_type()) - .unwrap_or(DataType::Float64); - let values = ScalarValue::List(Some(values), Arc::new(Field::new("item", dtype, true))); - (name, values) + let values = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(values)?, + ))); + Ok((name, values)) }) + .collect::>>()?; + let props: Vec<_> = props + .iter() .sorted_by_key(|(n, _)| n.clone()) - .collect(); - + .collect::>(); let props: Vec<_> = props .iter() .map(|(name, value)| (name.as_str(), value.clone())) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index 6e6956555..ec7d6c047 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -5,12 +5,12 @@ use std::convert::TryFrom; use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_expr::expr::Case; -use datafusion_expr::{ceil, expr, lit, Between, Expr, ExprSchemable}; +use datafusion_expr::{ceil, expr, lit, Between, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::column::flat_col; -use vegafusion_common::data::scalar::ScalarValue; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValue}; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_common::datatypes::{ @@ -131,8 +131,10 @@ impl FieldSpec { field_col.get_type(schema)?, DataType::Timestamp(TimeUnit::Millisecond, _) ) { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![field_col], }) } else { @@ -142,8 +144,8 @@ impl FieldSpec { let expr = match self.typ { SelectionType::Enum => { let field_type = field_col.get_type(schema)?; - let list_scalars = if let ScalarValue::List(Some(elements), _) = &values { - elements.clone() + let list_scalars = if let ScalarValue::List(array) = &values { + array.value(0).to_scalar_vec()? } else { // convert values to single element list vec![values.clone()] @@ -178,7 +180,8 @@ impl FieldSpec { }; let (low, high) = match &values { - ScalarValue::List(Some(elements), _) if elements.len() == 2 => { + ScalarValue::List(array) if array.value(0).len() == 2 => { + let elements = array.value(0).to_scalar_vec()?; let first = Self::cast_test_scalar( elements[0].clone(), &field_dtype, @@ -269,12 +272,16 @@ impl FieldSpec { if parse_datetime(&s, &Some(chrono_tz::UTC)).is_some() && is_numeric_datatype(field_type) => { - let timestamp_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + let timestamp_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![lit(s), lit(default_input_tz)], }); - let ms_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), + let ms_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![timestamp_expr], }); cast_to(ms_expr, field_type, schema) @@ -303,7 +310,8 @@ impl TryFrom for FieldSpec { fn try_from(value: ScalarValue) -> Result { match value { - ScalarValue::Struct(Some(values), fields) => { + ScalarValue::Struct(sa) => { + let fields = sa.fields(); let field_names: HashMap<_, _> = fields .iter() .enumerate() @@ -315,8 +323,10 @@ impl TryFrom for FieldSpec { .get("field") .with_context(|| "Missing required property 'field'".to_string())?; - let field = match values.get(*field_index) { - Some(ScalarValue::Utf8(Some(field))) => field.clone(), + let field_value = ScalarValue::try_from_array(sa.column(*field_index), 0)?; + + let field = match field_value { + ScalarValue::Utf8(Some(field)) => field.clone(), _ => { return Err(VegaFusionError::internal( "Expected field to be a string".to_string(), @@ -328,7 +338,8 @@ impl TryFrom for FieldSpec { let typ_index = field_names .get("type") .with_context(|| "Missing required property 'type'".to_string())?; - let typ = SelectionType::try_from(values.get(*typ_index).unwrap().clone())?; + let type_value = ScalarValue::try_from_array(sa.column(*typ_index), 0)?; + let typ = SelectionType::try_from(type_value)?; Ok(Self { field, typ }) } @@ -393,8 +404,9 @@ impl TryFrom for SelectionRow { fn try_from(value: ScalarValue) -> Result { match value { - ScalarValue::Struct(Some(struct_values), struct_fields) => { - let field_names: HashMap<_, _> = struct_fields + ScalarValue::Struct(sa) => { + let fields = sa.fields(); + let field_names: HashMap<_, _> = fields .iter() .enumerate() .map(|(ind, f)| (f.name().clone(), ind)) @@ -404,8 +416,9 @@ impl TryFrom for SelectionRow { let values_index = field_names .get("values") .with_context(|| "Missing required property 'values'".to_string())?; - let values = match struct_values.get(*values_index) { - Some(ScalarValue::List(Some(elements), _)) => elements.clone(), + let struct_value = ScalarValue::try_from_array(sa.column(*values_index), 0)?; + let values = match struct_value { + ScalarValue::List(array) => array.value(0).to_scalar_vec()?, _ => { return Err(VegaFusionError::internal( "Expected 'values' to be an array".to_string(), @@ -419,9 +432,10 @@ impl TryFrom for SelectionRow { .with_context(|| "Missing required property 'fields'".to_string())?; let mut fields: Vec = Vec::new(); - match struct_values.get(*fields_index) { - Some(ScalarValue::List(Some(elements), _)) => { - for el in elements.iter() { + let struct_field = ScalarValue::try_from_array(sa.column(*fields_index), 0)?; + match struct_field { + ScalarValue::List(array) => { + for el in array.value(0).to_scalar_vec()?.iter() { fields.push(FieldSpec::try_from(el.clone())?) } } @@ -508,8 +522,8 @@ pub fn vl_selection_test_fn( let op = parse_args(args)?; // Extract vector of rows for selection dataset - let rows = if let ScalarValue::List(Some(elements), _) = table.to_scalar_value()? { - elements + let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { + array.value(0).to_scalar_vec()? } else { unreachable!() }; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs index 93fd3c521..9835c1ba4 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; @@ -41,23 +41,25 @@ pub fn time_format_fn( // Special case for ISO-8601 format with milliseconds. The UTC_TIMESTAMP_TO_STR_UDF // is compatible with more SQL dialects, so we want to use it if possible let udf_args = vec![timestamptz_expr, lit(&format_tz_str)]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone())), args: udf_args, })) } else { // General case if format_tz_str.to_ascii_lowercase() != "utc" { - timestamptz_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*FROM_UTC_TIMESTAMP_UDF).clone()), + timestamptz_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*FROM_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![timestamptz_expr, lit(format_tz_str)], }) } let udf_args = vec![timestamptz_expr, lit(format_str)]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*FORMAT_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*FORMAT_TIMESTAMP_UDF).clone())), args: udf_args, })) } @@ -76,15 +78,15 @@ pub fn utc_format_fn( // Special case for ISO-8601 format with milliseconds. The UTC_TIMESTAMP_TO_STR_UDF // is compatible with more SQL dialects, so we want to use it if possible let udf_args = vec![timestamptz_expr, lit("UTC")]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone())), args: udf_args, })) } else { // General case let udf_args = vec![timestamptz_expr, lit(format_str)]; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*FORMAT_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*FORMAT_TIMESTAMP_UDF).clone())), args: udf_args, })) } @@ -101,14 +103,16 @@ fn to_timestamptz_expr(arg: &Expr, schema: &DFSchema, default_input_tz: &str) -> data_type: DataType::Timestamp(TimeUnit::Millisecond, None), }), DataType::Timestamp(_, _) => arg.clone(), - DataType::Utf8 => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg.clone(), lit(default_input_tz)], }), DataType::Null => arg.clone(), dtype if is_numeric_datatype(&dtype) || matches!(dtype, DataType::Boolean) => { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs index c5de56276..1a6cade0f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::call::TzTransformFn; use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::datafusion_common::DFSchema; @@ -19,8 +19,8 @@ pub fn make_local_datepart_transform(part: &str, tx: Option Expr>) - let arg = extract_timestamp_arg(&part, args, schema, &tz_config.default_input_tz.to_string())?; let udf_args = vec![lit(part.clone()), arg, lit(tz_config.local_tz.to_string())]; - let mut expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let mut expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: udf_args, }); @@ -42,8 +42,8 @@ pub fn make_utc_datepart_transform(part: &str, tx: Option Expr>) -> let arg = extract_timestamp_arg(&part, args, schema, &tz_config.default_input_tz.to_string())?; let udf_args = vec![lit(part.clone()), arg, lit("UTC")]; - let mut expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let mut expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: udf_args, }); @@ -62,7 +62,7 @@ fn extract_timestamp_arg( schema: &DFSchema, default_input_tz: &str, ) -> Result { - if let Some(arg) = args.get(0) { + if let Some(arg) = args.first() { Ok(match arg.get_type(schema)? { DataType::Date32 => Expr::Cast(expr::Cast { expr: Box::new(arg.clone()), @@ -73,12 +73,16 @@ fn extract_timestamp_arg( data_type: DataType::Timestamp(TimeUnit::Millisecond, None), }), DataType::Timestamp(_, _) => arg.clone(), - DataType::Utf8 => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![arg.clone(), lit(default_input_tz)], }), - dtype if is_numeric_datatype(&dtype) => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }), dtype => { diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs index 364f9e3fc..b79440491 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::ops::Deref; use std::str::FromStr; use std::sync::Arc; @@ -43,13 +43,15 @@ pub fn to_date_transform( tz_config.default_input_tz }; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![arg, lit(default_input_tz.to_string())], })) } else if is_numeric_datatype(&dtype) { - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(arg, &DataType::Int64, schema)?], })) } else { @@ -71,8 +73,10 @@ pub fn datetime_transform_fn( if is_string_datatype(&dtype) { let default_input_tz_str = tz_config.default_input_tz.to_string(); - arg = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + arg = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![arg, lit(default_input_tz_str)], }) } @@ -81,8 +85,8 @@ pub fn datetime_transform_fn( } else { let udf_args = extract_datetime_component_args(args, &tz_config.default_input_tz.to_string(), schema)?; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: udf_args, })) } @@ -95,8 +99,8 @@ pub fn make_datetime_components_fn( ) -> Result { let udf_args = extract_datetime_component_args(args, &tz_config.default_input_tz.to_string(), schema)?; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.deref().clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.deref().clone())), args: udf_args, })) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs index 42f4442e2..730d8b73f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -23,18 +23,24 @@ pub fn time_fn(tz_config: &RuntimeTzConfig, args: &[Expr], schema: &DFSchema) -> // Dispatch handling on data type let expr = match arg.get_type(schema)? { DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64 => { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), args: vec![arg.clone()], }) } DataType::Utf8 => { let mut udf_args = vec![lit(tz_config.default_input_tz.to_string())]; udf_args.extend(Vec::from(args)); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), - args: vec![Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), + )), + args: vec![Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*STR_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![arg.clone(), lit(tz_config.default_input_tz.to_string())], })], }) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs index 15535feb0..4d8d3fc8c 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs @@ -1,6 +1,6 @@ use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::{expr, lit, Expr}; +use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::error::VegaFusionError; @@ -79,8 +79,8 @@ pub fn time_offset_fn( let mut udf_args = vec![lit(tz_config.local_tz.to_string())]; udf_args.extend(Vec::from(args)); - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*DATE_ADD_TZ_UDF).clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_ADD_TZ_UDF).clone())), args: vec![ lit(unit), step, diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs index 3ded9fa88..269cde3bf 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs @@ -1,10 +1,9 @@ -use datafusion_expr::{expr, lit, Expr, ExprSchemable}; -use std::ops::Deref; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition, ScalarUDF}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_common::error::{Result, ResultWithContext, VegaFusionError}; -use vegafusion_datafusion_udfs::udfs::math::isfinite::ISFINITE_UDF; +use vegafusion_datafusion_udfs::udfs::math::isfinite::IsFiniteUDF; /// `isFinite(value)` /// @@ -20,9 +19,10 @@ pub fn is_finite_fn(args: &[Expr], schema: &DFSchema) -> Result { Ok(match dtype { DataType::Float16 | DataType::Float32 | DataType::Float64 => { - let is_finite_udf = ISFINITE_UDF.deref().clone(); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(is_finite_udf), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ScalarUDF::from( + IsFiniteUDF::new(), + ))), args: vec![arg], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs index a37c89bda..212f5503d 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/mod.rs @@ -1,3 +1,4 @@ +pub mod array; pub mod control_flow; pub mod data; pub mod date_time; diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index 190128977..4c6b70461 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -6,7 +6,8 @@ use crate::expression::compiler::builtin_functions::date_time::datetime::{ use crate::expression::compiler::builtin_functions::type_checking::isvalid::is_valid_fn; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; -use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarUDF}; +use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, ScalarUDF}; +use datafusion_functions::expr_fn::isnan; use std::collections::HashMap; use std::ops::Deref; use std::str::FromStr; @@ -19,11 +20,10 @@ use vegafusion_common::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_core::proto::gen::expression::{ expression, literal, CallExpression, Expression, Literal, }; -use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; -use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; -use vegafusion_datafusion_udfs::udfs::array::span::SPAN_UDF; -use vegafusion_datafusion_udfs::udfs::math::isnan::ISNAN_UDF; +use vegafusion_datafusion_udfs::udfs::array::indexof::IndexOfUDF; +use vegafusion_datafusion_udfs::udfs::array::span::SpanUDF; +use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::builtin_functions::data::data_fn::data_fn; use crate::expression::compiler::builtin_functions::data::vl_selection_resolve::vl_selection_resolve_fn; use crate::expression::compiler::builtin_functions::data::vl_selection_test::vl_selection_test_fn; @@ -49,6 +49,7 @@ use crate::task_graph::timezone::RuntimeTzConfig; pub type MacroFn = Arc Result + Send + Sync>; pub type TransformFn = Arc Result + Send + Sync>; +pub type ScalarTransformFn = Arc Expr + Send + Sync>; pub type TzTransformFn = Arc Result + Send + Sync>; pub type DataFn = Arc< @@ -65,6 +66,9 @@ pub enum VegaFusionCallable { /// A function that operates on the compiled arguments and produces a new expression. Transform(TransformFn), + /// An infallible function that operates on a single compiled argument + UnaryTransform(ScalarTransformFn), + /// A function that uses the local timezone to operate on the compiled arguments and /// produces a new expression. LocalTransform(TzTransformFn), @@ -133,20 +137,20 @@ pub fn compile_call( } VegaFusionCallable::ScalarUDF { udf, cast } => { let args = compile_scalar_arguments(node, config, schema, cast)?; - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(udf.clone()), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(udf.clone())), args, })) } VegaFusionCallable::BuiltinScalarFunction { function, cast } => { let args = compile_scalar_arguments(node, config, schema, cast)?; Ok(Expr::ScalarFunction(expr::ScalarFunction { - fun: *function, + func_def: ScalarFunctionDefinition::BuiltIn(*function), args, })) } VegaFusionCallable::Data(callee) => { - if let Some(v) = node.arguments.get(0) { + if let Some(v) = node.arguments.first() { match v.expr() { expression::Expr::Literal(Literal { value: Some(literal::Value::String(name)), @@ -184,6 +188,18 @@ pub fn compile_call( let args = compile_scalar_arguments(node, config, schema, &None)?; callable(&args, schema) } + VegaFusionCallable::UnaryTransform(callable) => { + let mut args = compile_scalar_arguments(node, config, schema, &None)?; + if args.len() != 1 { + Err(VegaFusionError::internal(format!( + "The {} function requires 1 argument. Received {}", + &node.callee, + args.len() + ))) + } else { + Ok(callable(args.pop().unwrap())) + } + } VegaFusionCallable::LocalTransform(callable) => { let args = compile_scalar_arguments(node, config, schema, &None)?; let tz_config = config @@ -236,10 +252,7 @@ pub fn default_callables() -> HashMap { callables.insert( "isNaN".to_string(), - VegaFusionCallable::ScalarUDF { - udf: ISNAN_UDF.deref().clone(), - cast: None, - }, + VegaFusionCallable::UnaryTransform(Arc::new(isnan)), ); callables.insert( @@ -259,16 +272,13 @@ pub fn default_callables() -> HashMap { callables.insert( "length".to_string(), - VegaFusionCallable::ScalarUDF { - udf: LENGTH_UDF.deref().clone(), - cast: None, - }, + VegaFusionCallable::Transform(Arc::new(length_transform)), ); callables.insert( "span".to_string(), VegaFusionCallable::ScalarUDF { - udf: SPAN_UDF.deref().clone(), + udf: ScalarUDF::from(SpanUDF::new()), cast: None, }, ); @@ -276,7 +286,7 @@ pub fn default_callables() -> HashMap { callables.insert( "indexof".to_string(), VegaFusionCallable::ScalarUDF { - udf: INDEXOF_UDF.deref().clone(), + udf: ScalarUDF::from(IndexOfUDF::new()), cast: None, }, ); diff --git a/vegafusion-runtime/src/expression/compiler/conditional.rs b/vegafusion-runtime/src/expression/compiler/conditional.rs index 7828856f7..42050ebf2 100644 --- a/vegafusion-runtime/src/expression/compiler/conditional.rs +++ b/vegafusion-runtime/src/expression/compiler/conditional.rs @@ -41,8 +41,8 @@ pub fn compile_conditional( if let Expr::Case(case) = alternate_expr { // Flatten nested case statements - let mut when_then_expr = case.when_then_expr; - when_then_expr.extend(vec![(Box::new(test), Box::new(consequent_expr))]); + let mut when_then_expr = vec![(Box::new(test), Box::new(consequent_expr))]; + when_then_expr.extend(case.when_then_expr); Ok(Expr::Case(Case { expr: None, when_then_expr, diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index b097de44a..0b153a6a8 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -1,9 +1,9 @@ +use crate::expression::compiler::builtin_functions::array::length::length_transform; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; -use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr}; +use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use std::convert::TryFrom; -use std::ops::Deref; use std::sync::Arc; use vegafusion_common::arrow::array::Int64Array; use vegafusion_common::arrow::compute::cast; @@ -13,7 +13,6 @@ use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; use vegafusion_common::datatypes::{data_type, is_numeric_datatype}; use vegafusion_core::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_core::proto::gen::expression::{Identifier, MemberExpression}; -use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; use vegafusion_datafusion_udfs::udfs::member::{make_get_element_udf, make_get_object_member_udf}; pub fn compile_member( @@ -33,7 +32,7 @@ pub fn compile_member( )?; let prop_str = evaluated_property.to_string(); if is_numeric_datatype(&evaluated_property.data_type()) { - let int_array = cast(&evaluated_property.to_array(), &DataType::Int64).unwrap(); + let int_array = cast(&evaluated_property.to_array()?, &DataType::Int64).unwrap(); let int_array = int_array.as_any().downcast_ref::().unwrap(); index = Some(int_array.value(0) as usize); } else { @@ -73,8 +72,11 @@ pub fn compile_member( let expr = match dtype { DataType::Struct(ref fields) => { if fields.iter().any(|f| f.name() == &property_string) { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_get_object_member_udf(&dtype, &property_string)?), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_object_member_udf( + &dtype, + &property_string, + )?)), args: vec![compiled_object], }) } else { @@ -84,17 +86,12 @@ pub fn compile_member( } _ => { if property_string == "length" { - // Special case to treat foo.length as length(foo) when foo is not an object - // make_length() - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(LENGTH_UDF.deref().clone()), - args: vec![compiled_object], - }) + length_transform(&[compiled_object], schema)? } else if matches!(dtype, DataType::Utf8 | DataType::LargeUtf8) { if let Some(index) = index { // SQL substr function is 1-indexed so add one Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Substr, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Substr), args: vec![compiled_object, lit((index + 1) as i64), lit(1i64)], }) } else { @@ -104,8 +101,10 @@ pub fn compile_member( } } else if matches!(dtype, DataType::List(_) | DataType::FixedSizeList(_, _)) { if let Some(index) = index { - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_get_element_udf(index as i32)), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_element_udf( + index as i32, + ))), args: vec![compiled_object], }) } else { diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 951763421..b77420a06 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -63,17 +63,20 @@ mod test_compile { use vegafusion_core::expression::parser::parse; use crate::task_graph::timezone::RuntimeTzConfig; + use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Case, TryCast}; - use datafusion_expr::{concat, expr, lit, BuiltinScalarFunction, Expr, Operator}; + use datafusion_expr::{ + concat, expr, lit, BuiltinScalarFunction, Expr, Operator, ScalarFunctionDefinition, + }; use std::collections::HashMap; use std::convert::TryFrom; - use std::ops::Deref; + use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, Field, Schema}; use vegafusion_common::column::flat_col; + use vegafusion_core::arrow::array::{new_empty_array, Float64Array}; use vegafusion_core::arrow::datatypes::Fields; - use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; use vegafusion_datafusion_udfs::udfs::object::make_object_constructor_udf; #[test] @@ -172,7 +175,7 @@ mod test_compile { // unary not should cast numeric value to boolean let expected_expr = !Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(32.0)), @@ -202,7 +205,7 @@ mod test_compile { expr: None, when_then_expr: vec![( Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(32.0)), @@ -257,7 +260,7 @@ mod test_compile { expr: None, when_then_expr: vec![( Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(5.0)), @@ -389,8 +392,8 @@ mod test_compile { let expr = parse("[1, 2, 3]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(1.0), lit(2.0), lit(3.0)], }); println!("expr: {result_expr:?}"); @@ -399,14 +402,9 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![ - ScalarValue::from(1.0), - ScalarValue::from(2.0), - ScalarValue::from(3.0), - ]), - Arc::new(Field::new("item", DataType::Float64, true)), - ); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![1.0, 2.0, 3.0]), + )))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -417,20 +415,17 @@ mod test_compile { let expr = parse("[]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![], }); println!("expr: {result_expr:?}"); assert_eq!(result_expr, expected_expr); - // Check evaluated value. Empty array is given Float64 data type let result_value = result_expr.eval_to_scalar().unwrap(); - - let expected_value = ScalarValue::List( - Some(vec![]), - Arc::new(Field::new("item", DataType::Float64, true)), - ); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array( + &DataType::Null, + )))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -441,19 +436,19 @@ mod test_compile { let expr = parse("[[1, 2], [3, 4], [5, 6]]").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![ - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(1.0), lit(2.0)], }), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(3.0), lit(4.0)], }), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ARRAY_CONSTRUCTOR_UDF.deref().clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::MakeArray), args: vec![lit(5.0), lit(6.0)], }), ], @@ -463,27 +458,20 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![ - ScalarValue::List( - Some(vec![ScalarValue::from(1.0), ScalarValue::from(2.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ScalarValue::List( - Some(vec![ScalarValue::from(3.0), ScalarValue::from(4.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ScalarValue::List( - Some(vec![ScalarValue::from(5.0), ScalarValue::from(6.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ]), - Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), - true, - )), - ); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![ + ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![1.0, 2.0]), + )))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![3.0, 4.0]), + )))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new( + Float64Array::from(vec![5.0, 6.0]), + )))), + ]) + .unwrap(), + ))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -494,8 +482,8 @@ mod test_compile { let expr = parse("{a: 1, 'two': {three: 3}}").unwrap(); let result_expr = compile(&expr, &Default::default(), None).unwrap(); - let expected_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_object_constructor_udf( + let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_object_constructor_udf( &["a".to_string(), "two".to_string()], &[ DataType::Float64, @@ -505,14 +493,14 @@ mod test_compile { true, )])), ], - )), + ))), args: vec![ lit(1.0), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(make_object_constructor_udf( + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(make_object_constructor_udf( &["three".to_string()], &[DataType::Float64], - )), + ))), args: vec![lit(3.0)], }), ], @@ -615,7 +603,7 @@ mod test_compile { expr: None, when_then_expr: vec![( Box::new(Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![ Expr::TryCast(TryCast { expr: Box::new(lit(32.0)), diff --git a/vegafusion-runtime/src/expression/compiler/object.rs b/vegafusion-runtime/src/expression/compiler/object.rs index 3a1689061..364718e7d 100644 --- a/vegafusion-runtime/src/expression/compiler/object.rs +++ b/vegafusion-runtime/src/expression/compiler/object.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{expr, Expr, ExprSchemable}; +use datafusion_expr::{expr, Expr, ExprSchemable, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -26,8 +26,8 @@ pub fn compile_object( let udf = make_object_constructor_udf(keys.as_slice(), value_types.as_slice()); - Ok(Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(udf), + Ok(Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(udf)), args: values, })) } diff --git a/vegafusion-runtime/src/expression/compiler/utils.rs b/vegafusion-runtime/src/expression/compiler/utils.rs index e5905050b..7a5a657c3 100644 --- a/vegafusion-runtime/src/expression/compiler/utils.rs +++ b/vegafusion-runtime/src/expression/compiler/utils.rs @@ -45,7 +45,7 @@ impl ExprHelpers for Expr { Ok(scalar) } else { Err(VegaFusionError::internal(format!( - "Failed to evaluate expression to scalar value: {self}" + "Failed to evaluate expression to scalar value: {self}\nsimplified to: {simplified_expr}\n" ))) } } diff --git a/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs b/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs index f83ac8ae0..a32183aa0 100644 --- a/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs +++ b/vegafusion-runtime/src/pre_transform/destringify_selection_datetimes.rs @@ -27,9 +27,9 @@ impl DestringifySelectionDatetimesVisitor { impl MutChartVisitor for DestringifySelectionDatetimesVisitor { fn visit_data(&mut self, data: &mut DataSpec, _scope: &[u32]) -> Result<()> { if let Some(Value::Array(values)) = &data.values { - if let Some(Value::Object(value0)) = values.get(0) { + if let Some(Value::Object(value0)) = values.first() { let columns: HashSet<_> = value0.keys().cloned().collect(); - let store_columns: HashSet<_> = vec!["unit", "fields", "values"] + let store_columns: HashSet<_> = ["unit", "fields", "values"] .iter() .map(|f| f.to_string()) .collect(); @@ -37,7 +37,7 @@ impl MutChartVisitor for DestringifySelectionDatetimesVisitor { // We have a selection store dataset with no transforms // Extract the values array if let Value::Array(values) = &value0["values"] { - if let Some(Value::Array(values)) = values.get(0) { + if let Some(Value::Array(values)) = values.first() { // Nested array, as in the case of an interval selection let is_date_str: Vec<_> = values .iter() diff --git a/vegafusion-runtime/src/task_graph/runtime.rs b/vegafusion-runtime/src/task_graph/runtime.rs index ced685d57..ce5d01e54 100644 --- a/vegafusion-runtime/src/task_graph/runtime.rs +++ b/vegafusion-runtime/src/task_graph/runtime.rs @@ -1023,8 +1023,8 @@ impl ChartState { // Filter to update nodes in the comm plan let indices: Vec<_> = indices .iter() + .filter(|&node| server_to_client.contains(node)) .cloned() - .filter(|node| server_to_client.contains(node)) .collect(); let response_task_values = runtime diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index ce4ca1c56..11b7e776a 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -5,6 +5,7 @@ use datafusion_expr::{avg, count, count_distinct, lit, max, min, sum, Expr}; use std::collections::HashMap; use async_trait::async_trait; +use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{aggregate_function, expr}; use std::sync::Arc; use vegafusion_common::column::{flat_col, unescaped_col}; @@ -162,35 +163,45 @@ pub fn make_agg_expr_for_col_expr( AggregateOp::Max => max(column), AggregateOp::Sum => sum(numeric_column()?), AggregateOp::Median => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::Median, + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Median, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Variance => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::Variance, + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Variance, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Variancep => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::VariancePop, + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::VariancePop, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Stdev => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::Stddev, + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Stddev, + ), distinct: false, args: vec![numeric_column()?], filter: None, order_by: None, }), AggregateOp::Stdevp => Expr::AggregateFunction(expr::AggregateFunction { - fun: aggregate_function::AggregateFunction::StddevPop, + func_def: AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::StddevPop, + ), distinct: false, args: vec![numeric_column()?], filter: None, @@ -218,15 +229,17 @@ pub fn make_agg_expr_for_col_expr( }); count_distinct(column) + max(missing) } - AggregateOp::Q1 => Expr::AggregateUDF(expr::AggregateUDF { - fun: Arc::new((*Q1_UDF).clone()), + AggregateOp::Q1 => Expr::AggregateFunction(expr::AggregateFunction { + func_def: AggregateFunctionDefinition::UDF(Arc::new((*Q1_UDF).clone())), args: vec![numeric_column()?], + distinct: false, filter: None, order_by: None, }), - AggregateOp::Q3 => Expr::AggregateUDF(expr::AggregateUDF { - fun: Arc::new((*Q3_UDF).clone()), + AggregateOp::Q3 => Expr::AggregateFunction(expr::AggregateFunction { + func_def: AggregateFunctionDefinition::UDF(Arc::new((*Q3_UDF).clone())), args: vec![numeric_column()?], + distinct: false, filter: None, order_by: None, }), diff --git a/vegafusion-runtime/src/transform/bin.rs b/vegafusion-runtime/src/transform/bin.rs index 553309216..66b6d4fed 100644 --- a/vegafusion-runtime/src/transform/bin.rs +++ b/vegafusion-runtime/src/transform/bin.rs @@ -7,6 +7,7 @@ use async_trait::async_trait; use datafusion_expr::lit; use datafusion_common::scalar::ScalarValue; +use datafusion_common::utils::array_into_list_array; use datafusion_common::DFSchema; use datafusion_expr::{abs, floor, when, Expr}; use float_cmp::approx_eq; @@ -15,7 +16,6 @@ use std::sync::Arc; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datatypes::to_numeric; -use vegafusion_core::arrow::datatypes::{DataType, Field}; use vegafusion_core::error::{Result, VegaFusionError}; use vegafusion_core::proto::gen::transforms::Bin; use vegafusion_core::task_graph::task_value::TaskValue; @@ -52,7 +52,9 @@ impl TransformTrait for Bin { let bin_index = floor((numeric_field.clone().sub(lit(start)).div(lit(step))).add(lit(1.0e-14))) .alias(bin_index_name); - let sql_df = sql_df.select(vec![Expr::Wildcard, bin_index]).await?; + let sql_df = sql_df + .select(vec![Expr::Wildcard { qualifier: None }, bin_index]) + .await?; // Add column with bin start let bin_start = (flat_col(bin_index_name).mul(lit(step))).add(lit(start)); @@ -121,11 +123,9 @@ impl TransformTrait for Bin { fn compute_output_value(bin_tx: &Bin, start: f64, stop: f64, step: f64) -> Option { let mut fname = bin_tx.field.clone(); fname.insert_str(0, "bin_"); - - let fields = ScalarValue::List( - Some(vec![ScalarValue::from(bin_tx.field.as_str())]), - Arc::new(Field::new("item", DataType::Utf8, true)), - ); + let fields = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![ScalarValue::from(bin_tx.field.as_str())]).ok()?, + ))); if bin_tx.signal.is_some() { Some(TaskValue::Scalar(ScalarValue::from(vec![ diff --git a/vegafusion-runtime/src/transform/collect.rs b/vegafusion-runtime/src/transform/collect.rs index 2c213db80..8e067a09f 100644 --- a/vegafusion-runtime/src/transform/collect.rs +++ b/vegafusion-runtime/src/transform/collect.rs @@ -1,17 +1,14 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; -use datafusion_expr::{expr, Expr}; +use datafusion_expr::{expr, Expr, WindowFunctionDefinition}; use std::sync::Arc; use vegafusion_core::error::{Result, ResultWithContext}; use vegafusion_core::proto::gen::transforms::{Collect, SortOrder}; use async_trait::async_trait; -use datafusion_common::scalar::ScalarValue; -use datafusion_expr::{ - window_function, BuiltInWindowFunction, WindowFrame, WindowFrameBound, WindowFrameUnits, -}; +use datafusion_expr::{BuiltInWindowFunction, WindowFrame}; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_core::task_graph::task_value::TaskValue; @@ -47,17 +44,11 @@ impl TransformTrait for Collect { // criteria. This column becomes the new ORDER_COL, which will be sorted at the end of // the pipeline. let order_col = Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( - BuiltInWindowFunction::RowNumber, - ), + fun: WindowFunctionDefinition::BuiltInWindowFunction(BuiltInWindowFunction::RowNumber), args: vec![], partition_by: vec![], order_by: sort_exprs, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(ORDER_COL); diff --git a/vegafusion-runtime/src/transform/extent.rs b/vegafusion-runtime/src/transform/extent.rs index 338f46926..2458afdf6 100644 --- a/vegafusion-runtime/src/transform/extent.rs +++ b/vegafusion-runtime/src/transform/extent.rs @@ -2,10 +2,10 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; +use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::{max, min, Expr}; use std::sync::Arc; -use vegafusion_common::arrow::datatypes::Field; use vegafusion_common::column::unescaped_col; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datatypes::to_numeric; @@ -61,10 +61,8 @@ fn extract_extent_list(table: &VegaFusionTable) -> Result { let max_val_scalar = ScalarValue::try_from_array(max_val_array, 0).unwrap(); // Build two-element list of the extents - let element_datatype = min_val_scalar.data_type(); - let extent_list = TaskValue::Scalar(ScalarValue::List( - Some(vec![min_val_scalar, max_val_scalar]), - Arc::new(Field::new("item", element_datatype, true)), - )); + let extent_list = TaskValue::Scalar(ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![min_val_scalar, max_val_scalar])?, + )))); Ok(extent_list) } diff --git a/vegafusion-runtime/src/transform/fold.rs b/vegafusion-runtime/src/transform/fold.rs index 9e6a3da26..579a7a5df 100644 --- a/vegafusion-runtime/src/transform/fold.rs +++ b/vegafusion-runtime/src/transform/fold.rs @@ -21,7 +21,7 @@ impl TransformTrait for Fold { let key_col = unescape_field( &self .r#as - .get(0) + .first() .cloned() .unwrap_or_else(|| "key".to_string()), ); diff --git a/vegafusion-runtime/src/transform/identifier.rs b/vegafusion-runtime/src/transform/identifier.rs index 57640d67a..1398f1e7b 100644 --- a/vegafusion-runtime/src/transform/identifier.rs +++ b/vegafusion-runtime/src/transform/identifier.rs @@ -2,13 +2,9 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; -use datafusion_expr::{ - expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, WindowFrameUnits, - WindowFunction, -}; +use datafusion_expr::{expr, BuiltInWindowFunction, Expr, WindowFrame, WindowFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; -use vegafusion_common::data::scalar::ScalarValue; use vegafusion_common::data::ORDER_COL; use vegafusion_common::error::Result; use vegafusion_core::proto::gen::transforms::Identifier; @@ -24,7 +20,7 @@ impl TransformTrait for Identifier { ) -> Result<(Arc, Vec)> { // Add row number column with the desired name, sorted by the input order column let row_number_expr = Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunction::BuiltInWindowFunction(BuiltInWindowFunction::RowNumber), + fun: WindowFunctionDefinition::BuiltInWindowFunction(BuiltInWindowFunction::RowNumber), args: Vec::new(), partition_by: Vec::new(), order_by: vec![Expr::Sort(expr::Sort { @@ -32,16 +28,12 @@ impl TransformTrait for Identifier { asc: true, nulls_first: false, })], - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(&self.r#as); let result = dataframe - .select(vec![Expr::Wildcard, row_number_expr]) + .select(vec![Expr::Wildcard { qualifier: None }, row_number_expr]) .await?; Ok((result, Default::default())) diff --git a/vegafusion-runtime/src/transform/timeunit.rs b/vegafusion-runtime/src/transform/timeunit.rs index c00c515a1..ad99aa591 100644 --- a/vegafusion-runtime/src/transform/timeunit.rs +++ b/vegafusion-runtime/src/transform/timeunit.rs @@ -11,7 +11,7 @@ use vegafusion_core::proto::gen::transforms::{TimeUnit, TimeUnitTimeZone, TimeUn use vegafusion_core::task_graph::task_value::TaskValue; use datafusion_expr::expr::Cast; -use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable}; +use datafusion_expr::{expr, floor, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; use itertools::Itertools; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::datatypes::{cast_to, is_numeric_datatype}; @@ -53,8 +53,8 @@ fn timeunit_date_trunc( // Compute input timestamp expression based on timezone let tz_str = local_tz.clone().unwrap_or_else(|| "UTC".to_string()); - let start_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + let start_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit(part_str), field_col, lit(tz_str)], }); @@ -92,8 +92,8 @@ fn timeunit_date_part_tz( // Year if units_set.contains(&TimeUnitUnit::Year) { - make_timestamptz_args[0] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[0] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("year"), field_col.clone(), lit(&tz_str)], }); @@ -102,8 +102,8 @@ fn timeunit_date_part_tz( // Quarter if units_set.contains(&TimeUnitUnit::Quarter) { - let month = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let month = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("month"), field_col.clone(), lit(&tz_str)], }) .sub(lit(1.0)); @@ -118,8 +118,8 @@ fn timeunit_date_part_tz( // Month if units_set.contains(&TimeUnitUnit::Month) { - make_timestamptz_args[1] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[1] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("month"), field_col.clone(), lit(&tz_str)], }) .sub(lit(1.0)); @@ -129,8 +129,8 @@ fn timeunit_date_part_tz( // Date if units_set.contains(&TimeUnitUnit::Date) { - make_timestamptz_args[2] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[2] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("day"), field_col.clone(), lit(&tz_str)], }); @@ -139,8 +139,8 @@ fn timeunit_date_part_tz( // Hour if units_set.contains(&TimeUnitUnit::Hours) { - make_timestamptz_args[3] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[3] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), field_col.clone(), lit(&tz_str)], }); @@ -149,8 +149,8 @@ fn timeunit_date_part_tz( // Minute if units_set.contains(&TimeUnitUnit::Minutes) { - make_timestamptz_args[4] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[4] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("minute"), field_col.clone(), lit(&tz_str)], }); @@ -159,8 +159,8 @@ fn timeunit_date_part_tz( // Second if units_set.contains(&TimeUnitUnit::Seconds) { - make_timestamptz_args[5] = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + make_timestamptz_args[5] = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("second"), field_col, lit(&tz_str)], }); @@ -168,8 +168,8 @@ fn timeunit_date_part_tz( } // Construct expression to make timestamp from components - let start_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + let start_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: make_timestamptz_args, }); @@ -185,12 +185,14 @@ fn to_timestamp_col(field: &str, schema: &DFSchema, default_input_tz: &String) - &DataType::Timestamp(ArrowTimeUnit::Millisecond, None), schema, )?, - DataType::Utf8 => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), + DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), args: vec![field_col, lit(default_input_tz)], }), - dtype if is_numeric_datatype(&dtype) => Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), + dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), + )), args: vec![cast_to(field_col, &DataType::Int64, schema)?], }), dtype => { @@ -215,8 +217,8 @@ fn timeunit_weekday( // Use DATE_PART_TZ to extract the weekday // where Sunday is 0 and Saturday is 6 - let weekday0 = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + let weekday0 = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("dow"), field_col, lit(tz_str)], }); @@ -237,8 +239,8 @@ fn timeunit_weekday( ]; // Construct expression to make timestamp from components - let start_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), + let start_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), args: make_timestamptz_args, }); @@ -253,18 +255,18 @@ fn timeunit_custom_udf( default_input_tz: &String, local_tz: &Option, ) -> Result<(Expr, (i32, String))> { - let units_mask = vec![ - units_set.contains(&TimeUnitUnit::Year), // 0 - units_set.contains(&TimeUnitUnit::Quarter), // 1 - units_set.contains(&TimeUnitUnit::Month), // 2 - units_set.contains(&TimeUnitUnit::Date), // 3 - units_set.contains(&TimeUnitUnit::Week), // 4 - units_set.contains(&TimeUnitUnit::Day), // 5 - units_set.contains(&TimeUnitUnit::DayOfYear), // 6 - units_set.contains(&TimeUnitUnit::Hours), // 7 - units_set.contains(&TimeUnitUnit::Minutes), // 8 - units_set.contains(&TimeUnitUnit::Seconds), // 9 - units_set.contains(&TimeUnitUnit::Milliseconds), // 10 + let units_mask = [ + units_set.contains(&TimeUnitUnit::Year), // 0 + units_set.contains(&TimeUnitUnit::Quarter), // 1 + units_set.contains(&TimeUnitUnit::Month), // 2 + units_set.contains(&TimeUnitUnit::Date), // 3 + units_set.contains(&TimeUnitUnit::Week), // 4 + units_set.contains(&TimeUnitUnit::Day), // 5 + units_set.contains(&TimeUnitUnit::DayOfYear), // 6 + units_set.contains(&TimeUnitUnit::Hours), // 7 + units_set.contains(&TimeUnitUnit::Minutes), // 8 + units_set.contains(&TimeUnitUnit::Seconds), // 9 + units_set.contains(&TimeUnitUnit::Milliseconds), ]; let timeunit_start_udf = &TIMEUNIT_START_UDF; @@ -505,8 +507,8 @@ impl TransformTrait for TimeUnit { }; let tz_str = local_tz.unwrap_or_else(|| "UTC".to_string()); - let timeunit_end_expr = Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new((*DATE_ADD_TZ_UDF).clone()), + let timeunit_end_expr = Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_ADD_TZ_UDF).clone())), args: vec![ lit(&interval.1), lit(interval.0), diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index 5e36db142..05fc57a0d 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -3,7 +3,7 @@ use crate::transform::TransformTrait; use async_trait::async_trait; use datafusion_common::ScalarValue; -use datafusion_expr::{aggregate_function, expr, lit, Expr}; +use datafusion_expr::{aggregate_function, expr, lit, Expr, WindowFrame, WindowFunctionDefinition}; use std::sync::Arc; use vegafusion_core::error::Result; use vegafusion_core::proto::gen::transforms::{ @@ -11,9 +11,7 @@ use vegafusion_core::proto::gen::transforms::{ }; use vegafusion_core::task_graph::task_value::TaskValue; -use datafusion_expr::{ - window_frame, BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits, WindowFunction, -}; +use datafusion_expr::{BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits}; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_common::datatypes::to_numeric; @@ -86,15 +84,12 @@ impl TransformTrait for Window { let ignore_peers = self.ignore_peers.unwrap_or(false); - let window_frame = window_frame::WindowFrame { - units: if ignore_peers { - WindowFrameUnits::Rows - } else { - WindowFrameUnits::Groups - }, - start_bound, - end_bound, + let units = if ignore_peers { + WindowFrameUnits::Rows + } else { + WindowFrameUnits::Groups }; + let window_frame = WindowFrame::new_bounds(units, start_bound, end_bound); let schema_df = dataframe.schema_df()?; let window_exprs = self @@ -146,7 +141,10 @@ impl TransformTrait for Window { ))) } }; - (WindowFunction::AggregateFunction(agg_fn), vec![arg]) + ( + WindowFunctionDefinition::AggregateFunction(agg_fn), + vec![arg], + ) } window_transform_op::Op::WindowOp(op) => { let op = WindowOp::try_from(*op).unwrap(); @@ -173,7 +171,10 @@ impl TransformTrait for Window { ))) } }; - (WindowFunction::BuiltInWindowFunction(window_fn), args) + ( + WindowFunctionDefinition::BuiltInWindowFunction(window_fn), + args, + ) } }; diff --git a/vegafusion-runtime/tests/test_expression_evaluation.rs b/vegafusion-runtime/tests/test_expression_evaluation.rs index 8ba4ef8ec..7dc599206 100644 --- a/vegafusion-runtime/tests/test_expression_evaluation.rs +++ b/vegafusion-runtime/tests/test_expression_evaluation.rs @@ -289,7 +289,9 @@ mod test_math_functions { case("sqrt(4)"), case("tan(1)"), case("log(2.718281828)"), - case("pow(2, 3)") + case("pow(2, 3)"), + case("isFinite(2)"), + case("isFinite(NaN)") )] fn test(expr: &str) { check_scalar_evaluation(expr, &config_a()) diff --git a/vegafusion-runtime/tests/test_image_comparison.rs b/vegafusion-runtime/tests/test_image_comparison.rs index 261919ae0..62c8d359c 100644 --- a/vegafusion-runtime/tests/test_image_comparison.rs +++ b/vegafusion-runtime/tests/test_image_comparison.rs @@ -754,8 +754,8 @@ mod test_vegalite_specs { case("vegalite/point_no_axis_domain_grid", 0.001), // (random function is non-deterministic so use higher tolerance) - case("vegalite/point_offset_random", 0.3), - case("vegalite/point_ordinal_bin_offset_random", 0.3), + case("vegalite/point_offset_random", 0.8), + case("vegalite/point_ordinal_bin_offset_random", 0.8), case("vegalite/point_ordinal_color", 0.001), case("vegalite/point_overlap", 0.001), diff --git a/vegafusion-runtime/tests/test_transform_formula.rs b/vegafusion-runtime/tests/test_transform_formula.rs index fc59daff4..9aeb07380 100644 --- a/vegafusion-runtime/tests/test_transform_formula.rs +++ b/vegafusion-runtime/tests/test_transform_formula.rs @@ -238,3 +238,52 @@ fn test_formula_recursive_ternary_doesnt_overflow_stack() { &eq_config, ); } + +#[test] +fn test_formula_span() { + let dataset = vega_json_dataset("penguins"); + + let transform_specs: Vec = serde_json::from_value(json!([ + { + "type": "formula", + "expr": "span([1, 2])", + "as": "span1" + }, + { + "type": "formula", + "expr": "span([])", + "as": "span2" + }, + { + "type": "formula", + "expr": "span(null)", + "as": "span3" + }, + { + "type": "formula", + "expr": "span(23)", + "as": "span4" + }, + { + "type": "formula", + "expr": "span([17])", + "as": "span5" + }, + { + "type": "formula", + "expr": "span([1, 2, 5, 6, 7])", + "as": "span6" + }, + ])) + .unwrap(); + + let comp_config = Default::default(); + let eq_config = Default::default(); + + check_transform_evaluation( + &dataset, + transform_specs.as_slice(), + &comp_config, + &eq_config, + ); +} diff --git a/vegafusion-runtime/tests/test_vegajs_runtime.rs b/vegafusion-runtime/tests/test_vegajs_runtime.rs index 65fb14354..85b792845 100644 --- a/vegafusion-runtime/tests/test_vegajs_runtime.rs +++ b/vegafusion-runtime/tests/test_vegajs_runtime.rs @@ -4,14 +4,15 @@ extern crate lazy_static; mod util; use crate::util::vegajs_runtime::{vegajs_runtime, ExportImage, ExportImageFormat}; +use datafusion_common::utils::array_into_list_array; use datafusion_common::ScalarValue; use serde_json::json; use std::collections::HashMap; use std::fs; use std::sync::Arc; -use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::table::VegaFusionTable; +use vegafusion_core::arrow::array::Float64Array; use vegafusion_core::planning::watch::{ ExportUpdateBatch, ExportUpdateJSON, ExportUpdateNamespace, Watch, WatchNamespace, }; @@ -122,10 +123,9 @@ fn test_evaluate_filter_transform() { // Check extent signal assert_eq!( result_signals, - vec![ScalarValue::List( - Some(vec![ScalarValue::from(6.0), ScalarValue::from(10.0)]), - Arc::new(Field::new("item", DataType::Float64, true)) - )] + vec![ScalarValue::List(Arc::new(array_into_list_array( + Arc::new(Float64Array::from(vec![6.0, 10.0])) + )))] ); let expected_dataset = VegaFusionTable::from_json(&json!([ diff --git a/vegafusion-runtime/tests/util/equality.rs b/vegafusion-runtime/tests/util/equality.rs index 719d5ac6f..cafa777c0 100644 --- a/vegafusion-runtime/tests/util/equality.rs +++ b/vegafusion-runtime/tests/util/equality.rs @@ -179,30 +179,31 @@ pub fn assert_scalars_almost_equals( index: usize, ) { match (lhs, rhs) { - ( - ScalarValue::Struct(Some(lhs_vals), lhs_fields), - ScalarValue::Struct(Some(rhs_vals), rhs_fields), - ) => { - let lhs_map: HashMap<_, _> = lhs_fields + (ScalarValue::Struct(lhs_sa), ScalarValue::Struct(rhs_sa)) => { + let lhs_map: HashMap<_, _> = lhs_sa + .fields() .iter() - .zip(lhs_vals.iter()) - .filter_map(|(f, val)| { - if DROP_COLS.contains(&f.name().as_str()) { + .enumerate() + .filter_map(|(field_ind, field)| { + if DROP_COLS.contains(&field.name().as_str()) { None } else { - Some((f.name().clone(), val.clone())) + let val = ScalarValue::try_from_array(lhs_sa.column(field_ind), 0).unwrap(); + Some((field.name().clone(), val)) } }) .collect(); - let rhs_map: HashMap<_, _> = rhs_fields + let rhs_map: HashMap<_, _> = rhs_sa + .fields() .iter() - .zip(rhs_vals.iter()) - .filter_map(|(f, val)| { - if DROP_COLS.contains(&f.name().as_str()) { + .enumerate() + .filter_map(|(field_ind, field)| { + if DROP_COLS.contains(&field.name().as_str()) { None } else { - Some((f.name().clone(), val.clone())) + let val = ScalarValue::try_from_array(rhs_sa.column(field_ind), 0).unwrap(); + Some((field.name().clone(), val)) } }) .collect(); diff --git a/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs b/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs index 1af836671..e32152bb7 100644 --- a/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs +++ b/vegafusion-runtime/tests/util/vegajs_runtime/mod.rs @@ -1,5 +1,4 @@ use datafusion_common::ScalarValue; -use dssim::{Dssim, DssimImage}; use regex::Regex; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::{json, Value}; @@ -7,6 +6,7 @@ use std::collections::HashMap; use self::super::estree_expression::ESTreeExpression; use itertools::Itertools; +use pixelmatch::{pixelmatch, Options}; use std::io::{Read, Write}; use std::ops::Deref; use std::process::{Child, Command, Stdio}; @@ -471,51 +471,37 @@ impl ExportImage { Ok(path) } - pub fn to_dssim(&self, attr: &Dssim) -> Result> { - if !matches!(self, ExportImage::Png(_)) { - return Err(VegaFusionError::internal("Only PNG image supported")); + pub fn as_png_bytes(&self) -> Result> { + match self { + ExportImage::Png(png_b64) => { + #[allow(deprecated)] + let png_bytes = base64::decode(png_b64) + .external("Failed to decdode base64 encoded png image")?; + Ok(png_bytes) + } + _ => Err(VegaFusionError::internal("Only PNG image supported")), } - let tmpfile = tempfile::NamedTempFile::new().unwrap(); - let tmppath = tmpfile.path().to_str().unwrap(); - self.save(tmppath, false)?; - - let img = dssim::load_image(attr, tmppath) - .external("Failed to create DSSIM image for comparison")?; - Ok(img) } pub fn compare(&self, other: &Self) -> Result<(f64, Option>)> { - let mut attr = Dssim::new(); - attr.set_save_ssim_maps(1); - let this_img = self.to_dssim(&attr)?; - let other_img = other.to_dssim(&attr)?; - let (diff, ssim_maps) = attr.compare(&this_img, other_img); - // println!("ssim_map: {:?}", ssim_map); - - if diff > 0.0 { - let map_meta = ssim_maps[0].clone(); - let avgssim = map_meta.ssim as f32; - - let out: Vec<_> = map_meta - .map - .pixels() - .map(|ssim| { - let max = 1_f32 - ssim; - let maxsq = max * max; - rgb::RGBA8 { - r: to_byte(maxsq * 16.0), - g: to_byte(max * 3.0), - b: to_byte(max / ((1_f32 - avgssim) * 4_f32)), - a: 255, - } - }) - .collect(); - let png_res = - lodepng::encode32(&out, map_meta.map.width(), map_meta.map.height()).unwrap(); - Ok((diff.into(), Some(png_res))) - } else { - Ok((diff.into(), None)) - } + let self_png = self.as_png_bytes()?; + let other_png = other.as_png_bytes()?; + let mut img_out = Vec::new(); + let res = pixelmatch( + self_png.as_slice(), + other_png.as_slice(), + Some(&mut img_out), + None, + None, + Some(Options { + threshold: 0.1, + ..Default::default() + }), + ) + .expect("pixelmatch failed"); + + let diff_factor = (res as f64) / 10000.0; + Ok((diff_factor, Some(img_out))) } } diff --git a/vegafusion-server/Cargo.toml b/vegafusion-server/Cargo.toml index e42fe666e..3af027a72 100644 --- a/vegafusion-server/Cargo.toml +++ b/vegafusion-server/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" description = "VegaFusion Server" [features] -protobuf-src = [ "vegafusion-core/protobuf-src",] +protobuf-src = [ "vegafusion-core/protobuf-src", "dep:protobuf-src"] [dependencies] futures-util = "0.3.21" @@ -25,6 +25,7 @@ predicates = "3.0.2" [build-dependencies] tonic-build = "0.10.2" prost-build = "0.12.1" +protobuf-src = { version="1.1.0", optional = true } [dependencies.prost] workspace = true @@ -61,6 +62,3 @@ features = [ "tls",] [dependencies.clap] version = "4.2.1" features = [ "derive",] - -[target."cfg(target_family = \"unix\")".build-dependencies] -protobuf-src = "1.1.0" diff --git a/vegafusion-server/build.rs b/vegafusion-server/build.rs index cb0b48499..b40107494 100644 --- a/vegafusion-server/build.rs +++ b/vegafusion-server/build.rs @@ -1,5 +1,5 @@ fn main() -> Result<(), Box> { - #[cfg(target_family = "unix")] + #[cfg(feature = "protobuf-src")] std::env::set_var("PROTOC", protobuf_src::protoc()); tonic_build::compile_protos("proto/helloworld.proto")?; diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 4a24eef46..4b54cdec6 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -12,8 +12,8 @@ use sqlparser::ast::{ use datafusion_expr::expr::{BinaryExpr, Case, Cast, Sort}; use datafusion_expr::{ - expr, lit, AggregateFunction, Between, BuiltInWindowFunction, BuiltinScalarFunction, Expr, - ExprSchemable, Operator, WindowFrameBound, WindowFrameUnits, WindowFunction, + expr, lit, Between, BuiltInWindowFunction, Expr, ExprSchemable, Operator, WindowFrameBound, + WindowFrameUnits, WindowFunctionDefinition, }; use crate::compile::function_arg::ToSqlFunctionArg; @@ -95,6 +95,26 @@ impl ToSqlExpr for Expr { "ArrowAt cannot be converted to SQL".to_string(), )) } + Operator::LikeMatch => { + return Err(VegaFusionError::internal( + "LikeMatch cannot be converted to SQL".to_string(), + )) + } + Operator::ILikeMatch => { + return Err(VegaFusionError::internal( + "ILikeMatch cannot be converted to SQL".to_string(), + )) + } + Operator::NotLikeMatch => { + return Err(VegaFusionError::internal( + "NotLikeMatch cannot be converted to SQL".to_string(), + )) + } + Operator::NotILikeMatch => { + return Err(VegaFusionError::internal( + "NotILikeMatch cannot be converted to SQL".to_string(), + )) + } }; Ok(SqlExpr::Nested(Box::new(SqlExpr::BinaryOp { left: Box::new(left.to_sql(dialect, schema)?), @@ -187,6 +207,7 @@ impl ToSqlExpr for Expr { SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } }; @@ -220,14 +241,17 @@ impl ToSqlExpr for Expr { TryCastMode::Supported => SqlExpr::TryCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::JustUseCast => SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::SafeCast => SqlExpr::SafeCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::SupportedOnStringsOtherwiseJustCast => { if let DataType::Utf8 | DataType::LargeUtf8 = from_dtype { @@ -235,12 +259,14 @@ impl ToSqlExpr for Expr { SqlExpr::TryCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } } else { // Fall back to regular CAST SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } } } @@ -267,144 +293,25 @@ impl ToSqlExpr for Expr { // Sort expressions need to be handled at a higher level Err(VegaFusionError::internal("Sort cannot be converted to SQL")) } - Expr::ScalarFunction(expr::ScalarFunction { fun, args }) => { - let fun_name = match fun { - BuiltinScalarFunction::Abs => "abs", - BuiltinScalarFunction::Acos => "acos", - BuiltinScalarFunction::Asin => "asin", - BuiltinScalarFunction::Atan => "atan", - BuiltinScalarFunction::Atan2 => "atan2", - BuiltinScalarFunction::Ceil => "ceil", - BuiltinScalarFunction::Coalesce => "coalesce", - BuiltinScalarFunction::Cos => "cos", - BuiltinScalarFunction::Cbrt => "cbrt", - BuiltinScalarFunction::Digest => "digest", - BuiltinScalarFunction::Exp => "exp", - BuiltinScalarFunction::Floor => "floor", - BuiltinScalarFunction::Ln => "ln", - BuiltinScalarFunction::Log => "log", - BuiltinScalarFunction::Log10 => "log10", - BuiltinScalarFunction::Log2 => "log2", - BuiltinScalarFunction::Power => "pow", - BuiltinScalarFunction::Round => "round", - BuiltinScalarFunction::Signum => "signum", - BuiltinScalarFunction::Sin => "sin", - BuiltinScalarFunction::Sqrt => "sqrt", - BuiltinScalarFunction::Tan => "tan", - BuiltinScalarFunction::Trunc => "trunc", - BuiltinScalarFunction::MakeArray => "make_array", - BuiltinScalarFunction::Ascii => "ascii", - BuiltinScalarFunction::BitLength => "bit_length", - BuiltinScalarFunction::Btrim => "btrim", - BuiltinScalarFunction::CharacterLength => "length", - BuiltinScalarFunction::Chr => "chr", - BuiltinScalarFunction::Concat => "concat", - BuiltinScalarFunction::ConcatWithSeparator => "concat_ws", - BuiltinScalarFunction::DatePart => "date_part", - BuiltinScalarFunction::DateTrunc => "date_trunc", - BuiltinScalarFunction::DateBin => "date_bin", - BuiltinScalarFunction::InitCap => "initcap", - BuiltinScalarFunction::Left => "left", - BuiltinScalarFunction::Lpad => "lpad", - BuiltinScalarFunction::Lower => "lower", - BuiltinScalarFunction::Ltrim => "ltrim", - BuiltinScalarFunction::MD5 => "md5", - BuiltinScalarFunction::NullIf => "nullif", - BuiltinScalarFunction::OctetLength => "octet_length", - BuiltinScalarFunction::Random => "random", - BuiltinScalarFunction::RegexpReplace => "regexp_replace", - BuiltinScalarFunction::Repeat => "repeat", - BuiltinScalarFunction::Replace => "replace", - BuiltinScalarFunction::Reverse => "reverse", - BuiltinScalarFunction::Right => "right", - BuiltinScalarFunction::Rpad => "rpad", - BuiltinScalarFunction::Rtrim => "rtrim", - BuiltinScalarFunction::SHA224 => "sha224", - BuiltinScalarFunction::SHA256 => "sha256", - BuiltinScalarFunction::SHA384 => "sha384", - BuiltinScalarFunction::SHA512 => "sha512", - BuiltinScalarFunction::SplitPart => "split_part", - BuiltinScalarFunction::StartsWith => "starts_with", - BuiltinScalarFunction::Strpos => "strpos", - BuiltinScalarFunction::Substr => "substr", - BuiltinScalarFunction::ToHex => "to_hex", - BuiltinScalarFunction::ToTimestamp => "to_timestamp", - BuiltinScalarFunction::ToTimestampMillis => "to_timestamp_millis", - BuiltinScalarFunction::ToTimestampMicros => "to_timestamp_micros", - BuiltinScalarFunction::ToTimestampSeconds => "to_timestamp_seconds", - BuiltinScalarFunction::FromUnixtime => "from_unixtime", - BuiltinScalarFunction::Now => "now", - BuiltinScalarFunction::Translate => "translate", - BuiltinScalarFunction::Trim => "trim", - BuiltinScalarFunction::Upper => "upper", - BuiltinScalarFunction::RegexpMatch => "regexp_match", - BuiltinScalarFunction::Struct => "struct", - BuiltinScalarFunction::ArrowTypeof => "arrow_typeof", - BuiltinScalarFunction::CurrentDate => "current_date", - BuiltinScalarFunction::CurrentTime => "current_time", - BuiltinScalarFunction::Uuid => "uuid", - BuiltinScalarFunction::Acosh => "acosh", - BuiltinScalarFunction::Asinh => "asinh", - BuiltinScalarFunction::Atanh => "atanh", - BuiltinScalarFunction::Cosh => "cosh", - BuiltinScalarFunction::Degrees => "degrees", - BuiltinScalarFunction::Pi => "pi", - BuiltinScalarFunction::Radians => "radians", - BuiltinScalarFunction::Sinh => "sinh", - BuiltinScalarFunction::Tanh => "tanh", - BuiltinScalarFunction::Factorial => "factorial", - BuiltinScalarFunction::Gcd => "gcd", - BuiltinScalarFunction::Lcm => "lcm", - BuiltinScalarFunction::ArrayAppend => "array_append", - BuiltinScalarFunction::ArrayConcat => "array_concat", - BuiltinScalarFunction::ArrayDims => "array_dims", - BuiltinScalarFunction::ArrayLength => "array_length", - BuiltinScalarFunction::ArrayNdims => "array_ndims", - BuiltinScalarFunction::ArrayPosition => "array_position", - BuiltinScalarFunction::ArrayPositions => "array_positions", - BuiltinScalarFunction::ArrayPrepend => "array_prepend", - BuiltinScalarFunction::ArrayRemove => "array_remove", - BuiltinScalarFunction::ArrayReplace => "array_replace", - BuiltinScalarFunction::ArrayToString => "array_to_string", - BuiltinScalarFunction::Cardinality => "array_cardinality", - BuiltinScalarFunction::ArrayHas => "array_has", - BuiltinScalarFunction::ArrayHasAll => "array_has_all", - BuiltinScalarFunction::ArrayHasAny => "array_has_any", - BuiltinScalarFunction::ArrayPopBack => "array_pop_back", - BuiltinScalarFunction::ArrayElement => "array_element", - BuiltinScalarFunction::ArrayEmpty => "array_empty", - BuiltinScalarFunction::ArrayRemoveN => "array_remove_n", - BuiltinScalarFunction::ArrayRemoveAll => "array_remove_all", - BuiltinScalarFunction::ArrayRepeat => "array_repeat", - BuiltinScalarFunction::ArrayReplaceN => "array_replace_n", - BuiltinScalarFunction::ArrayReplaceAll => "array_replace_all", - BuiltinScalarFunction::ArraySlice => "array_slice", - BuiltinScalarFunction::Decode => "decode", - BuiltinScalarFunction::Encode => "encode", - BuiltinScalarFunction::Cot => "cot", - BuiltinScalarFunction::Isnan => "isnan", - BuiltinScalarFunction::Iszero => "iszero", - BuiltinScalarFunction::Nanvl => "nanvl", - BuiltinScalarFunction::Flatten => "flatten", - BuiltinScalarFunction::StringToArray => "string_to_array", + Expr::ScalarFunction(fun) => { + let fun_name = match fun.name().to_ascii_lowercase().as_str() { + "power" => "pow".to_string(), + fun_name => fun_name.to_string(), }; - translate_scalar_function(fun_name, args, dialect, schema) - } - Expr::ScalarUDF(expr::ScalarUDF { fun, args }) => { - translate_scalar_function(&fun.name, args, dialect, schema) + translate_scalar_function(&fun_name, &fun.args, dialect, schema) } Expr::AggregateFunction(expr::AggregateFunction { - fun, + func_def, args, distinct, .. - }) => { - let fun_name = aggr_fn_to_name(fun); - translate_aggregate_function(fun_name, args.as_slice(), *distinct, dialect, schema) - } - Expr::AggregateUDF(expr::AggregateUDF { fun, args, .. }) => { - translate_aggregate_function(&fun.name, args.as_slice(), false, dialect, schema) - } + }) => translate_aggregate_function( + &func_def.name().to_ascii_lowercase(), + args.as_slice(), + *distinct, + dialect, + schema, + ), Expr::WindowFunction(expr::WindowFunction { fun, args, @@ -414,10 +321,10 @@ impl ToSqlExpr for Expr { }) => { // Extract function name let (fun_name, supports_frame) = match fun { - WindowFunction::AggregateFunction(agg) => { - (aggr_fn_to_name(agg).to_string().to_ascii_lowercase(), true) + WindowFunctionDefinition::AggregateFunction(agg) => { + (agg.name().to_ascii_lowercase(), true) } - WindowFunction::BuiltInWindowFunction(win_fn) => { + WindowFunctionDefinition::BuiltInWindowFunction(win_fn) => { let is_navigation_function = matches!( win_fn, BuiltInWindowFunction::FirstValue @@ -436,8 +343,12 @@ impl ToSqlExpr for Expr { (win_fn.to_string().to_ascii_lowercase(), supports_frame) } - WindowFunction::AggregateUDF(udf) => (udf.name.to_ascii_lowercase(), true), - WindowFunction::WindowUDF(udf) => (udf.name.to_ascii_lowercase(), true), + WindowFunctionDefinition::AggregateUDF(udf) => { + (udf.name().to_ascii_lowercase(), true) + } + WindowFunctionDefinition::WindowUDF(udf) => { + (udf.name().to_ascii_lowercase(), true) + } }; // Handle unordered row_number @@ -447,6 +358,8 @@ impl ToSqlExpr for Expr { return Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![Ident::new(alt_fun)]), args: vec![], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -533,6 +446,8 @@ impl ToSqlExpr for Expr { quote_style: None, }]), args, + filter: None, + null_treatment: None, over: Some(over), distinct: false, special: false, @@ -582,7 +497,7 @@ impl ToSqlExpr for Expr { negated: *negated, }) } - Expr::Wildcard => Err(VegaFusionError::internal( + Expr::Wildcard { .. } => Err(VegaFusionError::internal( "Wildcard cannot be converted to SQL", )), Expr::Exists { .. } => Err(VegaFusionError::internal( @@ -594,9 +509,6 @@ impl ToSqlExpr for Expr { Expr::ScalarSubquery(_) => Err(VegaFusionError::internal( "ScalarSubquery cannot be converted to SQL", )), - Expr::QualifiedWildcard { .. } => Err(VegaFusionError::internal( - "QualifiedWildcard cannot be converted to SQL", - )), Expr::GroupingSet(_) => Err(VegaFusionError::internal( "GroupingSet cannot be converted to SQL", )), @@ -610,6 +522,9 @@ impl ToSqlExpr for Expr { Expr::OuterReferenceColumn(_, _) => Err(VegaFusionError::internal( "OuterReferenceColumn cannot be converted to SQL", )), + Expr::Unnest(_) => Err(VegaFusionError::internal( + "Unnest cannot be converted to SQL", + )), } } } @@ -631,6 +546,8 @@ fn translate_scalar_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -664,6 +581,8 @@ fn translate_aggregate_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args, + filter: None, + null_treatment: None, over: None, distinct, special: false, @@ -694,46 +613,6 @@ fn translate_function_args( .collect::>>() } -fn aggr_fn_to_name(fun: &AggregateFunction) -> &str { - match fun { - AggregateFunction::Min => "min", - AggregateFunction::Max => "max", - AggregateFunction::Count => "count", - AggregateFunction::Avg => "avg", - AggregateFunction::Sum => "sum", - AggregateFunction::Median => "median", - AggregateFunction::ApproxDistinct => "approx_distinct", - AggregateFunction::ArrayAgg => "array_agg", - AggregateFunction::Variance => "var", - AggregateFunction::VariancePop => "var_pop", - AggregateFunction::Stddev => "stddev", - AggregateFunction::StddevPop => "stddev_pop", - AggregateFunction::Covariance => "covar", - AggregateFunction::CovariancePop => "covar_pop", - AggregateFunction::Correlation => "corr", - AggregateFunction::ApproxPercentileCont => "approx_percentile_cont", - AggregateFunction::ApproxPercentileContWithWeight => "approx_percentile_cont_with_weight", - AggregateFunction::ApproxMedian => "approx_median", - AggregateFunction::Grouping => "grouping", - AggregateFunction::BitAnd => "bit_and", - AggregateFunction::BitOr => "bit_or", - AggregateFunction::BitXor => "bit_xor", - AggregateFunction::BoolAnd => "bool_and", - AggregateFunction::BoolOr => "bool_or", - AggregateFunction::FirstValue => "first_value", - AggregateFunction::LastValue => "last_value", - AggregateFunction::RegrSlope => "regr_slope", - AggregateFunction::RegrIntercept => "regr_intercept", - AggregateFunction::RegrCount => "regr_count", - AggregateFunction::RegrR2 => "regr_r2", - AggregateFunction::RegrAvgx => "regr_avgx", - AggregateFunction::RegrAvgy => "regr_avgy", - AggregateFunction::RegrSXX => "regr_sxx", - AggregateFunction::RegrSYY => "regr_syy", - AggregateFunction::RegrSXY => "regr_sxy", - } -} - fn compile_window_frame_bound( bound: &WindowFrameBound, dialect: &Dialect, @@ -763,7 +642,9 @@ mod tests { use arrow::datatypes::DataType; use datafusion_common::DFSchema; use datafusion_expr::expr::Cast; - use datafusion_expr::{expr, lit, Between, BuiltinScalarFunction, Expr}; + use datafusion_expr::{ + expr, lit, Between, BuiltinScalarFunction, Expr, ScalarFunctionDefinition, + }; use vegafusion_common::column::flat_col; fn schema() -> DFSchema { @@ -782,7 +663,7 @@ mod tests { #[test] pub fn test2() { let df_expr = Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Sin, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Sin), args: vec![lit(1.2)], }) + flat_col("B"); @@ -796,7 +677,7 @@ mod tests { #[test] pub fn test3() { let df_expr = Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Upper, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Upper), args: vec![lit("foo")], }); diff --git a/vegafusion-sql/src/compile/function_arg.rs b/vegafusion-sql/src/compile/function_arg.rs index 99b3a2f77..e257f8666 100644 --- a/vegafusion-sql/src/compile/function_arg.rs +++ b/vegafusion-sql/src/compile/function_arg.rs @@ -20,13 +20,13 @@ impl ToSqlFunctionArg for Expr { schema: &DFSchema, ) -> Result { Ok(match self { - Expr::Wildcard => SqlFunctionArgExpr::Wildcard, - Expr::QualifiedWildcard { qualifier } => { - SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { - value: qualifier.clone(), - quote_style: None, - }])) - } + Expr::Wildcard { qualifier: None } => SqlFunctionArgExpr::Wildcard, + Expr::Wildcard { + qualifier: Some(qualifier), + } => SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { + value: qualifier.clone(), + quote_style: None, + }])), expr => SqlFunctionArgExpr::Expr(expr.to_sql(dialect, schema)?), }) } diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index eefe47d1d..94801421b 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -5,15 +5,17 @@ use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; use datafusion_expr::{ - expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, - Signature, Volatility, + expr, lit, ColumnarValue, Expr, ScalarFunctionDefinition, ScalarUDF, ScalarUDFImpl, Signature, + Volatility, }; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr, Ident, ObjectName as SqlObjectName, Value as SqlValue, }; +use std::any::Any; use std::ops::Add; use std::sync::Arc; +use vegafusion_common::data::scalar::ArrayRefHelpers; use vegafusion_common::error::{Result, VegaFusionError}; pub trait ToSqlScalar { @@ -47,6 +49,7 @@ impl ToSqlScalar for ScalarValue { false, ))), data_type: cast_dtype, + format: None, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -79,6 +82,7 @@ impl ToSqlScalar for ScalarValue { false, ))), data_type: cast_dtype, + format: None, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -142,26 +146,26 @@ impl ToSqlScalar for ScalarValue { ScalarValue::FixedSizeBinary(_, _) => Err(VegaFusionError::internal( "FixedSizeBinary cannot be converted to SQL", )), - ScalarValue::List(args, _) => { + ScalarValue::List(array) => { let function_ident = Ident { value: "make_list".to_string(), quote_style: None, }; - - let args = args - .clone() - .unwrap_or_default() - .iter() - .map(|expr| { - Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr( - expr.to_sql(dialect)?, - ))) + let args = array + .value(0) + .to_scalar_vec()? + .into_iter() + .map(|v| { + let sql_expr = v.to_sql(dialect)?; + Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr(sql_expr))) }) .collect::>>()?; Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![function_ident]), args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -209,7 +213,7 @@ impl ToSqlScalar for ScalarValue { ScalarValue::IntervalMonthDayNano(_) => Err(VegaFusionError::internal( "IntervalMonthDayNano cannot be converted to SQL", )), - ScalarValue::Struct(_, _) => Err(VegaFusionError::internal( + ScalarValue::Struct(_) => Err(VegaFusionError::internal( "Struct cannot be converted to SQL", )), ScalarValue::Dictionary(_, _) => Err(VegaFusionError::internal( @@ -233,9 +237,6 @@ impl ToSqlScalar for ScalarValue { ScalarValue::Time64Nanosecond(_) => Err(VegaFusionError::internal( "Time64Nanosecond cannot be converted to SQL", )), - ScalarValue::Fixedsizelist(_, _, _) => Err(VegaFusionError::internal( - "Fixedsizelist cannot be converted to SQL", - )), ScalarValue::DurationSecond(_) => Err(VegaFusionError::internal( "DurationSecond cannot be converted to SQL", )), @@ -248,38 +249,75 @@ impl ToSqlScalar for ScalarValue { ScalarValue::DurationNanosecond(_) => Err(VegaFusionError::internal( "DurationNanosecond cannot be converted to SQL", )), + ScalarValue::FixedSizeList(_) => Err(VegaFusionError::internal( + "FixedSizeList cannot be converted to SQL", + )), + ScalarValue::LargeList(_) => Err(VegaFusionError::internal( + "LargeList cannot be converted to SQL", + )), } } } fn ms_to_timestamp(v: i64, dialect: &Dialect) -> Result { // Hack to recursively transform the epoch_ms_to_utc_timestamp - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); - let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); - let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| { - panic!("Placeholder UDF implementation should not be called") - }); - - let udf = ScalarUDF::new( - "epoch_ms_to_utc_timestamp", - &signature, - &return_type, - &scalar_fn, - ); - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(udf), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ScalarUDF::from( + EpochMsToUtcTimestampUDF::new(), + ))), args: vec![lit(v)], }) .to_sql(dialect, &DFSchema::empty()) } +// Hack to recursively transform the epoch_ms_to_utc_timestamp +#[derive(Debug, Clone)] +pub struct EpochMsToUtcTimestampUDF { + signature: Signature, +} + +impl Default for EpochMsToUtcTimestampUDF { + fn default() -> Self { + Self::new() + } +} + +impl EpochMsToUtcTimestampUDF { + pub fn new() -> Self { + let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable); + Self { signature } + } +} + +impl ScalarUDFImpl for EpochMsToUtcTimestampUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "epoch_ms_to_utc_timestamp" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> datafusion_common::Result { + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> datafusion_common::Result { + panic!("Placeholder UDF implementation should not be called") + } +} + fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { let epoch = chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); match days { None => Ok(SqlExpr::Cast { expr: Box::new(ScalarValue::Utf8(None).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, + format: None, }), Some(days) => { let date = epoch.add(chrono::Duration::days(*days as i64)); @@ -287,6 +325,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { Ok(SqlExpr::Cast { expr: Box::new(ScalarValue::from(date_str.as_str()).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, + format: None, }) } } diff --git a/vegafusion-sql/src/compile/select.rs b/vegafusion-sql/src/compile/select.rs index b55ff858e..1517998fa 100644 --- a/vegafusion-sql/src/compile/select.rs +++ b/vegafusion-sql/src/compile/select.rs @@ -2,7 +2,7 @@ use crate::compile::expr::ToSqlExpr; use crate::dialect::Dialect; use datafusion_common::DFSchema; use datafusion_expr::{expr, Expr}; -use sqlparser::ast::{Ident, SelectItem as SqlSelectItem}; +use sqlparser::ast::{Ident, ObjectName, SelectItem as SqlSelectItem}; use vegafusion_common::error::Result; pub trait ToSqlSelectItem { @@ -12,14 +12,25 @@ pub trait ToSqlSelectItem { impl ToSqlSelectItem for Expr { fn to_sql_select(&self, dialect: &Dialect, schema: &DFSchema) -> Result { Ok(match self { - Expr::Alias(expr::Alias { expr, name: alias }) => SqlSelectItem::ExprWithAlias { + Expr::Alias(expr::Alias { + expr, name: alias, .. + }) => SqlSelectItem::ExprWithAlias { expr: expr.to_sql(dialect, schema)?, alias: Ident { value: alias.clone(), quote_style: Some(dialect.quote_style), }, }, - Expr::Wildcard => SqlSelectItem::Wildcard(Default::default()), + Expr::Wildcard { qualifier: None } => SqlSelectItem::Wildcard(Default::default()), + Expr::Wildcard { + qualifier: Some(qualifier), + } => SqlSelectItem::QualifiedWildcard( + ObjectName(vec![Ident { + value: qualifier.to_string(), + quote_style: Some(dialect.quote_style), + }]), + Default::default(), + ), expr => SqlSelectItem::UnnamedExpr(expr.to_sql(dialect, schema)?), }) } @@ -40,7 +51,7 @@ mod tests { #[test] pub fn test_select_wildcard() { - let expr = Expr::Wildcard; + let expr = Expr::Wildcard { qualifier: None }; let sql_expr = expr .to_sql_select(&Dialect::datafusion(), &schema()) .unwrap(); diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index f8ca9d35b..24de7dd3c 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -14,6 +14,7 @@ use datafusion::optimizer::analyzer::type_coercion::TypeCoercion; use datafusion::prelude::{ CsvReadOptions as DfCsvReadOptions, ParquetReadOptions, SessionConfig, SessionContext, }; +use datafusion_expr::ScalarUDF; use log::Level; use object_store::aws::AmazonS3Builder; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; @@ -33,9 +34,7 @@ use vegafusion_dataframe::connection::Connection; use vegafusion_dataframe::csv::CsvReadOptions; use vegafusion_dataframe::dataframe::DataFrame; use vegafusion_datafusion_udfs::udafs::{Q1_UDF, Q3_UDF}; -use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; -use vegafusion_datafusion_udfs::udfs::array::indexof::INDEXOF_UDF; -use vegafusion_datafusion_udfs::udfs::array::length::LENGTH_UDF; +use vegafusion_datafusion_udfs::udfs::array::indexof::IndexOfUDF; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; use vegafusion_datafusion_udfs::udfs::datetime::date_to_utc_timestamp::DATE_TO_UTC_TIMESTAMP_UDF; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -47,8 +46,7 @@ use vegafusion_datafusion_udfs::udfs::datetime::timeunit::TIMEUNIT_START_UDF; use vegafusion_datafusion_udfs::udfs::datetime::to_utc_timestamp::TO_UTC_TIMESTAMP_UDF; use vegafusion_datafusion_udfs::udfs::datetime::utc_timestamp_to_epoch::UTC_TIMESTAMP_TO_EPOCH_MS; use vegafusion_datafusion_udfs::udfs::datetime::utc_timestamp_to_str::UTC_TIMESTAMP_TO_STR_UDF; -use vegafusion_datafusion_udfs::udfs::math::isfinite::ISFINITE_UDF; -use vegafusion_datafusion_udfs::udfs::math::isnan::ISNAN_UDF; +use vegafusion_datafusion_udfs::udfs::math::isfinite::IsFiniteUDF; #[cfg(feature = "pyarrow")] use {crate::connection::datafusion_py_datasource::PyDatasource, pyo3::PyObject}; @@ -117,11 +115,11 @@ impl Connection for DataFusionConnection { async fn tables(&self) -> Result> { let catalog_names = self.ctx.catalog_names(); - let first_catalog_name = catalog_names.get(0).unwrap(); + let first_catalog_name = catalog_names.first().unwrap(); let catalog = self.ctx.catalog(first_catalog_name).unwrap(); let schema_provider_names = catalog.schema_names(); - let first_schema_provider_name = schema_provider_names.get(0).unwrap(); + let first_schema_provider_name = schema_provider_names.first().unwrap(); let schema_provider = catalog.schema(first_schema_provider_name).unwrap(); let mut tables: HashMap = HashMap::new(); @@ -143,7 +141,7 @@ impl Connection for DataFusionConnection { let batch_schema = if table.batches.is_empty() { None } else { - Some(table.batches.get(0).unwrap().schema()) + Some(table.batches.first().unwrap().schema()) }; // Create memtable @@ -200,14 +198,12 @@ impl Connection for DataFusionConnection { writeln!(file, "{body}").unwrap(); } - let path = tempdir.path().to_str().unwrap(); - // Build final csv schema that combines the requested and inferred schemas - let final_schema = build_csv_schema(&df_csv_opts, path, &self.ctx).await?; + let final_schema = build_csv_schema(&df_csv_opts, &filepath, &self.ctx).await?; df_csv_opts = df_csv_opts.schema(&final_schema); // Load through VegaFusionTable so that temp file can be deleted - let df = self.ctx.read_csv(path, df_csv_opts).await?; + let df = self.ctx.read_csv(&filepath, df_csv_opts).await?; let schema: SchemaRef = Arc::new(df.schema().into()) as SchemaRef; let batches = df.collect().await?; @@ -497,11 +493,8 @@ pub fn make_datafusion_context() -> SessionContext { let ctx = SessionContext::new_with_state(session_state); - // isNan - ctx.register_udf((*ISNAN_UDF).clone()); - // isFinite - ctx.register_udf((*ISFINITE_UDF).clone()); + ctx.register_udf(ScalarUDF::from(IsFiniteUDF::new())); // datetime ctx.register_udf((*DATE_PART_TZ_UDF).clone()); @@ -521,9 +514,7 @@ pub fn make_datafusion_context() -> SessionContext { ctx.register_udf((*FORMAT_TIMESTAMP_UDF).clone()); // list - ctx.register_udf((*ARRAY_CONSTRUCTOR_UDF).clone()); - ctx.register_udf((*LENGTH_UDF).clone()); - ctx.register_udf((*INDEXOF_UDF).clone()); + ctx.register_udf(ScalarUDF::from(IndexOfUDF::new())); // q1/q3 aggregate functions ctx.register_udaf((*Q1_UDF).clone()); diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index 821d008cf..23eb6d160 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -136,7 +136,7 @@ impl ExecutionPlan for PyDatasourceExec { .collect::>(); let args = PyTuple::new(py, vec![column_names.into_py(py)]); let pa_table = self.db.py_datasource.call_method1(py, "fetch", args)?; - let table = VegaFusionTable::from_pyarrow(py, &pa_table.as_ref(py))?; + let table = VegaFusionTable::from_pyarrow(py, pa_table.as_ref(py))?; Ok(table) }) .map_err(|err| DataFusionError::Execution(err.to_string()))?; @@ -148,7 +148,7 @@ impl ExecutionPlan for PyDatasourceExec { )?)) } - fn statistics(&self) -> Statistics { - Statistics::default() + fn statistics(&self) -> datafusion_common::Result { + Ok(Statistics::new_unknown(self.schema().as_ref())) } } diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index 5ee92268d..7d6eacdb8 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -7,14 +7,15 @@ use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{Column, DFSchema, OwnedTableReference, ScalarValue}; +use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{ - abs, expr, is_null, lit, max, min, when, window_function, AggregateFunction, - BuiltInWindowFunction, BuiltinScalarFunction, Expr, ExprSchemable, WindowFrame, - WindowFrameBound, WindowFrameUnits, WindowFunction, + abs, expr, is_null, lit, max, min, when, AggregateFunction, BuiltInWindowFunction, + BuiltinScalarFunction, Expr, ExprSchemable, ScalarFunctionDefinition, WindowFrame, + WindowFunctionDefinition, }; use sqlparser::ast::{ - Cte, Expr as SqlExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, TableAlias, - TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, + Cte, Expr as SqlExpr, GroupByExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, + TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, }; use sqlparser::parser::Parser; use std::any::Any; @@ -278,7 +279,7 @@ impl SqlDataFrame { selection: None, from: Default::default(), lateral_views: Default::default(), - group_by: Default::default(), + group_by: GroupByExpr::Expressions(Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), @@ -320,9 +321,11 @@ impl SqlDataFrame { body: Box::new(values_body), order_by: Default::default(), limit: None, + limit_by: vec![], offset: None, fetch: None, locks: Default::default(), + for_clause: None, }; let (projection, table_alias) = if let ValuesMode::ValuesWithSelectColumnAliases { @@ -385,7 +388,7 @@ impl SqlDataFrame { }], lateral_views: Default::default(), selection: None, - group_by: Default::default(), + group_by: GroupByExpr::Expressions(Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), @@ -398,9 +401,11 @@ impl SqlDataFrame { body: Box::new(select_body), order_by: Default::default(), limit: None, + limit_by: vec![], offset: None, fetch: None, locks: Default::default(), + for_clause: None, } } }; @@ -822,7 +827,7 @@ impl SqlDataFrame { // 2. field index let field_order_col = format!("{order_field}_field"); let order_col = Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], @@ -839,11 +844,7 @@ impl SqlDataFrame { nulls_first: true, }), ], - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(order_field); @@ -893,25 +894,20 @@ impl SqlDataFrame { let partition_by: Vec<_> = groupby.iter().map(|group| flat_col(group)).collect(); let numeric_field = Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Coalesce), args: vec![to_numeric(flat_col(field), &self.schema_df()?)?, lit(0.0)], }); if let StackMode::Zero = mode { // Build window expression - let fun = WindowFunction::AggregateFunction(AggregateFunction::Sum); // Build window function to compute stacked value let window_expr = Expr::WindowFunction(expr::WindowFunction { - fun, + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![numeric_field.clone()], partition_by, order_by: orderby, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(stop_field); @@ -923,7 +919,7 @@ impl SqlDataFrame { // then union the results. This is required to make sure stacks do not overlap. Negative // values stack in the negative direction and positive values stack in the positive // direction. - let schema_exprs = vec![Expr::Wildcard, window_expr]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, window_expr]; let new_schema = make_new_schema_from_exprs( self.schema.as_ref(), schema_exprs.as_slice(), @@ -967,7 +963,10 @@ impl SqlDataFrame { // Create __stack column with numeric field let stack_col_name = "__stack"; let dataframe = self - .select(vec![Expr::Wildcard, numeric_field.alias(stack_col_name)]) + .select(vec![ + Expr::Wildcard { qualifier: None }, + numeric_field.alias(stack_col_name), + ]) .await?; let dataframe = dataframe @@ -978,7 +977,7 @@ impl SqlDataFrame { // Create aggregate for total of stack value let total_agg = Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Sum, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Sum), args: vec![flat_col(stack_col_name)], distinct: false, filter: None, @@ -990,7 +989,7 @@ impl SqlDataFrame { .to_string(); // Add __total column with total or total per partition - let schema_exprs = vec![Expr::Wildcard, total_agg]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, total_agg]; let new_schema = make_new_schema_from_exprs( &dataframe.schema(), schema_exprs.as_slice(), @@ -1028,22 +1027,20 @@ impl SqlDataFrame { // Build window function to compute cumulative sum of stack column let cumulative_field = "_cumulative"; - let fun = WindowFunction::AggregateFunction(AggregateFunction::Sum); + let fun = WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum); let window_expr = Expr::WindowFunction(expr::WindowFunction { fun, args: vec![flat_col(stack_col_name)], partition_by, order_by: orderby, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(cumulative_field); // Perform selection to add new field value - let dataframe = dataframe.select(vec![Expr::Wildcard, window_expr]).await?; + let dataframe = dataframe + .select(vec![Expr::Wildcard { qualifier: None }, window_expr]) + .await?; // Build final_selection let mut final_selection: Vec<_> = input_fields @@ -1066,7 +1063,7 @@ impl SqlDataFrame { .to_string(); // Compute new schema - let schema_exprs = vec![Expr::Wildcard, max_total]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, max_total]; let new_schema = make_new_schema_from_exprs( &dataframe.schema(), schema_exprs.as_slice(), @@ -1147,7 +1144,9 @@ impl SqlDataFrame { if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::Coalesce, + ), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1192,7 +1191,9 @@ impl SqlDataFrame { .map(|col_name| { if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::Coalesce, + ), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1210,7 +1211,9 @@ impl SqlDataFrame { .map(|col_name| { let expr = if col_name == field { Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Coalesce, + func_def: ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::Coalesce, + ), args: vec![flat_col(field), lit(value.clone())], }) .alias(col_name) @@ -1333,17 +1336,13 @@ impl SqlDataFrame { }; let order_col = Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], partition_by: vec![], order_by, - window_frame: WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - end_bound: WindowFrameBound::CurrentRow, - }, + window_frame: WindowFrame::new(Some(true)), }) .alias(order_field); @@ -1395,7 +1394,7 @@ fn make_new_schema_from_exprs( ) -> Result { let mut fields: Vec = Vec::new(); for expr in exprs { - if let Expr::Wildcard = expr { + if let Expr::Wildcard { .. } = expr { // Add field for each input schema field fields.extend(schema.fields().iter().map(|f| f.as_ref().clone())); } else { @@ -1472,7 +1471,7 @@ fn query_chain_to_cte(queries: &[Query], prefix: &str) -> Query { fn parse_sql_query(query: &str, dialect: &Dialect) -> Result { let statements: Vec = Parser::parse_sql(dialect.parser_dialect().as_ref(), query)?; - if let Some(statement) = statements.get(0) { + if let Some(statement) = statements.first() { if let Statement::Query(box_query) = statement { let query: &Query = box_query.as_ref(); Ok(query.clone()) diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index c42ee0514..af7b15372 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -297,7 +297,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -311,7 +311,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -423,11 +423,11 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec!["min", "max", "count", "avg", "sum"] + aggregate_functions: ["min", "max", "count", "avg", "sum"] .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -516,7 +516,7 @@ impl Dialect { (DataType::Float16, float64dtype.clone()), (DataType::Float32, float64dtype.clone()), (DataType::Float64, float64dtype), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -568,11 +568,11 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec!["min", "max", "count", "avg", "sum", "median", "corr"] + aggregate_functions: ["min", "max", "count", "avg", "sum", "median", "corr"] .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -689,7 +689,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -704,7 +704,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -796,7 +796,7 @@ impl Dialect { (DataType::Float16, SqlDataType::Float(None)), (DataType::Float32, SqlDataType::Float(None)), (DataType::Float64, SqlDataType::Double), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -885,7 +885,8 @@ impl Dialect { "vega_timeunit", "format_timestamp", "make_list", - "len", + "array_length", + "character_length", "indexof", ] .iter() @@ -913,7 +914,7 @@ impl Dialect { .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -962,7 +963,7 @@ impl Dialect { (DataType::Float16, SqlDataType::Float(None)), (DataType::Float32, SqlDataType::Float(None)), (DataType::Float64, SqlDataType::Double), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -1012,7 +1013,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1027,7 +1028,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1166,11 +1167,11 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec!["min", "max", "count", "avg", "sum", "var_pop", "stddev_pop"] + aggregate_functions: ["min", "max", "count", "avg", "sum", "var_pop", "stddev_pop"] .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1271,7 +1272,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1285,7 +1286,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1424,7 +1425,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1438,7 +1439,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1573,7 +1574,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - aggregate_functions: vec![ + aggregate_functions: [ "min", "max", "count", @@ -1588,7 +1589,7 @@ impl Dialect { .iter() .map(|s| s.to_string()) .collect(), - window_functions: vec![ + window_functions: [ "row_number", "rank", "dense_rank", @@ -1750,6 +1751,8 @@ impl BinaryOperatorTransformer for ModulusOpToFunction { quote_style: None, }]), args: vec![arg0, arg1], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1793,6 +1796,8 @@ impl FunctionTransformer for RenameFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1825,6 +1830,8 @@ impl FunctionTransformer for ExpWithPowFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1854,6 +1861,7 @@ impl FunctionTransformer for CastArgsFunctionTransformer { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Cast { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: self.cast_dtype.clone(), + format: None, }))) }) .collect::>>()?; @@ -1864,6 +1872,8 @@ impl FunctionTransformer for CastArgsFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1902,6 +1912,8 @@ impl FunctionTransformer for LogBaseTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1929,6 +1941,7 @@ impl FunctionTransformer for LogBaseWithLnTransformer { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Cast { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: cast_dtype.clone(), + format: None, }))) }) .collect::>>()? @@ -1942,6 +1955,8 @@ impl FunctionTransformer for LogBaseWithLnTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1957,6 +1972,8 @@ impl FunctionTransformer for LogBaseWithLnTransformer { quote_style: None, }]), args: vec![base_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index 2fc82170e..584b0fd6f 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -78,6 +78,8 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { SqlValue::SingleQuotedString(time_zone.to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -101,6 +103,8 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { SqlValue::SingleQuotedString(time_zone.to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -137,6 +141,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -161,6 +167,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { }, ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -178,6 +186,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -260,6 +270,8 @@ impl FunctionTransformer for DateAddTzDatabricksTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -350,6 +362,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -372,6 +386,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -395,6 +411,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index bff7289aa..f4d98ace3 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -111,6 +111,8 @@ impl FunctionTransformer for DatePartTzWithDatePartAndAtTimezoneTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -172,6 +174,8 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -202,6 +206,8 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( to_timezone_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -241,6 +247,8 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -259,6 +267,8 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -297,6 +307,8 @@ impl FunctionTransformer for DatePartTzMySqlTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -342,6 +354,8 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -360,6 +374,8 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs index 098d4f330..c9df3acb1 100644 --- a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs @@ -52,6 +52,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastAndAtTimeZoneTransformer let timestamp_arg = SqlExpr::Cast { expr: Box::new(date_arg), data_type: SqlDataType::Timestamp(None, SqlTimezoneInfo::None), + format: None, }; let utc_timestamp = if time_zone == "UTC" { @@ -97,6 +98,7 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { }]), Vec::new(), ), + format: None, }; if time_zone == "UTC" { @@ -117,6 +119,8 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -158,6 +162,8 @@ impl FunctionTransformer for DateToUtcTimestampWithFunctionTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -198,6 +204,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(date_arg), data_type: self.timestamp_type.clone(), + format: None, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -210,6 +217,8 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -252,6 +261,8 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -269,6 +280,8 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -302,6 +315,8 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -326,6 +341,8 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs index 859ed3d62..0abecdcfb 100644 --- a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs @@ -74,6 +74,8 @@ impl FunctionTransformer for DateTruncTzWithDateTruncAndAtTimezoneTransformer { quote_style: None, }]), args: vec![part_func_arg, ts_func_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -130,6 +132,8 @@ impl FunctionTransformer for DateTruncTzWithTimestampTruncTransformer { quote_style: None, }]), args: vec![ts_func_arg, part_func_arg, tz_func_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -179,6 +183,8 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -199,6 +205,8 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -241,6 +249,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -259,6 +269,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -279,6 +291,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -320,6 +334,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -338,6 +354,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -361,6 +379,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs index a32b04905..322b8843b 100644 --- a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs @@ -44,6 +44,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampBigQueryTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -88,6 +90,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( div_1000_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -101,6 +105,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( floor_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -119,6 +125,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(mod_1000_expr)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(from_unix_time_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -150,6 +158,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDuckDbTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -194,6 +204,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( div_1000_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -207,6 +219,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( floor_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -270,6 +284,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampSnowflakeTransformer { SqlValue::Number("3".to_string(), false), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs index 2b7203e9f..042463fc4 100644 --- a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs @@ -104,6 +104,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -120,6 +122,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -143,6 +147,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { }, ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -158,6 +164,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_expr)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_with_ms_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -175,6 +183,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -215,6 +225,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -234,6 +246,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.millisecond)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -254,6 +268,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -295,6 +311,8 @@ impl FunctionTransformer for MakeUtcTimestampDuckDbTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -348,6 +366,8 @@ impl FunctionTransformer for MakeUtcTimestampPostgresTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -402,6 +422,8 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { ))), })), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -425,6 +447,8 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs index 6f888a0fe..a14e2e087 100644 --- a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs @@ -54,6 +54,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastAndAtTimeZoneTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), + format: None, }; let utc_expr = if time_zone == "UTC" { cast_expr @@ -103,6 +104,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), + format: None, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -115,6 +117,8 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -161,6 +165,8 @@ impl FunctionTransformer for StrToUtcTimestampWithFunctionTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -195,6 +201,8 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -212,6 +220,8 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -245,6 +255,8 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -269,6 +281,8 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -306,6 +320,7 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { }]), Vec::new(), ), + format: None, }; if time_zone == "UTC" { @@ -326,6 +341,8 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs index dc573974f..05b5e23ab 100644 --- a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs @@ -117,6 +117,8 @@ impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -155,6 +157,8 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -172,6 +176,8 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs index 59c5ad598..f125d11ee 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs @@ -46,6 +46,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( ts_expr.clone(), ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -68,6 +70,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -116,6 +120,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( ts_expr.clone(), ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -138,6 +144,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -189,6 +197,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsPostgresTransform { right: Box::new(SqlExpr::Value(SqlValue::Number("1000".to_string(), false))), }, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -226,6 +236,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsSnowflakeTransform { }))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs index 3705124fe..79b2c29cb 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs @@ -58,6 +58,8 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -75,6 +77,8 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -114,6 +118,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -132,6 +138,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString("y-MM-dd HH:mm:ss.SSS".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -154,6 +162,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString("T".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -204,6 +214,8 @@ impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%S.%g".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -254,6 +266,8 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.MS".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -298,6 +312,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -316,6 +332,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.FF3".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/tests/expected/select.toml b/vegafusion-sql/tests/expected/select.toml index 3db7689e9..df2139879 100644 --- a/vegafusion-sql/tests/expected/select.toml +++ b/vegafusion-sql/tests/expected/select.toml @@ -134,7 +134,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES ('0'), ('1')) AS "_values" ("a")), values1 AS (SELECT "a", CAST("a" AS TINYINT) AS "i8", CAST("a" AS SMALLINT) AS "u8", CAST("a" AS SMALLINT) AS "i16", CAST("a" AS INT) AS "u16", CAST("a" AS INT) AS "i32", CAST("a" AS BIGINT) AS "u32", CAST("a" AS BIGINT) AS "i64", CAST("a" AS DOUBLE) AS "f32", CAST("a" AS DOUBLE) AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, CAST(`a` AS INT) AS `i8`, CAST(`a` AS INT) AS `u8`, CAST(`a` AS INT) AS `i16`, CAST(`a` AS INT) AS `u16`, CAST(`a` AS INT) AS `i32`, CAST(`a` AS INT) AS `u32`, CAST(`a` AS INT) AS `i64`, CAST(`a` AS float64) AS `f32`, CAST(`a` AS float64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, CAST(`a` AS INT) AS `i8`, CAST(`a` AS INT) AS `u8`, CAST(`a` AS INT) AS `i16`, CAST(`a` AS INT) AS `u16`, CAST(`a` AS INT) AS `i32`, CAST(`a` AS INT) AS `u32`, CAST(`a` AS INT) AS `i64`, CAST(`a` AS FLOAT64) AS `f32`, CAST(`a` AS FLOAT64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT '0' AS "a" UNION ALL SELECT '1' AS "a"), values1 AS (SELECT "a", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TINYINT) ELSE NULL END AS "i8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "u8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "i16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "u16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "i32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "u32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "i64", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS FLOAT) ELSE NULL END AS "f32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS DOUBLE) ELSE NULL END AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST @@ -174,7 +174,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES ('0'), ('1')) AS "_values" ("a")), values1 AS (SELECT "a", TRY_CAST("a" AS TINYINT) AS "i8", TRY_CAST("a" AS SMALLINT) AS "u8", TRY_CAST("a" AS SMALLINT) AS "i16", TRY_CAST("a" AS INT) AS "u16", TRY_CAST("a" AS INT) AS "i32", TRY_CAST("a" AS BIGINT) AS "u32", TRY_CAST("a" AS BIGINT) AS "i64", TRY_CAST("a" AS DOUBLE) AS "f32", TRY_CAST("a" AS DOUBLE) AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, SAFE_CAST(`a` AS INT) AS `i8`, SAFE_CAST(`a` AS INT) AS `u8`, SAFE_CAST(`a` AS INT) AS `i16`, SAFE_CAST(`a` AS INT) AS `u16`, SAFE_CAST(`a` AS INT) AS `i32`, SAFE_CAST(`a` AS INT) AS `u32`, SAFE_CAST(`a` AS INT) AS `i64`, SAFE_CAST(`a` AS float64) AS `f32`, SAFE_CAST(`a` AS float64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT '0' AS `a` UNION ALL SELECT '1' AS `a`), values1 AS (SELECT `a`, SAFE_CAST(`a` AS INT) AS `i8`, SAFE_CAST(`a` AS INT) AS `u8`, SAFE_CAST(`a` AS INT) AS `i16`, SAFE_CAST(`a` AS INT) AS `u16`, SAFE_CAST(`a` AS INT) AS `i32`, SAFE_CAST(`a` AS INT) AS `u32`, SAFE_CAST(`a` AS INT) AS `i64`, SAFE_CAST(`a` AS FLOAT64) AS `f32`, SAFE_CAST(`a` AS FLOAT64) AS `f64` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT '0' AS "a" UNION ALL SELECT '1' AS "a"), values1 AS (SELECT "a", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TINYINT) ELSE NULL END AS "i8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "u8", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS SMALLINT) ELSE NULL END AS "i16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "u16", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS INT) ELSE NULL END AS "i32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "u32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS BIGINT) ELSE NULL END AS "i64", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS FLOAT) ELSE NULL END AS "f32", CASE WHEN "a" IS NOT NULL THEN CAST("a" AS DOUBLE) ELSE NULL END AS "f64" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST @@ -211,34 +211,34 @@ result = ''' [cast_string] athena = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, true AS `c`, 'A' AS `d`, NULL AS `b` UNION ALL SELECT 1 AS `a`, false AS `c`, 'BB' AS `d`, 1.5 AS `b` UNION ALL SELECT NULL AS `a`, NULL AS `c`, 'CCC' AS `d`, 2.25 AS `b`), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, NULL AS `b`, true AS `c`, 'A' AS `d` UNION ALL SELECT 1 AS `a`, 1.5 AS `b`, false AS `c`, 'BB' AS `d` UNION ALL SELECT NULL AS `a`, 2.25 AS `b`, NULL AS `c`, 'CCC' AS `d`), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ databricks = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT CAST(`a` AS STRING) AS `a`, CAST(`b` AS STRING) AS `b`, CAST(`c` AS STRING) AS `c`, CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ datafusion = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS STRING) AS "a", CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS STRING) AS "a", CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ duckdb = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ mysql = """ -WITH values0 AS (SELECT * FROM (VALUES ROW(0, true, 'A', NULL), ROW(1, false, 'BB', 1.5), ROW(NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC +WITH values0 AS (SELECT * FROM (VALUES ROW(0, NULL, true, 'A'), ROW(1, 1.5, false, 'BB'), ROW(NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC """ postgres = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ redshift = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ snowflake = """ -WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "c", "COLUMN3" AS "d", "COLUMN4" AS "b" FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c", "COLUMN4" AS "d" FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC'))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ result = ''' +---+------+-------+-----+ @@ -252,34 +252,34 @@ result = ''' [try_cast_string] athena = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, true AS `c`, 'A' AS `d`, NULL AS `b` UNION ALL SELECT 1 AS `a`, false AS `c`, 'BB' AS `d`, 1.5 AS `b` UNION ALL SELECT NULL AS `a`, NULL AS `c`, 'CCC' AS `d`, 2.25 AS `b`), values1 AS (SELECT SAFE_CAST(`a` AS STRING) AS `a`, SAFE_CAST(`b` AS STRING) AS `b`, SAFE_CAST(`c` AS STRING) AS `c`, SAFE_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, NULL AS `b`, true AS `c`, 'A' AS `d` UNION ALL SELECT 1 AS `a`, 1.5 AS `b`, false AS `c`, 'BB' AS `d` UNION ALL SELECT NULL AS `a`, 2.25 AS `b`, NULL AS `c`, 'CCC' AS `d`), values1 AS (SELECT SAFE_CAST(`a` AS STRING) AS `a`, SAFE_CAST(`b` AS STRING) AS `b`, SAFE_CAST(`c` AS STRING) AS `c`, SAFE_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS VARCHAR) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS VARCHAR) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CAST("c" AS VARCHAR) ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS VARCHAR) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ databricks = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT TRY_CAST(`a` AS STRING) AS `a`, TRY_CAST(`b` AS STRING) AS `b`, TRY_CAST(`c` AS STRING) AS `c`, TRY_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT TRY_CAST(`a` AS STRING) AS `a`, TRY_CAST(`b` AS STRING) AS `b`, TRY_CAST(`c` AS STRING) AS `c`, TRY_CAST(`d` AS STRING) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ datafusion = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT TRY_CAST("a" AS STRING) AS "a", TRY_CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", TRY_CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT TRY_CAST("a" AS STRING) AS "a", TRY_CAST("b" AS STRING) AS "b", CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END AS "c", TRY_CAST("d" AS STRING) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ duckdb = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT TRY_CAST("a" AS VARCHAR) AS "a", TRY_CAST("b" AS VARCHAR) AS "b", TRY_CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ mysql = """ -WITH values0 AS (SELECT * FROM (VALUES ROW(0, true, 'A', NULL), ROW(1, false, 'BB', 1.5), ROW(NULL, NULL, 'CCC', 2.25)) AS `_values` (`a`, `c`, `d`, `b`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC +WITH values0 AS (SELECT * FROM (VALUES ROW(0, NULL, true, 'A'), ROW(1, 1.5, false, 'BB'), ROW(NULL, 2.25, NULL, 'CCC')) AS `_values` (`a`, `b`, `c`, `d`)), values1 AS (SELECT CAST(`a` AS CHAR) AS `a`, CAST(`b` AS CHAR) AS `b`, CASE WHEN (`c` = true) THEN 'true' WHEN (`c` = false) THEN 'false' ELSE NULL END AS `c`, CAST(`d` AS CHAR) AS `d` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC """ postgres = """ -WITH values0 AS (SELECT * FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25)) AS "_values" ("a", "c", "d", "b")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT * FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC')) AS "_values" ("a", "b", "c", "d")), values1 AS (SELECT CAST("a" AS TEXT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS TEXT) AS "c", CAST("d" AS TEXT) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ redshift = """ -WITH values0 AS (SELECT 0 AS "a", true AS "c", 'A' AS "d", NULL AS "b" UNION ALL SELECT 1 AS "a", false AS "c", 'BB' AS "d", 1.5 AS "b" UNION ALL SELECT NULL AS "a", NULL AS "c", 'CCC' AS "d", 2.25 AS "b"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS "a", NULL AS "b", true AS "c", 'A' AS "d" UNION ALL SELECT 1 AS "a", 1.5 AS "b", false AS "c", 'BB' AS "d" UNION ALL SELECT NULL AS "a", 2.25 AS "b", NULL AS "c", 'CCC' AS "d"), values1 AS (SELECT CASE WHEN "a" IS NOT NULL THEN CAST("a" AS TEXT) ELSE NULL END AS "a", CASE WHEN "b" IS NOT NULL THEN CAST("b" AS TEXT) ELSE NULL END AS "b", CASE WHEN "c" IS NOT NULL THEN CASE WHEN ("c" = true) THEN 'true' WHEN ("c" = false) THEN 'false' ELSE NULL END ELSE NULL END AS "c", CASE WHEN "d" IS NOT NULL THEN CAST("d" AS TEXT) ELSE NULL END AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ snowflake = """ -WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "c", "COLUMN3" AS "d", "COLUMN4" AS "b" FROM (VALUES (0, true, 'A', NULL), (1, false, 'BB', 1.5), (NULL, NULL, 'CCC', 2.25))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST +WITH values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c", "COLUMN4" AS "d" FROM (VALUES (0, NULL, true, 'A'), (1, 1.5, false, 'BB'), (NULL, 2.25, NULL, 'CCC'))), values1 AS (SELECT CAST("a" AS VARCHAR) AS "a", CAST("b" AS VARCHAR) AS "b", CAST("c" AS VARCHAR) AS "c", TRY_CAST("d" AS VARCHAR) AS "d" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ result = ''' +---+------+-------+-----+ @@ -296,7 +296,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES (0)) AS "_values" ("a")), values1 AS (SELECT "a", NULL AS "ninf", NULL AS "nan", NULL AS "inf" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`), values1 AS (SELECT `a`, CAST('-inf' AS float64) AS `ninf`, CAST('NaN' AS float64) AS `nan`, CAST('inf' AS float64) AS `inf` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`), values1 AS (SELECT `a`, CAST('-inf' AS FLOAT64) AS `ninf`, CAST('NaN' AS FLOAT64) AS `nan`, CAST('inf' AS FLOAT64) AS `inf` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT 0 AS "a"), values1 AS (SELECT "a", CAST('-inf' AS DOUBLE) AS "ninf", CAST('NaN' AS DOUBLE) AS "nan", CAST('inf' AS DOUBLE) AS "inf" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST @@ -379,7 +379,7 @@ athena = """ WITH values0 AS (SELECT * FROM (VALUES (0, 0.0), (1, -1.5), (2, NULL), (3, NULL), (4, NULL)) AS "_values" ("a", "b")), values1 AS (SELECT "a", "b", "a" IS NOT NULL AS "f1", "b" IS NOT NULL AS "f2" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST """ bigquery = """ -WITH values0 AS (SELECT 0 AS `a`, 0.0 AS `b` UNION ALL SELECT 1 AS `a`, -1.5 AS `b` UNION ALL SELECT 2 AS `a`, CAST('-inf' AS float64) AS `b` UNION ALL SELECT 3 AS `a`, CAST('inf' AS float64) AS `b` UNION ALL SELECT 4 AS `a`, CAST('NaN' AS float64) AS `b`), values1 AS (SELECT `a`, `b`, CAST(`a` AS STRING) NOT IN (CAST(CAST('-inf' AS float64) AS STRING), CAST(CAST('inf' AS float64) AS STRING), CAST(CAST('NaN' AS float64) AS STRING)) AS `f1`, CAST(`b` AS STRING) NOT IN (CAST(CAST('-inf' AS float64) AS STRING), CAST(CAST('inf' AS float64) AS STRING), CAST(CAST('NaN' AS float64) AS STRING)) AS `f2` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST +WITH values0 AS (SELECT 0 AS `a`, 0.0 AS `b` UNION ALL SELECT 1 AS `a`, -1.5 AS `b` UNION ALL SELECT 2 AS `a`, CAST('-inf' AS FLOAT64) AS `b` UNION ALL SELECT 3 AS `a`, CAST('inf' AS FLOAT64) AS `b` UNION ALL SELECT 4 AS `a`, CAST('NaN' AS FLOAT64) AS `b`), values1 AS (SELECT `a`, `b`, CAST(`a` AS STRING) NOT IN (CAST(CAST('-inf' AS FLOAT64) AS STRING), CAST(CAST('inf' AS FLOAT64) AS STRING), CAST(CAST('NaN' AS FLOAT64) AS STRING)) AS `f1`, CAST(`b` AS STRING) NOT IN (CAST(CAST('-inf' AS FLOAT64) AS STRING), CAST(CAST('inf' AS FLOAT64) AS STRING), CAST(CAST('NaN' AS FLOAT64) AS STRING)) AS `f2` FROM values0) SELECT * FROM values1 ORDER BY `a` ASC NULLS FIRST """ clickhouse = """ WITH values0 AS (SELECT 0 AS "a", 0.0 AS "b" UNION ALL SELECT 1 AS "a", -1.5 AS "b" UNION ALL SELECT 2 AS "a", CAST('-inf' AS DOUBLE) AS "b" UNION ALL SELECT 3 AS "a", CAST('inf' AS DOUBLE) AS "b" UNION ALL SELECT 4 AS "a", CAST('NaN' AS DOUBLE) AS "b"), values1 AS (SELECT "a", "b", isFinite("a") AS "f1", isFinite("b") AS "f2" FROM values0) SELECT * FROM values1 ORDER BY "a" ASC NULLS FIRST diff --git a/vegafusion-sql/tests/test_aggregate.rs b/vegafusion-sql/tests/test_aggregate.rs index 6c296f08e..3ff95e2eb 100644 --- a/vegafusion-sql/tests/test_aggregate.rs +++ b/vegafusion-sql/tests/test_aggregate.rs @@ -72,6 +72,7 @@ mod test_simple_aggs { #[cfg(test)] mod test_median_agg { use crate::*; + use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -95,7 +96,7 @@ mod test_median_agg { vec![ count(flat_col("a")).alias("count_a"), Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Median, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Median), args: vec![flat_col("a")], distinct: false, filter: None, @@ -122,6 +123,7 @@ mod test_median_agg { #[cfg(test)] mod test_variance_aggs { use crate::*; + use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -144,7 +146,7 @@ mod test_variance_aggs { vec![flat_col("b")], vec![ round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Stddev, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Stddev), args: vec![flat_col("a")], distinct: false, filter: None, @@ -154,7 +156,9 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::StddevPop, + func_def: AggregateFunctionDefinition::BuiltIn( + AggregateFunction::StddevPop, + ), args: vec![flat_col("a")], distinct: false, filter: None, @@ -164,7 +168,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_pop_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::Variance, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Variance), args: vec![flat_col("a")], distinct: false, filter: None, @@ -174,7 +178,9 @@ mod test_variance_aggs { .div(lit(100)) .alias("var_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - fun: AggregateFunction::VariancePop, + func_def: AggregateFunctionDefinition::BuiltIn( + AggregateFunction::VariancePop, + ), args: vec![flat_col("a")], distinct: false, filter: None, diff --git a/vegafusion-sql/tests/test_select.rs b/vegafusion-sql/tests/test_select.rs index c1c52ff70..7132c2b14 100644 --- a/vegafusion-sql/tests/test_select.rs +++ b/vegafusion-sql/tests/test_select.rs @@ -485,12 +485,12 @@ mod test_non_finite_numbers { #[cfg(test)] mod test_scalar_math_functions { use crate::*; - use datafusion_expr::{expr, BuiltinScalarFunction, Expr}; + use datafusion_expr::{expr, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use vegafusion_common::column::flat_col; fn make_scalar_fn1(fun: BuiltinScalarFunction, arg: &str, alias: &str) -> Expr { Expr::ScalarFunction(expr::ScalarFunction { - fun, + func_def: ScalarFunctionDefinition::BuiltIn(fun), args: vec![flat_col(arg)], }) .alias(alias) @@ -498,7 +498,7 @@ mod test_scalar_math_functions { fn make_scalar_fn2(fun: BuiltinScalarFunction, arg1: &str, arg2: &str, alias: &str) -> Expr { Expr::ScalarFunction(expr::ScalarFunction { - fun, + func_def: ScalarFunctionDefinition::BuiltIn(fun), args: vec![flat_col(arg1), flat_col(arg2)], }) .alias(alias) @@ -577,7 +577,7 @@ mod test_is_finite { use arrow::array::{Float64Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, Expr}; + use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; @@ -613,13 +613,13 @@ mod test_is_finite { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ISFINITE_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ISFINITE_UDF.clone())), args: vec![flat_col("a")], }) .alias("f1"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(ISFINITE_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(ISFINITE_UDF.clone())), args: vec![flat_col("b")], }) .alias("f2"), @@ -650,7 +650,7 @@ mod test_is_finite { #[cfg(test)] mod test_str_to_utc_timestamp { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -674,8 +674,10 @@ mod test_str_to_utc_timestamp { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -712,7 +714,7 @@ mod test_str_to_utc_timestamp { #[cfg(test)] mod test_date_part_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; @@ -737,8 +739,10 @@ mod test_date_part_tz { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -750,18 +754,18 @@ mod test_date_part_tz { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), flat_col("b_utc"), lit("UTC")], }) .alias("hours_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), flat_col("b_utc"), lit("America/Los_Angeles")], }) .alias("hours_la"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_PART_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), args: vec![lit("hour"), flat_col("b_utc"), lit("America/New_York")], }) .alias("hours_nyc"), @@ -801,7 +805,7 @@ mod test_date_part_tz { #[cfg(test)] mod test_date_trunc_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_trunc_tz::DATE_TRUNC_TZ_UDF; @@ -826,8 +830,10 @@ mod test_date_trunc_tz { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -839,18 +845,18 @@ mod test_date_trunc_tz { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit("day"), flat_col("b_utc"), lit("UTC")], }) .alias("day_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit("day"), flat_col("b_utc"), lit("America/Los_Angeles")], }) .alias("day_la"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TRUNC_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), args: vec![lit("day"), flat_col("b_utc"), lit("America/New_York")], }) .alias("day_nyc"), @@ -890,7 +896,7 @@ mod test_date_trunc_tz { #[cfg(test)] mod test_make_timestamp_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::make_utc_timestamp::MAKE_UTC_TIMESTAMP; @@ -912,8 +918,8 @@ mod test_make_timestamp_tz { let df_result = df .select(vec![ flat_col("a"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), args: vec![ flat_col("Y"), flat_col("M"), @@ -926,8 +932,8 @@ mod test_make_timestamp_tz { ], }) .alias("ts_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), args: vec![ flat_col("Y"), flat_col("M"), @@ -940,8 +946,8 @@ mod test_make_timestamp_tz { ], }) .alias("ts_nyc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(MAKE_UTC_TIMESTAMP.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), args: vec![ flat_col("Y"), flat_col("M"), @@ -987,7 +993,7 @@ mod test_make_timestamp_tz { #[cfg(test)] mod test_epoch_to_utc_timestamp { use crate::*; - use datafusion_expr::{expr, Expr}; + use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -1009,8 +1015,10 @@ mod test_epoch_to_utc_timestamp { .select(vec![ flat_col("a"), flat_col("t"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("t")], }) .alias("t_utc"), @@ -1047,7 +1055,7 @@ mod test_epoch_to_utc_timestamp { #[cfg(test)] mod test_utc_timestamp_to_epoch_ms { use crate::*; - use datafusion_expr::{expr, Expr}; + use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -1071,8 +1079,10 @@ mod test_utc_timestamp_to_epoch_ms { .select(vec![ flat_col("a"), flat_col("t"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("t")], }) .alias("t_utc"), @@ -1084,8 +1094,10 @@ mod test_utc_timestamp_to_epoch_ms { flat_col("a"), flat_col("t"), flat_col("t_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(UTC_TIMESTAMP_TO_EPOCH_MS.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + UTC_TIMESTAMP_TO_EPOCH_MS.clone(), + )), args: vec![flat_col("t_utc")], }) .alias("epoch_millis"), @@ -1125,7 +1137,7 @@ mod test_utc_timestamp_to_epoch_ms { #[cfg(test)] mod test_date_add_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -1149,8 +1161,10 @@ mod test_date_add_tz { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("UTC")], }) .alias("b_utc"), @@ -1162,13 +1176,13 @@ mod test_date_add_tz { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_ADD_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_ADD_TZ_UDF.clone())), args: vec![lit("month"), lit(1), flat_col("b_utc"), lit("UTC")], }) .alias("month_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_ADD_TZ_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_ADD_TZ_UDF.clone())), args: vec![ lit("month"), lit(1), @@ -1213,7 +1227,7 @@ mod test_date_add_tz { #[cfg(test)] mod test_utc_timestamp_to_str { use crate::*; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -1238,8 +1252,10 @@ mod test_utc_timestamp_to_str { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + STR_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("UTC")], }) .alias("b_utc"), @@ -1251,13 +1267,17 @@ mod test_utc_timestamp_to_str { flat_col("a"), flat_col("b"), flat_col("b_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(UTC_TIMESTAMP_TO_STR_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + UTC_TIMESTAMP_TO_STR_UDF.clone(), + )), args: vec![flat_col("b_utc"), lit("UTC")], }) .alias("str_utc"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(UTC_TIMESTAMP_TO_STR_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + UTC_TIMESTAMP_TO_STR_UDF.clone(), + )), args: vec![flat_col("b_utc"), lit("America/New_York")], }) .alias("str_nyc"), @@ -1300,7 +1320,7 @@ mod test_date_to_utc_timestamp { use arrow::array::{ArrayRef, Date32Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_to_utc_timestamp::DATE_TO_UTC_TIMESTAMP_UDF; @@ -1332,8 +1352,10 @@ mod test_date_to_utc_timestamp { .select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(DATE_TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new( + DATE_TO_UTC_TIMESTAMP_UDF.clone(), + )), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -1373,7 +1395,7 @@ mod test_timestamp_to_utc_timestamp { use arrow::array::{ArrayRef, Int32Array, TimestampMillisecondArray}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, lit, Expr}; + use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::to_utc_timestamp::TO_UTC_TIMESTAMP_UDF; @@ -1404,8 +1426,8 @@ mod test_timestamp_to_utc_timestamp { df.select(vec![ flat_col("a"), flat_col("b"), - Expr::ScalarUDF(expr::ScalarUDF { - fun: Arc::new(TO_UTC_TIMESTAMP_UDF.clone()), + Expr::ScalarFunction(expr::ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(TO_UTC_TIMESTAMP_UDF.clone())), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -1445,7 +1467,7 @@ mod test_timestamp_to_utc_timestamp { #[cfg(test)] mod test_string_ops { use crate::*; - use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr}; + use datafusion_expr::{expr, lit, BuiltinScalarFunction, Expr, ScalarFunctionDefinition}; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -1468,22 +1490,22 @@ mod test_string_ops { flat_col("b"), flat_col("c"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Substr, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Substr), args: vec![flat_col("b"), lit(2), lit(2)], }) .alias("b_substr"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Concat, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Concat), args: vec![flat_col("b"), lit(" "), flat_col("c")], }) .alias("bc_concat"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Upper, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Upper), args: vec![flat_col("b")], }) .alias("b_upper"), Expr::ScalarFunction(expr::ScalarFunction { - fun: BuiltinScalarFunction::Lower, + func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Lower), args: vec![flat_col("b")], }) .alias("b_lower"), diff --git a/vegafusion-sql/tests/test_window.rs b/vegafusion-sql/tests/test_window.rs index 7792b0ade..524033b01 100644 --- a/vegafusion-sql/tests/test_window.rs +++ b/vegafusion-sql/tests/test_window.rs @@ -4,8 +4,8 @@ extern crate lazy_static; mod utils; use datafusion_common::ScalarValue; use datafusion_expr::{ - col, expr, lit, window_function, AggregateFunction, BuiltInWindowFunction, Expr, WindowFrame, - WindowFrameBound, WindowFrameUnits, + col, expr, lit, AggregateFunction, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, + WindowFrameUnits, }; use rstest::rstest; use rstest_reuse::{self, *}; @@ -17,6 +17,7 @@ use vegafusion_sql::dataframe::SqlDataFrame; #[cfg(test)] mod test_simple_aggs_unbounded { use crate::*; + use datafusion_expr::WindowFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -39,18 +40,14 @@ mod test_simple_aggs_unbounded { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ flat_col("a"), flat_col("b"), flat_col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -58,9 +55,7 @@ mod test_simple_aggs_unbounded { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -68,7 +63,7 @@ mod test_simple_aggs_unbounded { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -76,7 +71,7 @@ mod test_simple_aggs_unbounded { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -84,7 +79,7 @@ mod test_simple_aggs_unbounded { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -115,6 +110,7 @@ mod test_simple_aggs_unbounded { #[cfg(test)] mod test_simple_aggs_unbounded_groups { use crate::*; + use datafusion_expr::WindowFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -137,18 +133,18 @@ mod test_simple_aggs_unbounded_groups { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Groups, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Groups, + WindowFrameBound::Preceding(ScalarValue::Null), + WindowFrameBound::CurrentRow, + ); let df_result = df .select(vec![ flat_col("a"), flat_col("b"), flat_col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -156,9 +152,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -166,7 +160,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -174,7 +168,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -182,7 +176,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -214,6 +208,7 @@ mod test_simple_aggs_unbounded_groups { #[cfg(test)] mod test_simple_aggs_bounded { use crate::*; + use datafusion_expr::WindowFunctionDefinition; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -236,18 +231,18 @@ mod test_simple_aggs_bounded { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::from(1)), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Rows, + WindowFrameBound::Preceding(ScalarValue::from(1)), + WindowFrameBound::CurrentRow, + ); let df_result = df .select(vec![ flat_col("a"), flat_col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -255,9 +250,7 @@ mod test_simple_aggs_bounded { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -265,7 +258,7 @@ mod test_simple_aggs_bounded { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -273,7 +266,7 @@ mod test_simple_aggs_bounded { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -281,7 +274,7 @@ mod test_simple_aggs_bounded { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -313,6 +306,7 @@ mod test_simple_aggs_bounded { #[cfg(test)] mod test_simple_aggs_bounded_groups { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -334,18 +328,18 @@ mod test_simple_aggs_bounded_groups { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Groups, - start_bound: WindowFrameBound::Preceding(ScalarValue::from(1)), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Groups, + WindowFrameBound::Preceding(ScalarValue::from(1)), + WindowFrameBound::CurrentRow, + ); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -353,9 +347,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction( - AggregateFunction::Count, - ), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -363,7 +355,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -371,7 +363,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -379,7 +371,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -426,6 +418,7 @@ mod test_simple_aggs_bounded_groups { #[cfg(test)] mod test_simple_window_fns { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -447,18 +440,14 @@ mod test_simple_window_fns { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], @@ -468,7 +457,7 @@ mod test_simple_window_fns { }) .alias("row_num"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Rank, ), args: vec![], @@ -478,7 +467,7 @@ mod test_simple_window_fns { }) .alias("rank"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::DenseRank, ), args: vec![], @@ -488,7 +477,7 @@ mod test_simple_window_fns { }) .alias("d_rank"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::FirstValue, ), args: vec![col("b")], @@ -498,7 +487,7 @@ mod test_simple_window_fns { }) .alias("first"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::LastValue, ), args: vec![col("b")], @@ -532,6 +521,7 @@ mod test_simple_window_fns { #[cfg(test)] mod test_advanced_window_fns { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -553,18 +543,14 @@ mod test_advanced_window_fns { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::NthValue, ), args: vec![col("b"), lit(1)], @@ -574,7 +560,7 @@ mod test_advanced_window_fns { }) .alias("nth1"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::CumeDist, ), args: vec![], @@ -584,7 +570,7 @@ mod test_advanced_window_fns { }) .alias("cdist"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Lag, ), args: vec![col("b")], @@ -594,7 +580,7 @@ mod test_advanced_window_fns { }) .alias("lag_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Lead, ), args: vec![col("b")], @@ -604,7 +590,7 @@ mod test_advanced_window_fns { }) .alias("lead_b"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::Ntile, ), args: vec![lit(2)], @@ -638,6 +624,7 @@ mod test_advanced_window_fns { #[cfg(test)] mod test_unordered_row_number { use crate::*; + use datafusion_expr::WindowFunctionDefinition; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -659,18 +646,14 @@ mod test_unordered_row_number { asc: true, nulls_first: true, })]; - let window_frame = WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(ScalarValue::Null), - end_bound: WindowFrameBound::CurrentRow, - }; + let window_frame = WindowFrame::new(Some(true)); let df_result = df .select(vec![ col("a"), col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: window_function::WindowFunction::BuiltInWindowFunction( + fun: WindowFunctionDefinition::BuiltInWindowFunction( BuiltInWindowFunction::RowNumber, ), args: vec![], diff --git a/vegafusion-wasm/src/lib.rs b/vegafusion-wasm/src/lib.rs index c28304ae9..1e1c20b75 100644 --- a/vegafusion-wasm/src/lib.rs +++ b/vegafusion-wasm/src/lib.rs @@ -268,8 +268,8 @@ impl MsgReceiver { // Filter to update nodes in the comm plan let updated_nodes: Vec<_> = updated_nodes .iter() + .filter(|&node| server_to_client.contains(node)) .cloned() - .filter(|node| server_to_client.contains(node)) .collect(); let request_msg = QueryRequest { @@ -314,8 +314,8 @@ impl MsgReceiver { // Filter to update nodes in the comm plan let updated_nodes: Vec<_> = updated_nodes .iter() + .filter(|&node| server_to_client.contains(node)) .cloned() - .filter(|node| server_to_client.contains(node)) .collect(); if !updated_nodes.is_empty() {