From a011ab182343f98d45f5123a698fe87322db0405 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 10 Aug 2024 20:18:09 -0400 Subject: [PATCH 01/28] compile for DataFusion 38 --- Cargo.lock | 660 ++++++++++++++---- Cargo.toml | 32 +- vegafusion-common/Cargo.toml | 5 + vegafusion-common/src/error.rs | 12 +- vegafusion-python-embed/Cargo.toml | 2 +- vegafusion-runtime/Cargo.toml | 3 + vegafusion-runtime/src/data/tasks.rs | 26 +- .../data/vl_selection_test.rs | 14 +- .../date_time/date_format.rs | 20 +- .../builtin_functions/date_time/date_parts.rs | 14 +- .../builtin_functions/date_time/datetime.rs | 16 +- .../builtin_functions/date_time/time.rs | 14 +- .../date_time/time_offset.rs | 4 +- .../builtin_functions/math/isfinite.rs | 6 +- .../src/expression/compiler/call.rs | 4 +- .../src/expression/compiler/member.rs | 11 +- .../src/expression/compiler/mod.rs | 10 +- .../src/expression/compiler/object.rs | 4 +- vegafusion-runtime/src/transform/aggregate.rs | 10 +- vegafusion-runtime/src/transform/timeunit.rs | 32 +- vegafusion-runtime/src/transform/window.rs | 68 +- vegafusion-sql/src/compile/expr.rs | 65 +- vegafusion-sql/src/compile/scalar.rs | 61 +- vegafusion-sql/src/dataframe/mod.rs | 5 + vegafusion-sql/src/dialect/mod.rs | 95 +-- .../src/dialect/transforms/date_add_tz.rs | 255 +++---- .../src/dialect/transforms/date_part_tz.rs | 201 +++--- .../transforms/date_to_utc_timestamp.rs | 173 ++--- .../src/dialect/transforms/date_trunc_tz.rs | 222 +++--- .../transforms/epoch_ms_to_utc_timestamp.rs | 132 ++-- .../dialect/transforms/make_utc_timestamp.rs | 322 +++++---- .../transforms/str_to_utc_timestamp.rs | 169 ++--- .../dialect/transforms/to_utc_timestamp.rs | 85 ++- .../transforms/utc_timestamp_to_epoch_ms.rs | 129 ++-- .../transforms/utc_timestamp_to_str.rs | 221 +++--- vegafusion-sql/src/dialect/utils.rs | 5 + 36 files changed, 1803 insertions(+), 1304 deletions(-) create mode 100644 vegafusion-sql/src/dialect/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 15a9a737..4a6356f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,9 +156,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" dependencies = [ "arrow-arith", "arrow-array", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" dependencies = [ "arrow-array", "arrow-buffer", @@ -193,16 +193,16 @@ dependencies = [ [[package]] name = "arrow-array" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" dependencies = [ "ahash", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz", + "chrono-tz 0.9.0", "half", "hashbrown 0.14.3", "num", @@ -210,9 +210,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" dependencies = [ "bytes", "half", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" dependencies = [ "arrow-array", "arrow-buffer", @@ -261,9 +261,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" dependencies = [ "arrow-buffer", "arrow-schema", @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -288,9 +288,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" dependencies = [ "arrow-array", "arrow-buffer", @@ -308,9 +308,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" dependencies = [ "arrow-array", "arrow-buffer", @@ -323,9 +323,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" dependencies = [ "ahash", "arrow-array", @@ -333,23 +333,22 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.3", ] [[package]] name = "arrow-schema" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" dependencies = [ "bitflags 2.4.2", ] [[package]] name = "arrow-select" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" dependencies = [ "ahash", "arrow-array", @@ -361,9 +360,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" dependencies = [ "arrow-array", "arrow-buffer", @@ -660,9 +659,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -671,7 +670,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -686,8 +685,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -782,9 +781,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -793,9 +792,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -906,7 +905,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.2.1", + "phf", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build 0.3.0", "phf", ] @@ -921,6 +931,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -1236,9 +1257,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05fb4eeeb7109393a0739ac5b8fd892f95ccef691421491c85544f7997366f68" +checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" dependencies = [ "ahash", "arrow", @@ -1260,6 +1281,7 @@ dependencies = [ "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-sql", "flate2", @@ -1271,9 +1293,10 @@ dependencies = [ "itertools 0.12.1", "log", "num_cpus", - "object_store", + "object_store 0.10.2", "parking_lot 0.12.1", "parquet", + "paste", "pin-project-lite", "rand", "sqlparser", @@ -1288,9 +1311,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "741aeac15c82f239f2fc17deccaab19873abbd62987be20023689b15fa72fa09" +checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" dependencies = [ "ahash", "arrow", @@ -1299,10 +1322,11 @@ dependencies = [ "arrow-schema", "chrono", "half", + "hashbrown 0.14.3", "instant", "libc", "num_cpus", - "object_store", + "object_store 0.10.2", "parquet", "pyo3", "sqlparser", @@ -1310,18 +1334,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ddfb8d8cb51646a30da0122ecfffb81ca16919ae9a3495a9e7468bdcd52b8" +checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282122f90b20e8f98ebfa101e4bf20e718fd2684cf81bef4e8c6366571c64404" +checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" dependencies = [ "arrow", "chrono", @@ -1331,7 +1355,7 @@ dependencies = [ "futures", "hashbrown 0.14.3", "log", - "object_store", + "object_store 0.10.2", "parking_lot 0.12.1", "rand", "tempfile", @@ -1340,13 +1364,14 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5478588f733df0dfd87a62671c7478f590952c95fa2fa5c137e3ff2929491e22" +checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", @@ -1358,9 +1383,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4afd261cea6ac9c3ca1192fd5e9f940596d8e9208c5b1333f4961405db53185" +checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" dependencies = [ "arrow", "base64 0.22.1", @@ -1385,11 +1410,13 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b36a6c4838ab94b5bf8f7a96ce6ce059d805c5d1dcaa6ace49e034eb65cd999" +checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" dependencies = [ + "ahash", "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1401,9 +1428,9 @@ dependencies = [ [[package]] name = "datafusion-functions-array" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fdd200a6233f48d3362e7ccb784f926f759100e44ae2137a5e2dcb986a59c4" +checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" dependencies = [ "arrow", "arrow-array", @@ -1421,9 +1448,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54f2820938810e8a2d71228fd6f59f33396aebc5f5f687fcbf14de5aab6a7e1a" +checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" dependencies = [ "arrow", "async-trait", @@ -1440,9 +1467,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9adf8eb12716f52ddf01e09eb6c94d3c9b291e062c05c91b839a448bddba2ff8" +checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" dependencies = [ "ahash", "arrow", @@ -1471,20 +1498,21 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5472c3230584c150197b3f2c23f2392b9dc54dbfb62ad41e7e36447cfce4be" +checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" dependencies = [ "arrow", "datafusion-common", "datafusion-expr", + "rand", ] [[package]] name = "datafusion-physical-plan" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18ae750c38389685a8b62e5b899bbbec488950755ad6d218f3662d35b800c4fe" +checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" dependencies = [ "ahash", "arrow", @@ -1516,24 +1544,38 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f4d2df0e7ba676fe9c0b7cbc0768ffc8f736600b58d305c6c70555c1259bd4" +checksum = "db8eb8d706c73f01c0e2630c64ffc61c33831b064a813ec08a3e094dc3190c0f" dependencies = [ "arrow", "chrono", "datafusion", "datafusion-common", "datafusion-expr", - "object_store", + "datafusion-proto-common", + "object_store 0.10.2", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde7115772d326eeb78a1c77c974e7753b1538e1747675cb6b428058b654b31c" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "object_store 0.9.1", "prost", ] [[package]] name = "datafusion-sql" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "befc67a3cdfbfa76853f43b10ac27337821bb98e519ab6baf431fcc0bcfcafdb" +checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" dependencies = [ "arrow", "arrow-array", @@ -1541,6 +1583,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "log", + "regex", "sqlparser", "strum 0.26.1", ] @@ -1708,9 +1751,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1941,7 +1984,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", + "indexmap 2.2.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", "indexmap 2.2.3", "slab", "tokio", @@ -2032,6 +2094,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2039,7 +2112,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2077,9 +2173,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -2091,6 +2187,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2098,11 +2214,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", - "rustls", + "http 0.2.11", + "hyper 0.14.28", + "rustls 0.21.10", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", ] [[package]] @@ -2111,12 +2245,32 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-util" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "hyper 1.4.1", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2268,6 +2422,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -2701,19 +2864,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", - "base64 0.21.7", "bytes", "chrono", "futures", "humantime", - "hyper", "itertools 0.12.1", + "parking_lot 0.12.1", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "object_store" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "humantime", + "hyper 1.4.1", + "itertools 0.13.0", "md-5", "parking_lot 0.12.1", "percent-encoding", "quick-xml", "rand", - "reqwest", + "reqwest 0.12.5", "ring", "serde", "serde_json", @@ -2822,9 +3006,9 @@ dependencies = [ [[package]] name = "parquet" -version = "51.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" +checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" dependencies = [ "ahash", "arrow-array", @@ -2845,7 +3029,7 @@ dependencies = [ "lz4_flex", "num", "num-bigint", - "object_store", + "object_store 0.10.2", "paste", "seq-macro", "snap", @@ -2853,6 +3037,7 @@ dependencies = [ "tokio", "twox-hash", "zstd", + "zstd-sys", ] [[package]] @@ -2866,9 +3051,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" @@ -3178,9 +3363,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ "cfg-if", "indoc", @@ -3196,9 +3381,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -3206,9 +3391,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -3216,9 +3401,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -3228,9 +3413,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck", "proc-macro2", @@ -3241,9 +3426,9 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850" +checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c" dependencies = [ "pyo3", "serde", @@ -3251,14 +3436,62 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quinn" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.12", + "socket2 0.5.6", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls 0.23.12", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2 0.5.6", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.35" @@ -3382,11 +3615,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -3395,16 +3628,59 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.21.10", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-rustls 0.27.2", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tokio-rustls 0.26.0", "tokio-util", "tower-service", "url", @@ -3412,8 +3688,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -3424,8 +3699,8 @@ checksum = "88a3e86aa6053e59030e7ce2d2a3b258dd08fc2d337d52f73f6cb480f5858690" dependencies = [ "anyhow", "async-trait", - "http", - "reqwest", + "http 0.2.11", + "reqwest 0.11.24", "serde", "task-local-extensions", "thiserror", @@ -3442,10 +3717,10 @@ dependencies = [ "chrono", "futures", "getrandom", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.28", "parking_lot 0.11.2", - "reqwest", + "reqwest 0.11.24", "reqwest-middleware", "retry-policies", "task-local-extensions", @@ -3536,6 +3811,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -3580,18 +3861,33 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.6", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] @@ -3605,6 +3901,22 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3615,6 +3927,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.102.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -3849,9 +4172,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.45.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", "sqlparser_derive", @@ -3955,6 +4278,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "system-configuration" version = "0.5.1" @@ -4177,7 +4506,18 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.10", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.12", + "rustls-pki-types", "tokio", ] @@ -4251,18 +4591,18 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", "prost", - "rustls", - "rustls-pemfile", + "rustls 0.21.10", + "rustls-pemfile 1.0.4", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tokio-stream", "tower", "tower-layer", @@ -4291,9 +4631,9 @@ checksum = "0fddb2a37b247e6adcb9f239f4e5cefdcc5ed526141a416b943929f13aea2cce" dependencies = [ "base64 0.21.7", "bytes", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "pin-project", "tokio-stream", "tonic", @@ -4333,8 +4673,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -4512,8 +4852,9 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-proto", + "datafusion-proto-common", "jni", - "object_store", + "object_store 0.10.2", "pyo3", "serde_json", "sqlparser", @@ -4568,7 +4909,7 @@ name = "vegafusion-datafusion-udfs" version = "1.6.9" dependencies = [ "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "datafusion-functions", "datafusion-physical-expr", "lazy_static", @@ -4624,11 +4965,12 @@ dependencies = [ "base64 0.21.7", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "criterion", "datafusion-common", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", @@ -4643,13 +4985,13 @@ dependencies = [ "log", "lru", "num-traits", - "object_store", + "object_store 0.10.2", "ordered-float 3.9.2", "pixelmatch", "prost", "prost-types", "regex", - "reqwest", + "reqwest 0.11.24", "reqwest-middleware", "reqwest-retry", "rgb", @@ -4675,7 +5017,7 @@ dependencies = [ "assert_cmd", "clap 4.5.1", "futures-util", - "h2", + "h2 0.3.24", "predicates", "prost", "prost-build", @@ -4707,10 +5049,10 @@ dependencies = [ "deterministic-hash", "lazy_static", "log", - "object_store", + "object_store 0.10.2", "pyo3", "regex", - "reqwest", + "reqwest 0.11.24", "reqwest-middleware", "reqwest-retry", "rstest", @@ -5205,6 +5547,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xz2" version = "0.1.7" @@ -5234,6 +5586,12 @@ dependencies = [ "syn 2.0.50", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + [[package]] name = "zstd" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index 4d9850d1..2e60f7be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,40 +14,46 @@ members = [ ] [workspace.dependencies] -arrow = { version = "51.0.0", default-features = false } -sqlparser = { version = "0.45.0" } +arrow = { version = "52.0.0", default-features = false } +sqlparser = { version = "0.47.0" } chrono = { version = "0.4.31", default-features = false } reqwest = { version = "0.11.22", default-features = false } tokio = { version = "1.36.0" } -pyo3 = { version = "0.20.0" } -pythonize = { version = "0.20.0" } +pyo3 = { version = "0.21.1" } +pythonize = { version = "0.21.1" } prost = { version = "0.12.3" } prost-types = { version = "0.12.3" } -object_store = { version= "0.9.1" } +object_store = { version= "0.10.1" } [workspace.dependencies.datafusion] -version = "38.0.0" +version = "39.0.0" [workspace.dependencies.datafusion-common] -version = "38.0.0" +version = "39.0.0" [workspace.dependencies.datafusion-expr] -version = "38.0.0" +version = "39.0.0" [workspace.dependencies.datafusion-proto] -version = "38.0.0" +version = "39.0.0" + +[workspace.dependencies.datafusion-proto-common] +version = "39.0.0" [workspace.dependencies.datafusion-physical-expr] -version = "38.0.0" +version = "39.0.0" [workspace.dependencies.datafusion-optimizer] -version = "38.0.0" +version = "39.0.0" [workspace.dependencies.datafusion-functions] -version = "38.0.0" +version = "39.0.0" [workspace.dependencies.datafusion-functions-array] -version = "38.0.0" +version = "39.0.0" + +[workspace.dependencies.datafusion-functions-aggregate] +version = "39.0.0" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/Cargo.toml b/vegafusion-common/Cargo.toml index 669cc4c2..56645cfa 100644 --- a/vegafusion-common/Cargo.toml +++ b/vegafusion-common/Cargo.toml @@ -9,6 +9,7 @@ license = "BSD-3-Clause" pyarrow = [ "pyo3", "arrow/pyarrow",] json = [ "serde_json/preserve_order", "arrow/json", "chrono",] prettyprint = [ "arrow/prettyprint",] +proto = ["datafusion-proto", "datafusion-proto-common"] [dependencies] thiserror = "^1.0.29" @@ -43,6 +44,10 @@ workspace = true workspace = true optional = true +[dependencies.datafusion-proto-common] +workspace = true +optional = true + [dependencies.pyo3] workspace = true optional = true diff --git a/vegafusion-common/src/error.rs b/vegafusion-common/src/error.rs index 2815c40e..228759c6 100644 --- a/vegafusion-common/src/error.rs +++ b/vegafusion-common/src/error.rs @@ -4,8 +4,8 @@ use std::num::ParseFloatError; use std::result; use thiserror::Error; -#[cfg(feature = "datafusion-proto")] -use datafusion_proto::logical_plan::to_proto::Error as DataFusionProtoError; +#[cfg(feature = "proto")] +use datafusion_proto_common::to_proto::Error as DataFusionProtoError; #[cfg(feature = "pyo3")] use pyo3::{exceptions::PyValueError, PyErr}; @@ -67,7 +67,7 @@ pub enum VegaFusionError { #[error("DataFusion error: {0}\n{1}")] DataFusionError(DataFusionError, ErrorContext), - #[cfg(feature = "datafusion-proto")] + #[cfg(feature = "proto")] #[error("DataFusion proto error: {0}\n{1}")] DataFusionProtoError(DataFusionProtoError, ErrorContext), @@ -148,7 +148,7 @@ impl VegaFusionError { context.contexts.push(context_fn().into()); VegaFusionError::DataFusionError(err, context) } - #[cfg(feature = "datafusion-proto")] + #[cfg(feature = "proto")] DataFusionProtoError(err, mut context) => { context.contexts.push(context_fn().into()); VegaFusionError::DataFusionProtoError(err, context) @@ -249,7 +249,7 @@ impl VegaFusionError { DataFusionError(err, context) => { VegaFusionError::ExternalError(err.to_string(), context.clone()) } - #[cfg(feature = "datafusion-proto")] + #[cfg(feature = "proto")] DataFusionProtoError(err, context) => { VegaFusionError::ExternalError(err.to_string(), context.clone()) } @@ -335,7 +335,7 @@ impl From for VegaFusionError { } } -#[cfg(feature = "datafusion-proto")] +#[cfg(feature = "proto")] impl From for VegaFusionError { fn from(err: DataFusionProtoError) -> Self { Self::DataFusionProtoError(err, Default::default()) diff --git a/vegafusion-python-embed/Cargo.toml b/vegafusion-python-embed/Cargo.toml index fa6ddc20..24a5c765 100644 --- a/vegafusion-python-embed/Cargo.toml +++ b/vegafusion-python-embed/Cargo.toml @@ -40,7 +40,7 @@ features = [ "pyarrow",] [dependencies.vegafusion-common] path = "../vegafusion-common" -features = [ "pyo3", "datafusion-proto", "base64",] +features = [ "pyo3", "proto", "base64",] version = "1.6.9" [dependencies.vegafusion-core] diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index a013cb57..07f3c3fc 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -106,6 +106,9 @@ workspace = true [dependencies.datafusion-functions-array] workspace = true +[dependencies.datafusion-functions-aggregate] +workspace = true + [dependencies.tokio] workspace = true features = [ "macros", "rt-multi-thread", "fs",] diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index ff7343de..6b104f7c 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -5,7 +5,7 @@ use crate::task_graph::task::TaskCall; use async_trait::async_trait; -use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr}; use std::collections::{HashMap, HashSet}; use std::path::Path; @@ -337,9 +337,7 @@ async fn process_datetimes( .unwrap_or_else(|| "UTC".to_string()); Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*STR_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![flat_col(&spec.name), lit(default_input_tz_str)], }) } else if is_integer_datatype(dtype) { @@ -347,9 +345,7 @@ async fn process_datetimes( let tz_config = tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*MAKE_UTC_TIMESTAMP).clone(), - )), + func: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), args: vec![ flat_col(&spec.name), // year lit(0), // month @@ -400,9 +396,7 @@ async fn process_datetimes( Some(tz) => { // Timestamp has explicit timezone Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), args: vec![flat_col(field.name()), lit(tz.as_ref())], }) } @@ -412,9 +406,7 @@ async fn process_datetimes( tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -427,9 +419,7 @@ async fn process_datetimes( tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*TO_UTC_TIMESTAMP_UDF).clone()), args: vec![ flat_col(field.name()), lit(tz_config.default_input_tz.to_string()), @@ -441,9 +431,7 @@ async fn process_datetimes( tz_config.with_context(|| "No local timezone info provided")?; Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*DATE_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*DATE_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![flat_col(field.name()), lit(tz_config.local_tz.to_string())], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index 4c8d4304..98555797 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -5,7 +5,7 @@ use std::convert::TryFrom; use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_expr::expr::Case; -use datafusion_expr::{expr, lit, Between, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Between, Expr, ExprSchemable}; use datafusion_functions::expr_fn::ceil; use std::str::FromStr; use std::sync::Arc; @@ -133,9 +133,7 @@ impl FieldSpec { DataType::Timestamp(TimeUnit::Millisecond, _) ) { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), - )), + func: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), args: vec![field_col], }) } else { @@ -274,15 +272,11 @@ impl FieldSpec { && is_numeric_datatype(field_type) => { let timestamp_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*STR_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![lit(s), lit(default_input_tz)], }); let ms_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), - )), + func: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), args: vec![timestamp_expr], }); cast_to(ms_expr, field_type, schema) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs index 9835c1ba..bd446834 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_format.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; @@ -42,16 +42,14 @@ pub fn time_format_fn( // is compatible with more SQL dialects, so we want to use it if possible let udf_args = vec![timestamptz_expr, lit(&format_tz_str)]; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone())), + func: Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone()), args: udf_args, })) } else { // General case if format_tz_str.to_ascii_lowercase() != "utc" { timestamptz_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*FROM_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*FROM_UTC_TIMESTAMP_UDF).clone()), args: vec![timestamptz_expr, lit(format_tz_str)], }) } @@ -59,7 +57,7 @@ pub fn time_format_fn( let udf_args = vec![timestamptz_expr, lit(format_str)]; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*FORMAT_TIMESTAMP_UDF).clone())), + func: Arc::new((*FORMAT_TIMESTAMP_UDF).clone()), args: udf_args, })) } @@ -79,14 +77,14 @@ pub fn utc_format_fn( // is compatible with more SQL dialects, so we want to use it if possible let udf_args = vec![timestamptz_expr, lit("UTC")]; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone())), + func: Arc::new((*UTC_TIMESTAMP_TO_STR_UDF).clone()), args: udf_args, })) } else { // General case let udf_args = vec![timestamptz_expr, lit(format_str)]; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*FORMAT_TIMESTAMP_UDF).clone())), + func: Arc::new((*FORMAT_TIMESTAMP_UDF).clone()), args: udf_args, })) } @@ -104,15 +102,13 @@ fn to_timestamptz_expr(arg: &Expr, schema: &DFSchema, default_input_tz: &str) -> }), DataType::Timestamp(_, _) => arg.clone(), DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![arg.clone(), lit(default_input_tz)], }), DataType::Null => arg.clone(), dtype if is_numeric_datatype(&dtype) || matches!(dtype, DataType::Boolean) => { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs index 625d22c4..51a5d49f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::call::TzTransformFn; use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable}; use datafusion_functions::expr_fn::floor; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; @@ -21,7 +21,7 @@ pub fn make_local_datepart_transform(part: &str, tx: Option Expr>) - extract_timestamp_arg(&part, args, schema, &tz_config.default_input_tz.to_string())?; let udf_args = vec![lit(part.clone()), arg, lit(tz_config.local_tz.to_string())]; let mut expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: udf_args, }); @@ -44,7 +44,7 @@ pub fn make_utc_datepart_transform(part: &str, tx: Option Expr>) -> extract_timestamp_arg(&part, args, schema, &tz_config.default_input_tz.to_string())?; let udf_args = vec![lit(part.clone()), arg, lit("UTC")]; let mut expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: udf_args, }); @@ -75,15 +75,11 @@ fn extract_timestamp_arg( }), DataType::Timestamp(_, _) => arg.clone(), DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*STR_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![arg.clone(), lit(default_input_tz)], }), dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![cast_to(arg.clone(), &DataType::Int64, schema)?], }), dtype => { diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs index b7944049..4cf3a1cc 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/datetime.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable}; use std::ops::Deref; use std::str::FromStr; use std::sync::Arc; @@ -44,14 +44,12 @@ pub fn to_date_transform( }; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![arg, lit(default_input_tz.to_string())], })) } else if is_numeric_datatype(&dtype) { Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![cast_to(arg, &DataType::Int64, schema)?], })) } else { @@ -74,9 +72,7 @@ pub fn datetime_transform_fn( if is_string_datatype(&dtype) { let default_input_tz_str = tz_config.default_input_tz.to_string(); arg = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*STR_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![arg, lit(default_input_tz_str)], }) } @@ -86,7 +82,7 @@ pub fn datetime_transform_fn( let udf_args = extract_datetime_component_args(args, &tz_config.default_input_tz.to_string(), schema)?; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), + func: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), args: udf_args, })) } @@ -100,7 +96,7 @@ pub fn make_datetime_components_fn( let udf_args = extract_datetime_component_args(args, &tz_config.default_input_tz.to_string(), schema)?; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.deref().clone())), + func: Arc::new(MAKE_UTC_TIMESTAMP.deref().clone()), args: udf_args, })) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs index 730d8b73..baeacedc 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time.rs @@ -1,5 +1,5 @@ use crate::task_graph::timezone::RuntimeTzConfig; -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -24,9 +24,7 @@ pub fn time_fn(tz_config: &RuntimeTzConfig, args: &[Expr], schema: &DFSchema) -> let expr = match arg.get_type(schema)? { DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64 => { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), - )), + func: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), args: vec![arg.clone()], }) } @@ -34,13 +32,9 @@ pub fn time_fn(tz_config: &RuntimeTzConfig, args: &[Expr], schema: &DFSchema) -> let mut udf_args = vec![lit(tz_config.default_input_tz.to_string())]; udf_args.extend(Vec::from(args)); Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*UTC_TIMESTAMP_TO_EPOCH_MS).clone(), - )), + func: Arc::new((*UTC_TIMESTAMP_TO_EPOCH_MS).clone()), args: vec![Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*STR_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![arg.clone(), lit(tz_config.default_input_tz.to_string())], })], }) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs index 4d8d3fc8..cf6f606f 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/date_time/time_offset.rs @@ -1,6 +1,6 @@ use crate::task_graph::timezone::RuntimeTzConfig; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::error::VegaFusionError; @@ -80,7 +80,7 @@ pub fn time_offset_fn( let mut udf_args = vec![lit(tz_config.local_tz.to_string())]; udf_args.extend(Vec::from(args)); Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_ADD_TZ_UDF).clone())), + func: Arc::new((*DATE_ADD_TZ_UDF).clone()), args: vec![ lit(unit), step, diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs index 269cde3b..4a71a39e 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/math/isfinite.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition, ScalarUDF}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarUDF}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -20,9 +20,7 @@ pub fn is_finite_fn(args: &[Expr], schema: &DFSchema) -> Result { Ok(match dtype { DataType::Float16 | DataType::Float32 | DataType::Float64 => { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ScalarUDF::from( - IsFiniteUDF::new(), - ))), + func: Arc::new(ScalarUDF::from(IsFiniteUDF::new())), args: vec![arg], }) } diff --git a/vegafusion-runtime/src/expression/compiler/call.rs b/vegafusion-runtime/src/expression/compiler/call.rs index bd2acdc4..ece2f37c 100644 --- a/vegafusion-runtime/src/expression/compiler/call.rs +++ b/vegafusion-runtime/src/expression/compiler/call.rs @@ -6,7 +6,7 @@ use crate::expression::compiler::builtin_functions::date_time::datetime::{ use crate::expression::compiler::builtin_functions::type_checking::isvalid::is_valid_fn; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; -use datafusion_expr::{expr, Expr, ScalarFunctionDefinition, ScalarUDF}; +use datafusion_expr::{expr, Expr, ScalarUDF}; use datafusion_functions::expr_fn::isnan; use datafusion_functions::math::{ abs, acos, asin, atan, ceil, cos, exp, floor, ln, power, round, sin, sqrt, tan, @@ -133,7 +133,7 @@ pub fn compile_call( VegaFusionCallable::ScalarUDF { udf, cast } => { let args = compile_scalar_arguments(node, config, schema, cast)?; Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(udf.clone()), + func: udf.clone(), args, })) } diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index 86084311..408f07a9 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -2,7 +2,7 @@ use crate::expression::compiler::builtin_functions::array::length::length_transf use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; -use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr}; use datafusion_functions::expr_fn::substring; use std::convert::TryFrom; use std::sync::Arc; @@ -74,10 +74,7 @@ pub fn compile_member( DataType::Struct(ref fields) => { if fields.iter().any(|f| f.name() == &property_string) { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_object_member_udf( - &dtype, - &property_string, - )?)), + func: Arc::new(make_get_object_member_udf(&dtype, &property_string)?), args: vec![compiled_object], }) } else { @@ -100,9 +97,7 @@ pub fn compile_member( } else if matches!(dtype, DataType::List(_) | DataType::FixedSizeList(_, _)) { if let Some(index) = index { Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(make_get_element_udf( - index as i32, - ))), + func: Arc::new(make_get_element_udf(index as i32)), args: vec![compiled_object], }) } else { diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index d7b66bfd..909ae81e 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -68,7 +68,7 @@ mod test_compile { use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Case, TryCast}; - use datafusion_expr::{expr, lit, not, Expr, Operator, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, not, Expr, Operator}; use std::collections::HashMap; use std::convert::TryFrom; @@ -455,7 +455,7 @@ mod test_compile { let result_expr = compile(&expr, &Default::default(), None).unwrap(); let expected_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(make_object_constructor_udf( + func: Arc::new(make_object_constructor_udf( &["a".to_string(), "two".to_string()], &[ DataType::Float64, @@ -465,14 +465,14 @@ mod test_compile { true, )])), ], - ))), + )), args: vec![ lit(1.0), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(make_object_constructor_udf( + func: Arc::new(make_object_constructor_udf( &["three".to_string()], &[DataType::Float64], - ))), + )), args: vec![lit(3.0)], }), ], diff --git a/vegafusion-runtime/src/expression/compiler/object.rs b/vegafusion-runtime/src/expression/compiler/object.rs index 364718e7..58020682 100644 --- a/vegafusion-runtime/src/expression/compiler/object.rs +++ b/vegafusion-runtime/src/expression/compiler/object.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; -use datafusion_expr::{expr, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, Expr, ExprSchemable}; use std::sync::Arc; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::datafusion_common::DFSchema; @@ -27,7 +27,7 @@ pub fn compile_object( let udf = make_object_constructor_udf(keys.as_slice(), value_types.as_slice()); Ok(Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(udf)), + func: Arc::new(udf), args: values, })) } diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index cb77369e..346101af 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -2,6 +2,8 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use datafusion_expr::{avg, count, count_distinct, lit, max, min, sum, Expr}; +use datafusion_functions_aggregate::median::median_udaf; +use datafusion_functions_aggregate::variance::var_samp_udaf; use sqlparser::ast::NullTreatment; use std::collections::HashMap; @@ -164,9 +166,7 @@ pub fn make_agg_expr_for_col_expr( AggregateOp::Max => max(column), AggregateOp::Sum => sum(numeric_column()?), AggregateOp::Median => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - aggregate_function::AggregateFunction::Median, - ), + func_def: AggregateFunctionDefinition::UDF(median_udaf()), distinct: false, args: vec![numeric_column()?], filter: None, @@ -174,9 +174,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Variance => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - aggregate_function::AggregateFunction::Variance, - ), + func_def: AggregateFunctionDefinition::UDF(var_samp_udaf()), distinct: false, args: vec![numeric_column()?], filter: None, diff --git a/vegafusion-runtime/src/transform/timeunit.rs b/vegafusion-runtime/src/transform/timeunit.rs index d33ae7f1..1305ea96 100644 --- a/vegafusion-runtime/src/transform/timeunit.rs +++ b/vegafusion-runtime/src/transform/timeunit.rs @@ -12,7 +12,7 @@ use vegafusion_core::proto::gen::transforms::{TimeUnit, TimeUnitTimeZone, TimeUn use vegafusion_core::task_graph::task_value::TaskValue; use datafusion_expr::expr::Cast; -use datafusion_expr::{expr, lit, Expr, ExprSchemable, ScalarFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, ExprSchemable}; use itertools::Itertools; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::datatypes::{cast_to, is_numeric_datatype}; @@ -55,7 +55,7 @@ fn timeunit_date_trunc( let tz_str = local_tz.clone().unwrap_or_else(|| "UTC".to_string()); let start_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), + func: Arc::new(DATE_TRUNC_TZ_UDF.clone()), args: vec![lit(part_str), field_col, lit(tz_str)], }); @@ -94,7 +94,7 @@ fn timeunit_date_part_tz( // Year if units_set.contains(&TimeUnitUnit::Year) { make_timestamptz_args[0] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("year"), field_col.clone(), lit(&tz_str)], }); @@ -104,7 +104,7 @@ fn timeunit_date_part_tz( // Quarter if units_set.contains(&TimeUnitUnit::Quarter) { let month = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("month"), field_col.clone(), lit(&tz_str)], }) .sub(lit(1.0)); @@ -120,7 +120,7 @@ fn timeunit_date_part_tz( // Month if units_set.contains(&TimeUnitUnit::Month) { make_timestamptz_args[1] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("month"), field_col.clone(), lit(&tz_str)], }) .sub(lit(1.0)); @@ -131,7 +131,7 @@ fn timeunit_date_part_tz( // Date if units_set.contains(&TimeUnitUnit::Date) { make_timestamptz_args[2] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("day"), field_col.clone(), lit(&tz_str)], }); @@ -141,7 +141,7 @@ fn timeunit_date_part_tz( // Hour if units_set.contains(&TimeUnitUnit::Hours) { make_timestamptz_args[3] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("hour"), field_col.clone(), lit(&tz_str)], }); @@ -151,7 +151,7 @@ fn timeunit_date_part_tz( // Minute if units_set.contains(&TimeUnitUnit::Minutes) { make_timestamptz_args[4] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("minute"), field_col.clone(), lit(&tz_str)], }); @@ -161,7 +161,7 @@ fn timeunit_date_part_tz( // Second if units_set.contains(&TimeUnitUnit::Seconds) { make_timestamptz_args[5] = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("second"), field_col, lit(&tz_str)], }); @@ -170,7 +170,7 @@ fn timeunit_date_part_tz( // Construct expression to make timestamp from components let start_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), + func: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), args: make_timestamptz_args, }); @@ -187,13 +187,11 @@ fn to_timestamp_col(field: &str, schema: &DFSchema, default_input_tz: &String) - schema, )?, DataType::Utf8 => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone())), + func: Arc::new((*STR_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![field_col, lit(default_input_tz)], }), dtype if is_numeric_datatype(&dtype) => Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - (*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone(), - )), + func: Arc::new((*EPOCH_MS_TO_UTC_TIMESTAMP_UDF).clone()), args: vec![cast_to(field_col, &DataType::Int64, schema)?], }), dtype => { @@ -219,7 +217,7 @@ fn timeunit_weekday( // Use DATE_PART_TZ to extract the weekday // where Sunday is 0 and Saturday is 6 let weekday0 = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("dow"), field_col, lit(tz_str)], }); @@ -241,7 +239,7 @@ fn timeunit_weekday( // Construct expression to make timestamp from components let start_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*MAKE_UTC_TIMESTAMP).clone())), + func: Arc::new((*MAKE_UTC_TIMESTAMP).clone()), args: make_timestamptz_args, }); @@ -509,7 +507,7 @@ impl TransformTrait for TimeUnit { let tz_str = local_tz.unwrap_or_else(|| "UTC".to_string()); let timeunit_end_expr = Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new((*DATE_ADD_TZ_UDF).clone())), + func: Arc::new((*DATE_ADD_TZ_UDF).clone()), args: vec![ lit(&interval.1), lit(interval.0), diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index 33665277..45db29c3 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use datafusion_common::ScalarValue; use datafusion_expr::{aggregate_function, expr, lit, Expr, WindowFrame, WindowFunctionDefinition}; +use datafusion_functions_aggregate::variance::var_samp_udaf; use sqlparser::ast::NullTreatment; use std::sync::Arc; use vegafusion_core::error::Result; @@ -110,29 +111,58 @@ impl TransformTrait for Window { }; use AggregateOp::*; - let (agg_fn, arg) = match op { - Count => (aggregate_function::AggregateFunction::Count, lit(true)), - Sum => (aggregate_function::AggregateFunction::Sum, numeric_field()?), - Mean | Average => { - (aggregate_function::AggregateFunction::Avg, numeric_field()?) - } - Min => (aggregate_function::AggregateFunction::Min, numeric_field()?), - Max => (aggregate_function::AggregateFunction::Max, numeric_field()?), + match op { + Count => ( + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::Count, + ), + vec![lit(true)], + ), + Sum => ( + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::Sum, + ), + vec![numeric_field()?], + ), + Mean | Average => ( + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::Avg, + ), + vec![numeric_field()?], + ), + Min => ( + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::Min, + ), + vec![numeric_field()?], + ), + Max => ( + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::Max, + ), + vec![numeric_field()?], + ), Variance => ( - aggregate_function::AggregateFunction::Variance, - numeric_field()?, + WindowFunctionDefinition::AggregateUDF(var_samp_udaf()), + vec![numeric_field()?], ), Variancep => ( - aggregate_function::AggregateFunction::VariancePop, - numeric_field()?, + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::VariancePop, + ), + vec![numeric_field()?], ), Stdev => ( - aggregate_function::AggregateFunction::Stddev, - numeric_field()?, + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::Stddev, + ), + vec![numeric_field()?], ), Stdevp => ( - aggregate_function::AggregateFunction::StddevPop, - numeric_field()?, + WindowFunctionDefinition::AggregateFunction( + aggregate_function::AggregateFunction::StddevPop, + ), + vec![numeric_field()?], ), // ArrayAgg only available on master right now // Values => (aggregates::AggregateFunction::ArrayAgg, unescaped_col(field)), @@ -141,11 +171,7 @@ impl TransformTrait for Window { "Unsupported window aggregate: {op:?}" ))) } - }; - ( - WindowFunctionDefinition::AggregateFunction(agg_fn), - vec![arg], - ) + } } window_transform_op::Op::WindowOp(op) => { let op = WindowOp::try_from(*op).unwrap(); diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index c0e72853..7431f74f 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -3,9 +3,9 @@ use crate::compile::scalar::ToSqlScalar; use arrow::datatypes::DataType; use datafusion_common::{DFSchema, ScalarValue}; use sqlparser::ast::{ - BinaryOperator as SqlBinaryOperator, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, Ident, ObjectName as SqlObjectName, - UnaryOperator as SqlUnaryOperator, WindowFrame as SqlWindowFrame, + BinaryOperator as SqlBinaryOperator, CastKind, Expr as SqlExpr, Function as SqlFunction, + FunctionArg as SqlFunctionArg, FunctionArgumentList, FunctionArguments, Ident, + ObjectName as SqlObjectName, UnaryOperator as SqlUnaryOperator, WindowFrame as SqlWindowFrame, WindowFrameBound as SqlWindowBound, WindowFrameUnits as SqlWindowFrameUnits, WindowSpec as SqlWindowSpec, WindowType, }; @@ -146,9 +146,6 @@ impl ToSqlExpr for Expr { op: SqlUnaryOperator::Minus, expr: Box::new(expr.to_sql(dialect, schema)?), }))), - Expr::GetIndexedField { .. } => Err(VegaFusionError::internal( - "GetIndexedField cannot be converted to SQL", - )), Expr::Between(Between { expr, negated, @@ -208,6 +205,7 @@ impl ToSqlExpr for Expr { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, format: None, + kind: CastKind::Cast, } }; @@ -238,28 +236,32 @@ impl ToSqlExpr for Expr { transformer.transform(expr.as_ref(), dialect, schema)? } else { match &dialect.try_cast_mode { - TryCastMode::Supported => SqlExpr::TryCast { + TryCastMode::Supported => SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, format: None, + kind: CastKind::TryCast, }, TryCastMode::JustUseCast => SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, format: None, + kind: CastKind::Cast, }, - TryCastMode::SafeCast => SqlExpr::SafeCast { + TryCastMode::SafeCast => SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, format: None, + kind: CastKind::SafeCast, }, TryCastMode::SupportedOnStringsOtherwiseJustCast => { if let DataType::Utf8 | DataType::LargeUtf8 = from_dtype { // TRY_CAST is supported - SqlExpr::TryCast { + SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, format: None, + kind: CastKind::TryCast, } } else { // Fall back to regular CAST @@ -267,6 +269,7 @@ impl ToSqlExpr for Expr { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, format: None, + kind: CastKind::Cast, } } } @@ -358,14 +361,16 @@ impl ToSqlExpr for Expr { UnorderedRowNumberMode::AlternateScalarFunction(alt_fun) => { return Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![Ident::new(alt_fun)]), - args: vec![], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: vec![], - })) + within_group: vec![], + })); } UnorderedRowNumberMode::OrderByConstant => { vec![Expr::Sort(Sort { @@ -447,13 +452,15 @@ impl ToSqlExpr for Expr { value: fun_name, quote_style: None, }]), - args, + args: FunctionArguments::List(FunctionArgumentList { + args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: Some(over), - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }; Ok(SqlExpr::Function(sql_fun)) @@ -547,13 +554,15 @@ fn translate_scalar_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), - args, + args: FunctionArguments::List(FunctionArgumentList { + args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } else if let Some(transformer) = dialect.scalar_transformers.get(fun_name) { // Supported through AST transformation @@ -569,7 +578,7 @@ fn translate_scalar_function( fn translate_aggregate_function( fun_name: &str, args: &[Expr], - distinct: bool, + _distinct: bool, // Where should distinct go now? dialect: &Dialect, schema: &DFSchema, ) -> Result { @@ -582,13 +591,15 @@ fn translate_aggregate_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), - args, + args: FunctionArguments::List(FunctionArgumentList { + args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct, - special: false, - order_by: Default::default(), + within_group: vec![], })) } else if let Some(transformer) = dialect.aggregate_transformers.get(fun_name) { // Supported through AST transformation diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 873b7956..7b20d20e 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -5,16 +5,17 @@ use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; use datafusion_expr::{ - expr, lit, ColumnarValue, Expr, ScalarFunctionDefinition, ScalarUDF, ScalarUDFImpl, Signature, - Volatility, + expr, lit, ColumnarValue, Expr, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; use sqlparser::ast::{ - Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr, - Ident, ObjectName as SqlObjectName, Value as SqlValue, + CastKind, Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, + FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName as SqlObjectName, + Value as SqlValue, }; use std::any::Any; use std::ops::Add; use std::sync::Arc; + use vegafusion_common::data::scalar::ArrayRefHelpers; use vegafusion_common::error::{Result, VegaFusionError}; @@ -29,6 +30,40 @@ impl ToSqlScalar for ScalarValue { ScalarValue::Boolean(v) => Ok(SqlExpr::Value( v.map(SqlValue::Boolean).unwrap_or(SqlValue::Null), )), + ScalarValue::Float16(v) => v + .map(|v| { + let repr = if !v.is_finite() { + // Wrap inf, -inf, and nan in explicit cast + return if dialect.supports_non_finite_floats { + let cast_dtype = if let Some(dtype) = + dialect.cast_datatypes.get(&DataType::Float16) + { + dtype.clone() + } else { + return Err(VegaFusionError::sql_not_supported( + "Dialect does not support a Float16 data type", + )); + }; + Ok(SqlExpr::Cast { + expr: Box::new(SqlExpr::Value(SqlValue::Number( + format!("'{v}'"), + false, + ))), + data_type: cast_dtype, + format: None, + kind: CastKind::Cast, + }) + } else { + Ok(SqlExpr::Value(SqlValue::Null)) + }; + } else if v.to_f32().fract() == 0.0 { + format!("{v:.1}") + } else { + v.to_string() + }; + Ok(SqlExpr::Value(SqlValue::Number(repr, false))) + }) + .unwrap_or(Ok(SqlExpr::Value(SqlValue::Null))), ScalarValue::Float32(v) => v .map(|v| { let repr = if !v.is_finite() { @@ -50,6 +85,7 @@ impl ToSqlScalar for ScalarValue { ))), data_type: cast_dtype, format: None, + kind: CastKind::Cast, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -83,6 +119,7 @@ impl ToSqlScalar for ScalarValue { ))), data_type: cast_dtype, format: None, + kind: CastKind::Cast, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -163,13 +200,15 @@ impl ToSqlScalar for ScalarValue { Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![function_ident]), - args, + args: FunctionArguments::List(FunctionArgumentList { + args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } ScalarValue::Date32(v) => date32_to_date(v, dialect), @@ -265,9 +304,7 @@ impl ToSqlScalar for ScalarValue { fn ms_to_timestamp(v: i64, dialect: &Dialect) -> Result { // Hack to recursively transform the epoch_ms_to_utc_timestamp Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ScalarUDF::from( - EpochMsToUtcTimestampUDF::new(), - ))), + func: Arc::new(ScalarUDF::from(EpochMsToUtcTimestampUDF::new())), args: vec![lit(v)], }) .to_sql(dialect, &DFSchema::empty()) @@ -321,6 +358,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { expr: Box::new(ScalarValue::Utf8(None).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, format: None, + kind: CastKind::Cast, }), Some(days) => { let date = epoch.add(chrono::Duration::days(*days as i64)); @@ -329,6 +367,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { expr: Box::new(ScalarValue::from(date_str.as_str()).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, format: None, + kind: CastKind::Cast, }) } } diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index f7ab036f..10224d0f 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -272,6 +272,8 @@ impl SqlDataFrame { } expr_selects.push(Select { + window_before_qualify: false, + connect_by: None, value_table_mode: None, distinct: None, top: None, @@ -354,6 +356,7 @@ impl SqlDataFrame { opt_except: None, opt_rename: None, opt_replace: None, + opt_ilike: None, })]; let table_alias = TableAlias { @@ -397,6 +400,8 @@ impl SqlDataFrame { qualify: None, named_window: Default::default(), value_table_mode: None, + window_before_qualify: false, + connect_by: None, })); Query { with: None, diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index af7b1537..9ca6d67a 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -1,4 +1,5 @@ pub mod transforms; +pub mod utils; use crate::compile::expr::ToSqlExpr; use crate::dialect::transforms::date_add_tz::{ @@ -54,11 +55,11 @@ use datafusion_common::DFSchema; use datafusion_expr::{expr, lit, ExprSchemable}; use datafusion_expr::{when, Expr, Operator}; use sqlparser::ast::{ - BinaryOperator as SqlBinaryOperator, DataType as SqlDataType, Expr as SqlExpr, - Function as SqlFunction, Function, FunctionArg as SqlFunctionArg, FunctionArg, - FunctionArgExpr as SqlFunctionArgExpr, FunctionArgExpr, Ident as SqlIdent, Ident, - ObjectName as SqlObjectName, ObjectName, TimezoneInfo, Value as SqlValue, + BinaryOperator as SqlBinaryOperator, CastKind, DataType as SqlDataType, Expr as SqlExpr, + Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, + ObjectName, TimezoneInfo, Value as SqlValue, }; + use sqlparser::dialect::{ BigQueryDialect, ClickHouseDialect, Dialect as SqlParserDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, RedshiftSqlDialect, SnowflakeDialect, @@ -1742,21 +1743,23 @@ impl BinaryOperatorTransformer for ModulusOpToFunction { rhs: SqlExpr, _dialect: &Dialect, ) -> Result { - let arg0 = SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(lhs)); - let arg1 = SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(rhs)); + let arg0 = FunctionArg::Unnamed(FunctionArgExpr::Expr(lhs)); + let arg1 = FunctionArg::Unnamed(FunctionArgExpr::Expr(rhs)); - Ok(SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + Ok(SqlExpr::Function(Function { + name: ObjectName(vec![Ident { value: "MOD".to_string(), quote_style: None, }]), - args: vec![arg0, arg1], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![arg0, arg1], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -1770,7 +1773,7 @@ fn args_to_sql_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result> { +) -> Result> { args.iter() .map(|arg| { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr( @@ -1795,13 +1798,15 @@ impl FunctionTransformer for RenameFunctionTransformer { value: self.0.clone(), quote_style: None, }]), - args: sql_args, + args: FunctionArguments::List(FunctionArgumentList { + args: sql_args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -1829,13 +1834,15 @@ impl FunctionTransformer for ExpWithPowFunctionTransformer { value: "pow".to_string(), quote_style: None, }]), - args: sql_args, + args: FunctionArguments::List(FunctionArgumentList { + args: sql_args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -1862,6 +1869,7 @@ impl FunctionTransformer for CastArgsFunctionTransformer { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: self.cast_dtype.clone(), format: None, + kind: CastKind::Cast, }))) }) .collect::>>()?; @@ -1871,13 +1879,15 @@ impl FunctionTransformer for CastArgsFunctionTransformer { value: self.name.clone(), quote_style: None, }]), - args: sql_args, + args: FunctionArguments::List(FunctionArgumentList { + args: sql_args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -1896,7 +1906,7 @@ impl FunctionTransformer for LogBaseTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let mut sql_args = args_to_sql_args(args, dialect, schema)?; // Append base argument - let base_arg = SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + let base_arg = FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Value( SqlValue::Number(self.base.to_string(), false), ))); @@ -1911,13 +1921,15 @@ impl FunctionTransformer for LogBaseTransformer { value: "log".to_string(), quote_style: None, }]), - args: sql_args, + args: FunctionArguments::List(FunctionArgumentList { + args: sql_args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -1942,6 +1954,7 @@ impl FunctionTransformer for LogBaseWithLnTransformer { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: cast_dtype.clone(), format: None, + kind: CastKind::Cast, }))) }) .collect::>>()? @@ -1954,16 +1967,18 @@ impl FunctionTransformer for LogBaseWithLnTransformer { value: "ln".to_string(), quote_style: None, }]), - args: sql_args, + args: FunctionArguments::List(FunctionArgumentList { + args: sql_args, + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let base_arg = SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + let base_arg = FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Value( SqlValue::Number(self.base.to_string(), false), ))); let denominator = SqlExpr::Function(Function { @@ -1971,13 +1986,15 @@ impl FunctionTransformer for LogBaseWithLnTransformer { value: "ln".to_string(), quote_style: None, }]), - args: vec![base_arg], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![base_arg], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(SqlExpr::BinaryOp { diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index 584b0fd6..24aa868e 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -1,12 +1,13 @@ use crate::compile::expr::ToSqlExpr; use crate::dialect::transforms::date_part_tz::{at_time_zone_if_not_utc, part_to_date_time_field}; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ BinaryOperator as SqlBinaryOperator, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, - ObjectName as SqlObjectName, Value as SqlValue, + FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, ToExternalError, VegaFusionError}; @@ -15,7 +16,7 @@ fn process_date_add_tz_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(String, String, SqlExpr, String)> { +) -> Result<(String, String, SqlExpr, SqlExpr)> { if args.len() != 4 { return Err(VegaFusionError::sql_not_supported( "date_add_tz requires exactly four arguments", @@ -45,14 +46,6 @@ fn process_date_add_tz_args( .parse::() .external("Failed to parse interval step as integer")?; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg3 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Forth argument to date_add_tz must be a string literal", - )); - }; - // Handle special cases for intervals let (n_int, part) = match part.as_str() { "week" => (n_int * 7, "day".to_string()), @@ -60,11 +53,12 @@ fn process_date_add_tz_args( _ => (n_int, part), }; - Ok((part, n_int.to_string(), sql_arg2, time_zone)) + Ok((part, n_int.to_string(), sql_arg2, sql_arg3)) } -fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { - if time_zone == "UTC" { +fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { + let utc = make_utc_expr(); + if time_zone == &utc { ts_expr } else { SqlExpr::Function(SqlFunction { @@ -72,24 +66,27 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { value: "from_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(time_zone.to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) } } -fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { - if time_zone == "UTC" { +fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { + let utc = make_utc_expr(); + if time_zone == &utc { ts_expr } else { SqlExpr::Function(SqlFunction { @@ -97,18 +94,20 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { value: "to_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(time_zone.to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) } } @@ -135,18 +134,18 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { value: "datetime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.clone()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let date_time_field = part_to_date_time_field(&part)?; @@ -155,24 +154,26 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { value: "datetime_add".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Interval( - sqlparser::ast::Interval { - value: Box::new(SqlExpr::Value(SqlValue::Number(n_str, false))), - leading_field: Some(date_time_field), - leading_precision: None, - last_field: None, - fractional_seconds_precision: None, - }, - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Interval( + sqlparser::ast::Interval { + value: Box::new(SqlExpr::Value(SqlValue::Number(n_str, false))), + leading_field: Some(date_time_field), + leading_precision: None, + last_field: None, + fractional_seconds_precision: None, + }, + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let timestamp_expr = SqlExpr::Function(SqlFunction { @@ -180,18 +181,18 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { value: "timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_add_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_add_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(timestamp_expr) @@ -260,22 +261,26 @@ impl FunctionTransformer for DateAddTzDatabricksTransformer { value: "dateadd".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier(SqlIdent { - value: part, - quote_style: None, - }))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::Number(n_str, false), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier( + SqlIdent { + value: part, + quote_style: None, + }, + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::Number(n_str, false), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(maybe_to_utc(shifted_tz_expr, &time_zone)) @@ -317,16 +322,16 @@ impl FunctionTransformer for DateAddTzWithAtTimeZoneIntervalTransformer { op: SqlBinaryOperator::Plus, right: Box::new(interval), }; - - Ok(if time_zone == "UTC" { + let utc = make_utc_expr(); + Ok(if time_zone == utc { addition_expr } else { SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::Nested(Box::new(addition_expr))), - time_zone, + time_zone: Box::new(time_zone.clone()), }), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), } }) } @@ -344,8 +349,8 @@ impl DateAddTzSnowflakeTransformer { impl FunctionTransformer for DateAddTzSnowflakeTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (part, n_str, ts_expr, time_zone) = process_date_add_tz_args(args, dialect, schema)?; - - let timestamp_in_tz = if time_zone == "UTC" { + let utc = make_utc_expr(); + let timestamp_in_tz = if time_zone == utc { ts_expr } else { SqlExpr::Function(SqlFunction { @@ -353,21 +358,21 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.clone()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -376,25 +381,29 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { value: "timestampadd".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier(SqlIdent { - value: part, - quote_style: None, - }))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::Number(n_str, false), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier( + SqlIdent { + value: part, + quote_style: None, + }, + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::Number(n_str, false), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let date_add_in_utc = if time_zone == "UTC" { + let date_add_in_utc = if time_zone == utc { date_add_in_tz } else { SqlExpr::Function(SqlFunction { @@ -402,21 +411,19 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index 53d6bb8c..e163434e 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -1,11 +1,12 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ - DateTimeField as SqlDateTimeField, DateTimeField, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, - ObjectName as SqlObjectName, Value as SqlValue, + DateTimeField as SqlDateTimeField, Expr as SqlExpr, Function as SqlFunction, + FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -14,7 +15,7 @@ fn process_date_part_tz_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(String, SqlExpr, String)> { +) -> Result<(String, SqlExpr, SqlExpr)> { if args.len() != 3 { return Err(VegaFusionError::sql_not_supported( "date_part_tz requires exactly three arguments", @@ -32,36 +33,34 @@ fn process_date_part_tz_args( )); }; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg2 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Third argument to date_part_tz must be a string literal", - )); - }; - Ok((part, sql_arg1, time_zone)) + Ok((part, sql_arg1, sql_arg2)) } -pub fn at_time_zone_if_not_utc(arg: SqlExpr, time_zone: String, naive_timestamps: bool) -> SqlExpr { - if time_zone == "UTC" { +pub fn at_time_zone_if_not_utc( + arg: SqlExpr, + time_zone: SqlExpr, + naive_timestamps: bool, +) -> SqlExpr { + let utc = make_utc_expr(); + if time_zone == utc { arg } else if naive_timestamps { SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::AtTimeZone { timestamp: Box::new(arg), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }), - time_zone, + time_zone: Box::new(time_zone), } } else { SqlExpr::AtTimeZone { timestamp: Box::new(arg), - time_zone, + time_zone: Box::new(time_zone), } } } -pub fn part_to_date_time_field(part: &str) -> Result { +pub fn part_to_date_time_field(part: &str) -> Result { Ok(match part.to_ascii_lowercase().as_str() { "year" | "years" => SqlDateTimeField::Year, "month" | "months " => SqlDateTimeField::Month, @@ -105,18 +104,20 @@ impl FunctionTransformer for DatePartTzWithDatePartAndAtTimezoneTransformer { value: "date_part".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(part), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(part), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -168,18 +169,18 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { value: "toTimeZone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let part_function = match part.to_ascii_lowercase().as_str() { @@ -203,15 +204,17 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { value: part_function.to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - to_timezone_expr, - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + to_timezone_expr, + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -232,8 +235,8 @@ impl DatePartTzWithFromUtcAndDatePartTransformer { impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (part, sql_arg1, time_zone) = process_date_part_tz_args(args, dialect, schema)?; - - let timestamp_in_tz = if time_zone == "UTC" { + let utc = make_utc_expr(); + let timestamp_in_tz = if time_zone == utc { sql_arg1 } else { SqlExpr::Function(SqlFunction { @@ -241,18 +244,18 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { value: "from_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -261,18 +264,20 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { value: "date_part".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(part), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(part), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -289,8 +294,8 @@ impl DatePartTzMySqlTransformer { impl FunctionTransformer for DatePartTzMySqlTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (part, sql_arg1, time_zone) = process_date_part_tz_args(args, dialect, schema)?; - - let timestamp_in_tz = if time_zone == "UTC" { + let utc = make_utc_expr(); + let timestamp_in_tz = if time_zone == utc { sql_arg1 } else { SqlExpr::Function(SqlFunction { @@ -298,21 +303,19 @@ impl FunctionTransformer for DatePartTzMySqlTransformer { value: "convert_tz".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -336,8 +339,8 @@ impl DatePartTzSnowflakeTransformer { impl FunctionTransformer for DatePartTzSnowflakeTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (part, sql_arg1, time_zone) = process_date_part_tz_args(args, dialect, schema)?; - - let timestamp_in_tz = if time_zone == "UTC" { + let utc = make_utc_expr(); + let timestamp_in_tz = if time_zone == utc { sql_arg1 } else { SqlExpr::Function(SqlFunction { @@ -345,21 +348,19 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -368,18 +369,20 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { value: "date_part".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(part), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(part), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } diff --git a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs index c9df3acb..23b7acf6 100644 --- a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs @@ -1,11 +1,13 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ - DataType as SqlDataType, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, - ObjectName as SqlObjectName, TimezoneInfo as SqlTimezoneInfo, Value as SqlValue, + CastKind, DataType as SqlDataType, Expr as SqlExpr, Function as SqlFunction, + FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident as SqlIdent, ObjectName as SqlObjectName, + TimezoneInfo as SqlTimezoneInfo, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -14,7 +16,7 @@ fn process_date_to_utc_timestamp_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(SqlExpr, String)> { +) -> Result<(SqlExpr, SqlExpr)> { if args.len() != 2 { return Err(VegaFusionError::sql_not_supported( "to_utc_timestamp requires exactly two arguments", @@ -22,14 +24,7 @@ fn process_date_to_utc_timestamp_args( } let sql_arg0 = args[0].to_sql(dialect, schema)?; let sql_arg1 = args[1].to_sql(dialect, schema)?; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg1 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Second argument to to_utc_timestamp must be a string literal", - )); - }; - Ok((sql_arg0, time_zone)) + Ok((sql_arg0, sql_arg1)) } /// Convert date_to_utc_timestamp(d, tz) -> @@ -53,18 +48,19 @@ impl FunctionTransformer for DateToUtcTimestampWithCastAndAtTimeZoneTransformer expr: Box::new(date_arg), data_type: SqlDataType::Timestamp(None, SqlTimezoneInfo::None), format: None, + kind: CastKind::Cast, }; - - let utc_timestamp = if time_zone == "UTC" { + let utc = make_utc_expr(); + let utc_timestamp = if time_zone == utc { timestamp_arg } else { let at_tz_expr = SqlExpr::AtTimeZone { timestamp: Box::new(timestamp_arg), - time_zone, + time_zone: Box::new(time_zone), }; SqlExpr::AtTimeZone { timestamp: Box::new(at_tz_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), } }; @@ -99,9 +95,10 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { Vec::new(), ), format: None, + kind: CastKind::Cast, }; - - if time_zone == "UTC" { + let utc = make_utc_expr(); + if time_zone == utc { // No conversion needed Ok(cast_timestamp_ntz_expr) } else { @@ -110,21 +107,21 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(convert_tz_expr) @@ -156,18 +153,18 @@ impl FunctionTransformer for DateToUtcTimestampWithFunctionTransformer { value: self.function_name.clone(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -205,29 +202,31 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { expr: Box::new(date_arg), data_type: self.timestamp_type.clone(), format: None, + kind: CastKind::Cast, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: self.function_name.clone(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); + let utc = make_utc_expr(); if self.at_timezone_utc { Ok(SqlExpr::AtTimeZone { timestamp: Box::new(fn_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }) } else { Ok(fn_expr) @@ -255,18 +254,18 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { value: "toDateTime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let to_time_zone_expr = SqlExpr::Function(SqlFunction { @@ -274,18 +273,20 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { value: "toTimeZone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(to_date_time_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(to_date_time_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(to_time_zone_expr) @@ -314,16 +315,18 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { value: "timestamp".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - - if time_zone == "UTC" { + let utc = make_utc_expr(); + if time_zone == utc { // No conversion needed Ok(timestamp_expr) } else { @@ -332,21 +335,21 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { value: "convert_tz".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs index 0abecdcf..8822f2d6 100644 --- a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs @@ -1,12 +1,13 @@ use crate::compile::expr::ToSqlExpr; use crate::dialect::transforms::date_part_tz::at_time_zone_if_not_utc; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, - FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, ObjectName as SqlObjectName, - Value as SqlValue, + FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, FunctionArguments, + Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -15,7 +16,7 @@ fn process_date_trunc_tz_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(String, SqlExpr, String)> { +) -> Result<(String, SqlExpr, SqlExpr)> { if args.len() != 3 { return Err(VegaFusionError::sql_not_supported( "date_trunc_tz requires exactly three arguments", @@ -33,14 +34,7 @@ fn process_date_trunc_tz_args( )); }; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg2 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Third argument to date_trunc_tz must be a string literal", - )); - }; - Ok((part, sql_arg1, time_zone)) + Ok((part, sql_arg1, sql_arg2)) } /// Convert date_trunc_tz(part, ts, tz) -> @@ -73,29 +67,32 @@ impl FunctionTransformer for DateTruncTzWithDateTruncAndAtTimezoneTransformer { value: "date_trunc".to_string(), quote_style: None, }]), - args: vec![part_func_arg, ts_func_arg], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![part_func_arg, ts_func_arg], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); + let utc = make_utc_expr(); - let truncated_in_utc = if time_zone == "UTC" { + let truncated_in_utc = if time_zone == utc { truncated_in_tz } else if self.naive_timestamps { SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::AtTimeZone { timestamp: Box::new(truncated_in_tz), - time_zone, + time_zone: Box::new(time_zone), }), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), } } else { SqlExpr::AtTimeZone { timestamp: Box::new(truncated_in_tz), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), } }; Ok(truncated_in_utc) @@ -123,21 +120,21 @@ impl FunctionTransformer for DateTruncTzWithTimestampTruncTransformer { value: part, quote_style: None, }))); - let tz_func_arg = SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))); + let tz_func_arg = SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())); Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "timestamp_trunc".to_string(), quote_style: None, }]), - args: vec![ts_func_arg, part_func_arg, tz_func_arg], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ts_func_arg, part_func_arg, tz_func_arg], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -171,27 +168,27 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { ))) } }; - let trunc_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: trunc_function.to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.clone()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); + let utc = make_utc_expr(); - let in_timezone_expr = if time_zone == "UTC" { + let in_timezone_expr = if time_zone == utc { trunc_expr } else { SqlExpr::Function(SqlFunction { @@ -199,18 +196,18 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { value: "toTimeZone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(trunc_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(trunc_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -234,8 +231,9 @@ impl DateTruncTzWithFromUtcAndDateTruncTransformer { impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (part, sql_arg1, time_zone) = process_date_trunc_tz_args(args, dialect, schema)?; + let utc = make_utc_expr(); - let timestamp_in_tz = if time_zone == "UTC" { + let timestamp_in_tz = if time_zone == utc { sql_arg1 } else { SqlExpr::Function(SqlFunction { @@ -243,18 +241,18 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { value: "from_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.clone()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -263,21 +261,23 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { value: "date_trunc".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(part), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(part), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let date_trunc_in_utc = if time_zone == "UTC" { + let date_trunc_in_utc = if time_zone == utc { date_trunc_in_tz } else { SqlExpr::Function(SqlFunction { @@ -285,18 +285,18 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { value: "to_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -316,8 +316,9 @@ impl DateTruncTzSnowflakeTransformer { impl FunctionTransformer for DateTruncTzSnowflakeTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (part, sql_arg1, time_zone) = process_date_trunc_tz_args(args, dialect, schema)?; + let utc = make_utc_expr(); - let timestamp_in_tz = if time_zone == "UTC" { + let timestamp_in_tz = if time_zone == utc { sql_arg1 } else { SqlExpr::Function(SqlFunction { @@ -325,44 +326,45 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.clone()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; - let date_trunc_in_tz = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "date_trunc".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(part), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(part), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let date_trunc_in_utc = if time_zone == "UTC" { + let date_trunc_in_utc = if time_zone == utc { date_trunc_in_tz } else { SqlExpr::Function(SqlFunction { @@ -370,21 +372,19 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; diff --git a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs index 322b8843..6ad5f446 100644 --- a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs @@ -1,11 +1,12 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ BinaryOperator as SqlBinaryOperator, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, - ObjectName as SqlObjectName, Value as SqlValue, + FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -37,19 +38,20 @@ impl EpochMsToUtcTimestampBigQueryTransformer { impl FunctionTransformer for EpochMsToUtcTimestampBigQueryTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let ms_expr = process_epoch_ms_to_utc_timestamp_args(args, dialect, schema)?; - let ts_millis_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "timestamp_millis".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(ts_millis_expr) @@ -87,50 +89,58 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { value: "floor".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - div_1000_expr, - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + div_1000_expr, + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let from_unix_time_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "from_unixtime".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - floor_expr, - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + floor_expr, + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let dateadd_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "dateadd".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier(SqlIdent { - value: "millisecond".to_string(), - quote_style: None, - }))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(mod_1000_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(from_unix_time_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier( + SqlIdent { + value: "millisecond".to_string(), + quote_style: None, + }, + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(mod_1000_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(from_unix_time_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(dateadd_expr) @@ -157,13 +167,15 @@ impl FunctionTransformer for EpochMsToUtcTimestampDuckDbTransformer { value: "epoch_ms".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(epoch_ms_expr) @@ -184,7 +196,7 @@ impl EpochMsToUtcTimestampPostgresTransformer { impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let ms_expr = process_epoch_ms_to_utc_timestamp_args(args, dialect, schema)?; - + let utc = make_utc_expr(); let mod_1000_expr = SqlExpr::BinaryOp { left: Box::new(ms_expr.clone()), op: SqlBinaryOperator::Modulo, @@ -201,35 +213,39 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { value: "floor".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - div_1000_expr, - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + div_1000_expr, + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let to_timestamp_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "to_timestamp".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - floor_expr, - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + floor_expr, + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let to_timestamp_at_utc_expr = SqlExpr::AtTimeZone { timestamp: Box::new(to_timestamp_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }; let interval_expr = SqlExpr::Interval(sqlparser::ast::Interval { @@ -278,18 +294,20 @@ impl FunctionTransformer for EpochMsToUtcTimestampSnowflakeTransformer { value: "to_timestamp_ntz".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::Number("3".to_string(), false), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::Number("3".to_string(), false), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(to_timestamp_expr) diff --git a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs index 042463fc..4b0ff7c2 100644 --- a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs @@ -1,4 +1,5 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use arrow::datatypes::DataType; use datafusion_common::DFSchema; @@ -6,7 +7,8 @@ use datafusion_expr::{cast, lit, Expr}; use sqlparser::ast::{ BinaryOperator as SqlBinaryOperator, DateTimeField as SqlDateTimeField, Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, - Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, + FunctionArgumentList, FunctionArguments, Ident as SqlIdent, ObjectName as SqlObjectName, + Value as SqlValue, }; use std::ops::{Add, Div}; use std::sync::Arc; @@ -20,7 +22,7 @@ struct MakeUtcTimestampSqlArgs { pub minute: SqlExpr, pub second: SqlExpr, pub millisecond: SqlExpr, - pub time_zone: String, + pub time_zone: SqlExpr, } fn process_make_utc_timestamp_args( @@ -37,13 +39,6 @@ fn process_make_utc_timestamp_args( } let sql_tz_arg = args[7].to_sql(dialect, schema)?; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(time_zone)) = sql_tz_arg { - time_zone - } else { - return Err(VegaFusionError::sql_not_supported( - "Third argument to make_utc_timestamp must be a string literal", - )); - }; let month = if add_one_to_month { // Add one so that month ranges from 1 to 12 instead of 0 to 11 @@ -75,7 +70,7 @@ fn process_make_utc_timestamp_args( minute: args[4].to_sql(dialect, schema)?, second, millisecond, - time_zone, + time_zone: sql_tz_arg, }) } @@ -99,17 +94,19 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { value: "date".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let time_expr = SqlExpr::Function(SqlFunction { @@ -117,17 +114,19 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { value: "time".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let time_with_ms_expr = SqlExpr::Function(SqlFunction { @@ -135,24 +134,26 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { value: "time_add".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Interval( - sqlparser::ast::Interval { - value: Box::new(sql_args.millisecond), - leading_field: Some(SqlDateTimeField::Millisecond), - leading_precision: None, - last_field: None, - fractional_seconds_precision: None, - }, - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Interval( + sqlparser::ast::Interval { + value: Box::new(sql_args.millisecond), + leading_field: Some(SqlDateTimeField::Millisecond), + leading_precision: None, + last_field: None, + fractional_seconds_precision: None, + }, + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let datetime_expr = SqlExpr::Function(SqlFunction { @@ -160,16 +161,18 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { value: "datetime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_with_ms_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_with_ms_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let timestamp_expr = SqlExpr::Function(SqlFunction { @@ -177,18 +180,18 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { value: "timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(sql_args.time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(timestamp_expr) @@ -217,20 +220,22 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { value: "make_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let date_add_expr = SqlExpr::Function(SqlFunction { @@ -238,23 +243,28 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { value: "dateadd".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier(SqlIdent { - value: "millisecond".to_string(), - quote_style: None, - }))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.millisecond)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier( + SqlIdent { + value: "millisecond".to_string(), + quote_style: None, + }, + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.millisecond)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let ts_in_utc_expr = if sql_args.time_zone == "UTC" { + let utc = make_utc_expr(); + let ts_in_utc_expr = if sql_args.time_zone == utc { date_add_expr } else { SqlExpr::Function(SqlFunction { @@ -262,18 +272,18 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { value: "to_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(sql_args.time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -303,31 +313,34 @@ impl FunctionTransformer for MakeUtcTimestampDuckDbTransformer { value: "make_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let timestamp_in_utc_expr = if sql_args.time_zone == "UTC" { + let utc = make_utc_expr(); + let timestamp_in_utc_expr = if sql_args.time_zone == utc { make_timestamp_expr } else { SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::AtTimeZone { timestamp: Box::new(make_timestamp_expr), - time_zone: sql_args.time_zone, + time_zone: Box::new(sql_args.time_zone), }), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), } }; @@ -349,34 +362,34 @@ impl MakeUtcTimestampPostgresTransformer { impl FunctionTransformer for MakeUtcTimestampPostgresTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let sql_args = process_make_utc_timestamp_args(args, dialect, schema, true, true)?; - + let utc = make_utc_expr(); let make_timestamp_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "make_timestamptz".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(sql_args.time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let timestamp_in_utc_expr = SqlExpr::AtTimeZone { timestamp: Box::new(make_timestamp_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }; Ok(timestamp_in_utc_expr) @@ -405,32 +418,35 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { value: "timestamp_ntz_from_parts".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), - // Convert ms to ns - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::BinaryOp { - left: Box::new(sql_args.millisecond), - op: SqlBinaryOperator::Multiply, - right: Box::new(SqlExpr::Value(SqlValue::Number( - "1000000".to_string(), - false, - ))), - })), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.year)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.hour)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), + // Convert ms to ns + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::BinaryOp { + left: Box::new(sql_args.millisecond), + op: SqlBinaryOperator::Multiply, + right: Box::new(SqlExpr::Value(SqlValue::Number( + "1000000".to_string(), + false, + ))), + })), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - let timestamp_in_utc_expr = if sql_args.time_zone == "UTC" { + let utc = make_utc_expr(); + let timestamp_in_utc_expr = if sql_args.time_zone == utc { make_timestamp_expr } else { SqlExpr::Function(SqlFunction { @@ -438,21 +454,23 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(sql_args.time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(sql_args.time_zone.to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; diff --git a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs index a14e2e08..7be4c7d7 100644 --- a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs @@ -1,11 +1,12 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ - DataType as SqlDataType, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, - ObjectName as SqlObjectName, Value as SqlValue, + CastKind, DataType as SqlDataType, Expr as SqlExpr, Function as SqlFunction, + FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -14,7 +15,7 @@ fn process_str_to_utc_timestamp_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(SqlExpr, String)> { +) -> Result<(SqlExpr, SqlExpr)> { if args.len() != 2 { return Err(VegaFusionError::sql_not_supported( "str_to_utc_timestamp requires exactly two arguments", @@ -22,14 +23,7 @@ fn process_str_to_utc_timestamp_args( } let sql_arg0 = args[0].to_sql(dialect, schema)?; let sql_arg1 = args[1].to_sql(dialect, schema)?; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg1 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Second argument to str_to_utc_timestamp must be a string literal", - )); - }; - Ok((sql_arg0, time_zone)) + Ok((sql_arg0, sql_arg1)) } /// Convert str_to_utc_timestamp(ts, tz) -> @@ -55,17 +49,19 @@ impl FunctionTransformer for StrToUtcTimestampWithCastAndAtTimeZoneTransformer { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), format: None, + kind: CastKind::Cast, }; - let utc_expr = if time_zone == "UTC" { + let utc = make_utc_expr(); + let utc_expr = if time_zone == utc { cast_expr } else { let at_tz_expr = SqlExpr::AtTimeZone { timestamp: Box::new(cast_expr), - time_zone, + time_zone: Box::new(time_zone), }; SqlExpr::AtTimeZone { timestamp: Box::new(at_tz_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), } }; Ok(utc_expr) @@ -105,29 +101,31 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), format: None, + kind: CastKind::Cast, }; + let utc = make_utc_expr(); let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: self.function_name.clone(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); if self.at_timezone_utc { Ok(SqlExpr::AtTimeZone { timestamp: Box::new(fn_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }) } else { Ok(fn_expr) @@ -159,18 +157,18 @@ impl FunctionTransformer for StrToUtcTimestampWithFunctionTransformer { value: self.function_name.clone(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -195,18 +193,18 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { value: "toDateTime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let to_time_zone_expr = SqlExpr::Function(SqlFunction { @@ -214,18 +212,20 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { value: "toTimeZone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(to_date_time_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(to_date_time_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(to_time_zone_expr) @@ -249,21 +249,24 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (sql_arg0, time_zone) = process_str_to_utc_timestamp_args(args, dialect, schema)?; + let utc = make_utc_expr(); let timestamp_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { value: "timestamp".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); - if time_zone == "UTC" { + if time_zone == utc { // No conversion needed Ok(timestamp_expr) } else { @@ -272,21 +275,21 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { value: "convert_tz".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(convert_tz_expr) @@ -311,6 +314,7 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (sql_arg0, time_zone) = process_str_to_utc_timestamp_args(args, dialect, schema)?; + make_utc_expr(); let cast_timestamp_ntz_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: SqlDataType::Custom( @@ -321,9 +325,10 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { Vec::new(), ), format: None, + kind: CastKind::Cast, }; - - if time_zone == "UTC" { + let utc = make_utc_expr(); + if time_zone == utc { // No conversion needed Ok(cast_timestamp_ntz_expr) } else { @@ -332,21 +337,19 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs index 05b5e23a..fd8146db 100644 --- a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs @@ -1,12 +1,13 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use arrow::datatypes::DataType; use datafusion_common::DFSchema; use datafusion_expr::{Expr, ExprSchemable}; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, - FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, ObjectName as SqlObjectName, - Value as SqlValue, + FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, FunctionArguments, + Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -15,7 +16,7 @@ fn process_to_utc_timestamp_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(SqlExpr, String)> { +) -> Result<(SqlExpr, SqlExpr)> { if args.len() != 2 { return Err(VegaFusionError::sql_not_supported( "to_utc_timestamp requires exactly two arguments", @@ -23,14 +24,7 @@ fn process_to_utc_timestamp_args( } let sql_arg0 = args[0].to_sql(dialect, schema)?; let sql_arg1 = args[1].to_sql(dialect, schema)?; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg1 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Second argument to to_utc_timestamp must be a string literal", - )); - }; - Ok((sql_arg0, time_zone)) + Ok((sql_arg0, sql_arg1)) } /// Convert to_utc_timestamp(ts, tz) -> @@ -58,23 +52,24 @@ impl FunctionTransformer for ToUtcTimestampWithAtTimeZoneTransformer { None }; + let utc = make_utc_expr(); let (sql_arg0, time_zone) = process_to_utc_timestamp_args(args, dialect, schema)?; let utc_expr = if timestamps_tz.is_some() { SqlExpr::AtTimeZone { timestamp: Box::new(sql_arg0), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc.clone()), } - } else if time_zone == "UTC" { + } else if time_zone == utc { sql_arg0 } else { let at_tz_expr = SqlExpr::AtTimeZone { timestamp: Box::new(sql_arg0), - time_zone, + time_zone: Box::new(time_zone), }; SqlExpr::AtTimeZone { timestamp: Box::new(at_tz_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc.clone()), } }; @@ -98,8 +93,8 @@ impl ToUtcTimestampSnowflakeTransform { impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (ts_arg, time_zone) = process_to_utc_timestamp_args(args, dialect, schema)?; - - if time_zone == "UTC" { + let utc = make_utc_expr(); + if time_zone == utc { // No conversion needed Ok(ts_arg) } else { @@ -108,21 +103,21 @@ impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("UTC".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(convert_tz_expr) @@ -146,8 +141,8 @@ impl ToUtcTimestampBigQueryTransform { impl FunctionTransformer for ToUtcTimestampBigQueryTransform { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (ts_arg, time_zone) = process_to_utc_timestamp_args(args, dialect, schema)?; - - if time_zone == "UTC" { + let utc = make_utc_expr(); + if time_zone == utc { // No conversion needed Ok(ts_arg) } else { @@ -156,13 +151,15 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { value: "datetime".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let convert_tz_expr = SqlExpr::Function(SqlFunction { @@ -170,18 +167,18 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { value: "timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), + within_group: vec![], filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs index f125d11e..2ff85c48 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs @@ -5,7 +5,7 @@ use datafusion_expr::Expr; use sqlparser::ast::{ BinaryOperator as SqlBinaryOperator, DateTimeField as SqlDateTimeField, Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr as SqlFunctionArgExpr, - Ident as SqlIdent, Ident, ObjectName as SqlObjectName, Value as SqlValue, + FunctionArgumentList, FunctionArguments, Ident, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -39,19 +39,21 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { let ts_expr = process_utc_timestamp_to_epoch_ms_args(args, dialect, schema)?; let unix_timestamp_expr = SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + name: SqlObjectName(vec![Ident { value: "unix_timestamp".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - ts_expr.clone(), - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + ts_expr.clone(), + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let lhs = SqlExpr::BinaryOp { left: Box::new(unix_timestamp_expr), @@ -60,22 +62,24 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { }; let date_part_expr = SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + name: SqlObjectName(vec![Ident { value: "date_part".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("second".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("second".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let mod_1_expr = SqlExpr::Nested(Box::new(SqlExpr::BinaryOp { left: Box::new(date_part_expr), @@ -113,19 +117,21 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { let ts_expr = process_utc_timestamp_to_epoch_ms_args(args, dialect, schema)?; let unix_timestamp_expr = SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + name: SqlObjectName(vec![Ident { value: "epoch".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - ts_expr.clone(), - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + ts_expr.clone(), + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let lhs = SqlExpr::BinaryOp { left: Box::new(unix_timestamp_expr), @@ -134,22 +140,24 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { }; let date_part_expr = SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + name: SqlObjectName(vec![Ident { value: "date_part".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("millisecond".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("millisecond".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); let rhs = SqlExpr::BinaryOp { left: Box::new(date_part_expr), @@ -186,23 +194,28 @@ impl FunctionTransformer for UtcTimestampToEpochMsPostgresTransform { }; let epoch_expr = SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + name: SqlObjectName(vec![Ident { value: "floor".to_string(), quote_style: None, }]), - args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( - SqlExpr::BinaryOp { - left: Box::new(extract_expr), - op: SqlBinaryOperator::Multiply, - right: Box::new(SqlExpr::Value(SqlValue::Number("1000".to_string(), false))), - }, - ))], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( + SqlExpr::BinaryOp { + left: Box::new(extract_expr), + op: SqlBinaryOperator::Multiply, + right: Box::new(SqlExpr::Value(SqlValue::Number( + "1000".to_string(), + false, + ))), + }, + ))], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(epoch_expr) @@ -225,23 +238,25 @@ impl FunctionTransformer for UtcTimestampToEpochMsSnowflakeTransform { let ts_expr = process_utc_timestamp_to_epoch_ms_args(args, dialect, schema)?; let date_part_expr = SqlExpr::Function(SqlFunction { - name: SqlObjectName(vec![SqlIdent { + name: SqlObjectName(vec![Ident { value: "date_part".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier(Ident { - value: "epoch_millisecond".to_string(), - quote_style: None, - }))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Identifier(Ident { + value: "epoch_millisecond".to_string(), + quote_style: None, + }))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(date_part_expr) diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs index 79b2c29c..45e292a1 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs @@ -1,11 +1,12 @@ use crate::compile::expr::ToSqlExpr; +use crate::dialect::utils::make_utc_expr; use crate::dialect::{Dialect, FunctionTransformer}; use datafusion_common::DFSchema; use datafusion_expr::Expr; use sqlparser::ast::{ Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, - FunctionArgExpr as SqlFunctionArgExpr, Ident as SqlIdent, ObjectName as SqlObjectName, - Value as SqlValue, + FunctionArgExpr as SqlFunctionArgExpr, FunctionArgumentList, FunctionArguments, + Ident as SqlIdent, ObjectName as SqlObjectName, Value as SqlValue, }; use std::sync::Arc; use vegafusion_common::error::{Result, VegaFusionError}; @@ -14,7 +15,7 @@ fn process_utc_timestamp_to_str_args( args: &[Expr], dialect: &Dialect, schema: &DFSchema, -) -> Result<(SqlExpr, String)> { +) -> Result<(SqlExpr, SqlExpr)> { if args.len() != 2 { return Err(VegaFusionError::sql_not_supported( "str_to_utc_timestamp requires exactly two arguments", @@ -22,14 +23,7 @@ fn process_utc_timestamp_to_str_args( } let sql_arg0 = args[0].to_sql(dialect, schema)?; let sql_arg1 = args[1].to_sql(dialect, schema)?; - let time_zone = if let SqlExpr::Value(SqlValue::SingleQuotedString(timezone)) = sql_arg1 { - timezone - } else { - return Err(VegaFusionError::sql_not_supported( - "Second argument to str_to_utc_timestamp must be a string literal", - )); - }; - Ok((sql_arg0, time_zone)) + Ok((sql_arg0, sql_arg1)) } /// Convert utc_timestamp_to_str(ts, tz) -> @@ -52,18 +46,18 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { value: "datetime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(SqlExpr::Function(SqlFunction { @@ -71,18 +65,20 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { value: "format_datetime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%E3S".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%E3S".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], })) } } @@ -104,7 +100,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (ts_expr, time_zone) = process_utc_timestamp_to_str_args(args, dialect, schema)?; - let ts_in_tz_expr = if time_zone == "UTC" { + let utc = make_utc_expr(); + let ts_in_tz_expr = if time_zone == utc { ts_expr } else { SqlExpr::Function(SqlFunction { @@ -112,18 +109,18 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { value: "from_utc_timestamp".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -132,18 +129,20 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { value: "date_format".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("y-MM-dd HH:mm:ss.SSS".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("y-MM-dd HH:mm:ss.SSS".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); // There should be a better way to do this, but including the "T" directly in the format @@ -153,21 +152,23 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { value: "replace".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_format_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(" ".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("T".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_format_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString(" ".to_string()), + ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("T".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(replace_expr) @@ -190,16 +191,16 @@ impl UtcTimestampToStrDuckDBTransformer { impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (ts_expr, time_zone) = process_utc_timestamp_to_str_args(args, dialect, schema)?; - - let utc_expr = if time_zone == "UTC" { + let utc = make_utc_expr(); + let utc_expr = if time_zone == utc { ts_expr } else { SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::AtTimeZone { timestamp: Box::new(ts_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }), - time_zone, + time_zone: Box::new(time_zone), } }; @@ -208,18 +209,20 @@ impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { value: "strftime".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%S.%g".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%S.%g".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(strftime_expr) @@ -243,15 +246,16 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (ts_expr, time_zone) = process_utc_timestamp_to_str_args(args, dialect, schema)?; - let utc_expr = if time_zone == "UTC" { + let utc = make_utc_expr(); + let utc_expr = if time_zone == utc { ts_expr } else { SqlExpr::AtTimeZone { timestamp: Box::new(SqlExpr::AtTimeZone { timestamp: Box::new(ts_expr), - time_zone: "UTC".to_string(), + time_zone: Box::new(utc), }), - time_zone, + time_zone: Box::new(time_zone), } }; @@ -260,18 +264,20 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { value: "to_char".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.MS".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.MS".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(strftime_expr) @@ -295,7 +301,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { fn transform(&self, args: &[Expr], dialect: &Dialect, schema: &DFSchema) -> Result { let (ts_expr, time_zone) = process_utc_timestamp_to_str_args(args, dialect, schema)?; - let ts_in_tz_expr = if time_zone == "UTC" { + let utc = make_utc_expr(); + let ts_in_tz_expr = if time_zone == utc { ts_expr } else { SqlExpr::Function(SqlFunction { @@ -303,21 +310,19 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { value: "convert_timezone".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("UTC".to_string()), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone), - ))), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(utc)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }) }; @@ -326,18 +331,20 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { value: "to_varchar".to_string(), quote_style: None, }]), - args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.FF3".to_string()), - ))), - ], + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( + SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.FF3".to_string()), + ))), + ], + duplicate_treatment: None, + clauses: vec![], + }), filter: None, null_treatment: None, over: None, - distinct: false, - special: false, - order_by: Default::default(), + within_group: vec![], }); Ok(date_format_expr) diff --git a/vegafusion-sql/src/dialect/utils.rs b/vegafusion-sql/src/dialect/utils.rs new file mode 100644 index 00000000..20056945 --- /dev/null +++ b/vegafusion-sql/src/dialect/utils.rs @@ -0,0 +1,5 @@ +use sqlparser::ast::{Expr as SqlExpr, Value as SqlValue}; + +pub fn make_utc_expr() -> SqlExpr { + SqlExpr::Value(SqlValue::SingleQuotedString("UTC".to_string())) +} From 5f46cd1d8a58b47a8da753f3be926eae90956e77 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 10 Aug 2024 20:32:33 -0400 Subject: [PATCH 02/28] fix python build --- vegafusion-python-embed/src/lib.rs | 5 +++-- vegafusion-sql/src/connection/datafusion_py_datasource.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vegafusion-python-embed/src/lib.rs b/vegafusion-python-embed/src/lib.rs index f64ff570..ea3b3ce8 100644 --- a/vegafusion-python-embed/src/lib.rs +++ b/vegafusion-python-embed/src/lib.rs @@ -4,6 +4,7 @@ pub mod dataframe; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict, PyList, PyTuple}; +use std::borrow::Cow; use std::collections::HashMap; use std::sync::{Arc, Once}; use tokio::runtime::Runtime; @@ -623,8 +624,8 @@ fn vegafusion_embed(_py: Python, m: &PyModule) -> PyResult<()> { /// Helper function to parse an input Python string or dict as a ChartSpec fn parse_json_spec(chart_spec: PyObject) -> PyResult { Python::with_gil(|py| -> PyResult { - if let Ok(chart_spec) = chart_spec.extract::<&str>(py) { - match serde_json::from_str::(chart_spec) { + if let Ok(chart_spec) = chart_spec.extract::>(py) { + match serde_json::from_str::(&chart_spec) { Ok(chart_spec) => Ok(chart_spec), Err(err) => Err(PyValueError::new_err(format!( "Failed to parse chart_spec string as Vega: {err}" diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index 3c75d7cc..e3a38909 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -118,7 +118,7 @@ impl ExecutionPlan for PyDatasourceExec { self.projected_schema.clone() } - fn children(&self) -> Vec> { + fn children(&self) -> Vec<&Arc<(dyn ExecutionPlan + 'static)>> { Vec::new() } From 6839a343659603d44b49535b837751c5580ab176 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 10 Aug 2024 21:13:24 -0400 Subject: [PATCH 03/28] Fix pyo3 deprecations --- vegafusion-common/src/data/table.rs | 21 ++-- vegafusion-python-embed/src/connection.rs | 67 ++++++------ vegafusion-python-embed/src/dataframe.rs | 101 ++++++++++-------- vegafusion-python-embed/src/lib.rs | 54 ++++++---- .../connection/datafusion_py_datasource.rs | 6 +- vegafusion-sql/tests/test_aggregate.rs | 6 +- vegafusion-sql/tests/test_select.rs | 94 +++++++--------- 7 files changed, 179 insertions(+), 170 deletions(-) diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index 50568364..8a0ce9b4 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -34,9 +34,9 @@ use { use { arrow::pyarrow::{FromPyArrow, ToPyArrow}, pyo3::{ - prelude::PyModule, + prelude::*, types::{PyList, PyTuple}, - PyAny, PyErr, PyObject, Python, + Bound, PyAny, PyErr, PyObject, Python, }, }; @@ -268,18 +268,17 @@ impl VegaFusionTable { } #[cfg(feature = "pyarrow")] - pub fn from_pyarrow(py: Python, pyarrow_table: &PyAny) -> std::result::Result { + pub fn from_pyarrow(pyarrow_table: &Bound) -> std::result::Result { // Extract table.schema as a Rust Schema - let getattr_args = PyTuple::new(py, vec!["schema"]); - let schema_object = pyarrow_table.call_method1("__getattribute__", getattr_args)?; - let schema = Schema::from_pyarrow(schema_object)?; + let schema_object = pyarrow_table.getattr("schema")?; + let schema = Schema::from_pyarrow_bound(&schema_object)?; // Extract table.to_batches() as a Rust Vec let batches_object = pyarrow_table.call_method0("to_batches")?; let batches_list = batches_object.downcast::()?; let batches = batches_list .iter() - .map(|batch_any| Ok(RecordBatch::from_pyarrow(batch_any)?)) + .map(|batch_any| Ok(RecordBatch::from_pyarrow_bound(&batch_any)?)) .collect::>>()?; Ok(VegaFusionTable::try_new(Arc::new(schema), batches)?) @@ -288,14 +287,16 @@ impl VegaFusionTable { #[cfg(feature = "pyarrow")] pub fn to_pyarrow(&self, py: Python) -> std::result::Result { // Convert table's record batches into Python list of pyarrow batches - let pyarrow_module = PyModule::import(py, "pyarrow")?; + + use pyo3::types::PyAnyMethods; + let pyarrow_module = PyModule::import_bound(py, "pyarrow")?; let table_cls = pyarrow_module.getattr("Table")?; let batch_objects = self .batches .iter() .map(|batch| Ok(batch.to_pyarrow(py)?)) .collect::>>()?; - let batches_list = PyList::new(py, batch_objects); + let batches_list = PyList::new_bound(py, batch_objects); // Convert table's schema into pyarrow schema let schema = if let Some(batch) = self.batches.first() { @@ -308,7 +309,7 @@ impl VegaFusionTable { let schema_object = schema.to_pyarrow(py)?; // Build pyarrow table - let args = PyTuple::new(py, vec![batches_list.as_ref(), schema_object.as_ref(py)]); + let args = PyTuple::new_bound(py, vec![&batches_list, schema_object.bind(py)]); let pa_table = table_cls.call_method1("from_batches", args)?; Ok(PyObject::from(pa_table)) } diff --git a/vegafusion-python-embed/src/connection.rs b/vegafusion-python-embed/src/connection.rs index b14fd19c..cba6dfd5 100644 --- a/vegafusion-python-embed/src/connection.rs +++ b/vegafusion-python-embed/src/connection.rs @@ -49,13 +49,13 @@ fn get_dialect_and_fallback_connection( fn perform_fetch_query(query: &str, schema: &Schema, conn: &PyObject) -> Result { let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { - let query_object = PyString::new(py, query); + let query_object = PyString::new_bound(py, query); let query_object = query_object.as_ref(); let schema_object = schema.to_pyarrow(py)?; - let schema_object = schema_object.as_ref(py); - let args = PyTuple::new(py, vec![query_object, schema_object]); - let table_object = conn.call_method(py, "fetch_query", args, None)?; - VegaFusionTable::from_pyarrow(py, table_object.as_ref(py)) + let schema_object = schema_object.bind(py); + let args = PyTuple::new_bound(py, vec![query_object, schema_object]); + let table_object = conn.call_method_bound(py, "fetch_query", args, None)?; + VegaFusionTable::from_pyarrow(table_object.bind(py)) })?; Ok(table) } @@ -91,14 +91,14 @@ impl Connection for PySqlConnection { async fn tables(&self) -> Result> { let tables = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let tables_object = self.conn.call_method0(py, "tables")?; - let tables_dict = tables_object.downcast::(py)?; + let tables_dict = tables_object.downcast_bound::(py)?; let mut tables: HashMap = HashMap::new(); for key in tables_dict.keys() { - let value = tables_dict.get_item(key)?.unwrap(); + let value = tables_dict.get_item(key.clone())?.unwrap(); let key_string = key.extract::()?; - let value_schema = Schema::from_pyarrow(value)?; + let value_schema = Schema::from_pyarrow_bound(&value)?; tables.insert(key_string, value_schema); } Ok(tables) @@ -130,12 +130,14 @@ impl Connection for PySqlConnection { // Register table with Python connection let table_name_object = table_name.clone().into_py(py); let is_temporary_object = true.into_py(py); - let args = PyTuple::new(py, vec![table_name_object, pa_table, is_temporary_object]); + let args = + PyTuple::new_bound(py, vec![table_name_object, pa_table, is_temporary_object]); match self.conn.call_method1(py, "register_arrow", args) { Ok(_) => {} Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; // Check if we have a fallback connection and this is a RegistrationNotSupportedError if let Some(fallback_connection) = &self.fallback_conn { @@ -172,7 +174,7 @@ impl Connection for PySqlConnection { let inner_opts = opts.clone(); let fallback_connection = Python::with_gil(|py| -> std::result::Result<_, PyErr> { // Build Python CsvReadOptions - let vegafusion_module = PyModule::import(py, "vegafusion.connection")?; + let vegafusion_module = PyModule::import_bound(py, "vegafusion.connection")?; let csv_opts_class = vegafusion_module.getattr("CsvReadOptions")?; let pyschema = inner_opts @@ -188,28 +190,29 @@ impl Connection for PySqlConnection { ("file_extension", inner_opts.file_extension.into_py(py)), ("schema", pyschema), ] - .into_py_dict(py); - let args = PyTuple::empty(py); - let csv_opts = csv_opts_class.call(args, Some(kwargs))?; + .into_py_dict_bound(py); + let args = PyTuple::empty_bound(py); + let csv_opts = csv_opts_class.call(args, Some(&kwargs))?; // Register table with Python connection let table_name_object = table_name.clone().into_py(py); let path_name_object = url.to_string().into_py(py); let is_temporary_object = true.into_py(py); - let args = PyTuple::new( + let args = PyTuple::new_bound( py, vec![ - table_name_object.as_ref(py), - path_name_object.as_ref(py), - csv_opts, - is_temporary_object.as_ref(py), + table_name_object.bind(py), + path_name_object.bind(py), + &csv_opts, + is_temporary_object.bind(py), ], ); match self.conn.call_method1(py, "register_csv", args) { Ok(_) => {} Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; // Check if we have a fallback connection and this is a RegistrationNotSupportedError if let Some(fallback_connection) = &self.fallback_conn { @@ -249,18 +252,19 @@ impl Connection for PySqlConnection { let path_name_object = path.to_string().into_py(py); let is_temporary_object = true.into_py(py); - let args = PyTuple::new( + let args = PyTuple::new_bound( py, vec![ - table_name_object.as_ref(py), - path_name_object.as_ref(py), - is_temporary_object.as_ref(py), + table_name_object.bind(py), + path_name_object.bind(py), + is_temporary_object.bind(py), ], ); match self.conn.call_method1(py, "register_arrow_file", args) { Ok(_) => {} Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; // Check if we have a fallback connection and this is a RegistrationNotSupportedError if let Some(fallback_connection) = &self.fallback_conn { @@ -300,18 +304,19 @@ impl Connection for PySqlConnection { let path_name_object = path.to_string().into_py(py); let is_temporary_object = true.into_py(py); - let args = PyTuple::new( + let args = PyTuple::new_bound( py, vec![ - table_name_object.as_ref(py), - path_name_object.as_ref(py), - is_temporary_object.as_ref(py), + table_name_object.bind(py), + path_name_object.bind(py), + is_temporary_object.bind(py), ], ); match self.conn.call_method1(py, "register_parquet", args) { Ok(_) => {} Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; // Check if we have a fallback connection and this is a RegistrationNotSupportedError if let Some(fallback_connection) = &self.fallback_conn { @@ -377,7 +382,7 @@ impl PySqlDataset { let table_name = table_name_obj.extract::(py)?; let table_schema_obj = dataset.call_method0(py, "table_schema")?; - let table_schema = Schema::from_pyarrow(table_schema_obj.as_ref(py))?; + let table_schema = Schema::from_pyarrow_bound(table_schema_obj.bind(py))?; Ok((table_name, table_schema)) })?; diff --git a/vegafusion-python-embed/src/dataframe.rs b/vegafusion-python-embed/src/dataframe.rs index b5d11f6d..1c5b7878 100644 --- a/vegafusion-python-embed/src/dataframe.rs +++ b/vegafusion-python-embed/src/dataframe.rs @@ -5,7 +5,7 @@ use datafusion_proto::logical_plan::to_proto::serialize_expr; use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec; use prost::Message; use pyo3::prelude::PyModule; -use pyo3::types::{PyBytes, PyTuple}; +use pyo3::types::{PyAnyMethods, PyBytes, PyTuple, PyTypeMethods}; use pyo3::{pyclass, pymethods, IntoPy, PyErr, PyObject, Python}; use serde_json::Value; use std::any::Any; @@ -48,7 +48,7 @@ impl DataFrame for PyDataFrame { fn schema(&self) -> Schema { Python::with_gil(|py| -> std::result::Result<_, PyErr> { let schema_obj = self.dataframe.call_method0(py, "schema")?; - let schema = Schema::from_pyarrow(schema_obj.as_ref(py))?; + let schema = Schema::from_pyarrow_bound(schema_obj.bind(py))?; Ok(schema) }) .expect("Failed to return Schema of DataFrameDatasource") @@ -72,7 +72,7 @@ impl DataFrame for PyDataFrame { async fn collect(&self) -> Result { let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table_object = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table_object.as_ref(py)) + VegaFusionTable::from_pyarrow(table_object.bind(py)) })?; Ok(table) } @@ -85,12 +85,13 @@ impl DataFrame for PyDataFrame { let py_limit = limit.into_py(py); // Build arguments for Python sort method - let args = PyTuple::new(py, vec![py_exprs, py_limit]); + let args = PyTuple::new_bound(py, vec![py_exprs, py_limit]); - let new_py_df = match self.dataframe.call_method(py, "sort", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "sort", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -112,7 +113,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -125,12 +126,13 @@ impl DataFrame for PyDataFrame { let new_df = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let py_exprs = exprs_to_py(py, exprs.clone())?; // Build arguments for Python sort method - let args = PyTuple::new(py, vec![py_exprs]); + let args = PyTuple::new_bound(py, vec![py_exprs]); - let new_py_df = match self.dataframe.call_method(py, "select", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "select", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -152,7 +154,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -171,12 +173,16 @@ impl DataFrame for PyDataFrame { let py_aggr_exprs = exprs_to_py(py, aggr_exprs.clone())?; // Build arguments for Python sort method - let args = PyTuple::new(py, vec![py_group_exprs, py_aggr_exprs]); + let args = PyTuple::new_bound(py, vec![py_group_exprs, py_aggr_exprs]); - let new_py_df = match self.dataframe.call_method(py, "aggregate", args, None) { + let new_py_df = match self + .dataframe + .call_method_bound(py, "aggregate", args, None) + { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -198,7 +204,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -217,12 +223,16 @@ impl DataFrame for PyDataFrame { let py_aggr_exprs = exprs_to_py(py, aggr_exprs.clone())?; // Build arguments for Python sort method - let args = PyTuple::new(py, vec![py_group_exprs, py_aggr_exprs]); + let args = PyTuple::new_bound(py, vec![py_group_exprs, py_aggr_exprs]); - let new_py_df = match self.dataframe.call_method(py, "joinaggregate", args, None) { + let new_py_df = match self + .dataframe + .call_method_bound(py, "joinaggregate", args, None) + { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -244,7 +254,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -257,12 +267,13 @@ impl DataFrame for PyDataFrame { let new_df = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let py_exprs = expr_to_py(py, &predicate)?; // Build arguments for Python sort method - let args = PyTuple::new(py, vec![py_exprs]); + let args = PyTuple::new_bound(py, vec![py_exprs]); - let new_py_df = match self.dataframe.call_method(py, "filter", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "filter", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -284,7 +295,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -299,12 +310,13 @@ impl DataFrame for PyDataFrame { let py_limit = limit.into_py(py); // Build arguments for Python sort method - let args = PyTuple::new(py, vec![py_limit]); + let args = PyTuple::new_bound(py, vec![py_limit]); - let new_py_df = match self.dataframe.call_method(py, "limit", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "limit", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -326,7 +338,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -350,15 +362,16 @@ impl DataFrame for PyDataFrame { let py_order_field = order_field.into_py(py); // Build arguments for Python sort method - let args = PyTuple::new( + let args = PyTuple::new_bound( py, vec![py_fields, py_value_col, py_key_col, py_order_field], ); - let new_py_df = match self.dataframe.call_method(py, "fold", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "fold", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -380,7 +393,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -408,7 +421,7 @@ impl DataFrame for PyDataFrame { let py_mode = mode.to_string().to_ascii_lowercase().into_py(py); // Build arguments for Python sort method - let args = PyTuple::new( + let args = PyTuple::new_bound( py, vec![ py_field, @@ -420,10 +433,11 @@ impl DataFrame for PyDataFrame { ], ); - let new_py_df = match self.dataframe.call_method(py, "stack", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "stack", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -445,7 +459,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -473,15 +487,16 @@ impl DataFrame for PyDataFrame { let py_order_field = order_field.into_py(py); // Build arguments for Python sort method - let args = PyTuple::new( + let args = PyTuple::new_bound( py, vec![py_field, py_value, py_key, py_groupby, py_order_field], ); - let new_py_df = match self.dataframe.call_method(py, "impute", args, None) { + let new_py_df = match self.dataframe.call_method_bound(py, "impute", args, None) { Ok(new_py_df) => new_py_df, Err(err) => { - let exception_name = err.get_type(py).name()?; + let err_bound = err.get_type_bound(py); + let exception_name = err_bound.name()?; if exception_name == "DataFrameOperationNotSupportedError" { // Should fall back to fallback connection below return Ok(None); @@ -503,7 +518,7 @@ impl DataFrame for PyDataFrame { // Fallback let table = Python::with_gil(|py| -> std::result::Result<_, PyErr> { let table = self.dataframe.call_method0(py, "collect")?; - VegaFusionTable::from_pyarrow(py, table.as_ref(py)) + VegaFusionTable::from_pyarrow(table.bind(py)) })?; let new_df: Arc = self.fallback_conn.scan_arrow(table).await?; @@ -524,7 +539,7 @@ fn exprs_to_py(py: Python, exprs: Vec) -> Result { fn expr_to_py(py: Python, expr: &Expr) -> Result { let extension_codec = DefaultLogicalExtensionCodec {}; - let proto_module = PyModule::import(py, "vegafusion.proto.datafusion_pb2")?; + let proto_module = PyModule::import_bound(py, "vegafusion.proto.datafusion_pb2")?; let logical_expr_class = proto_module.getattr("LogicalExprNode")?; let proto_sort_expr = serialize_expr(expr, &extension_codec)?; @@ -532,11 +547,11 @@ fn expr_to_py(py: Python, expr: &Expr) -> Result { let sort_expr_bytes: Vec = proto_sort_expr.encode_to_vec(); // py_logical_expr = LogicalExprNode() - let py_logical_expr = logical_expr_class.call(PyTuple::empty(py), None)?; + let py_logical_expr = logical_expr_class.call(PyTuple::empty_bound(py), None)?; // py_logical_expr.ParseFromString(sort_expr_bytes) - let py_bytes = PyBytes::new(py, sort_expr_bytes.as_slice()); - let args = PyTuple::new(py, vec![py_bytes]); + let py_bytes = PyBytes::new_bound(py, sort_expr_bytes.as_slice()); + let args = PyTuple::new_bound(py, vec![py_bytes]); py_logical_expr.call_method("ParseFromString", args, None)?; // From &PyAny to PyObject to maintain ownership diff --git a/vegafusion-python-embed/src/lib.rs b/vegafusion-python-embed/src/lib.rs index ea3b3ce8..8469f313 100644 --- a/vegafusion-python-embed/src/lib.rs +++ b/vegafusion-python-embed/src/lib.rs @@ -16,7 +16,7 @@ use vegafusion_runtime::task_graph::runtime::{ChartState as RsChartState, VegaFu use crate::connection::{PySqlConnection, PySqlDataset}; use crate::dataframe::PyDataFrame; use env_logger::{Builder, Target}; -use pythonize::{depythonize, pythonize}; +use pythonize::{depythonize_bound, pythonize}; use serde_json::json; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_core::patch::patch_pre_transformed_spec; @@ -79,7 +79,7 @@ impl PyChartState { pub fn update(&self, py: Python, updates: Vec) -> PyResult> { let updates = updates .into_iter() - .map(|el| Ok(depythonize::(el.as_ref(py))?)) + .map(|el| Ok(depythonize_bound::(el.bind(py).clone())?)) .collect::>>()?; let result_updates = py.allow_threads(|| { @@ -142,27 +142,30 @@ struct PyVegaFusionRuntime { impl PyVegaFusionRuntime { fn process_inline_datasets( &self, - inline_datasets: Option<&PyDict>, + inline_datasets: Option<&Bound>, ) -> PyResult<(HashMap, bool)> { let mut any_main_thread = false; if let Some(inline_datasets) = inline_datasets { Python::with_gil(|py| -> PyResult<_> { - let vegafusion_dataset_module = PyModule::import(py, "vegafusion.dataset")?; + let vegafusion_dataset_module = PyModule::import_bound(py, "vegafusion.dataset")?; let sql_dataset_type = vegafusion_dataset_module.getattr("SqlDataset")?; let df_dataset_type = vegafusion_dataset_module.getattr("DataFrameDataset")?; - let vegafusion_datasource_module = PyModule::import(py, "vegafusion.datasource")?; + let vegafusion_datasource_module = + PyModule::import_bound(py, "vegafusion.datasource")?; let datasource_type = vegafusion_datasource_module.getattr("Datasource")?; let imported_datasets = inline_datasets .iter() .map(|(name, inline_dataset)| { - let dataset = if inline_dataset.is_instance(sql_dataset_type)? { + let inline_dataset = inline_dataset.to_object(py); + let inline_dataset = inline_dataset.bind(py); + let dataset = if inline_dataset.is_instance(&sql_dataset_type)? { let main_thread = inline_dataset .call_method0("main_thread")? .extract::()?; any_main_thread = any_main_thread || main_thread; - let sql_dataset = PySqlDataset::new(inline_dataset.into_py(py))?; + let sql_dataset = PySqlDataset::new(inline_dataset.to_object(py))?; let rt = if main_thread { &self.tokio_runtime_current_thread } else { @@ -172,15 +175,15 @@ impl PyVegaFusionRuntime { rt.block_on(sql_dataset.scan_table(&sql_dataset.table_name)) })?; VegaFusionDataset::DataFrame(df) - } else if inline_dataset.is_instance(df_dataset_type)? { + } else if inline_dataset.is_instance(&df_dataset_type)? { let main_thread = inline_dataset .call_method0("main_thread")? .extract::()?; any_main_thread = any_main_thread || main_thread; - let df = Arc::new(PyDataFrame::new(inline_dataset.into_py(py))?); + let df = Arc::new(PyDataFrame::new(inline_dataset.to_object(py))?); VegaFusionDataset::DataFrame(df) - } else if inline_dataset.is_instance(datasource_type)? { + } else if inline_dataset.is_instance(&datasource_type)? { let df = self.tokio_runtime_connection.block_on( self.runtime .conn @@ -192,7 +195,7 @@ impl PyVegaFusionRuntime { // We convert to ipc bytes for two reasons: // - It allows VegaFusionDataset to compute an accurate hash of the table // - It works around https://github.com/hex-inc/vegafusion/issues/268 - let table = VegaFusionTable::from_pyarrow(py, inline_dataset)?; + let table = VegaFusionTable::from_pyarrow(inline_dataset)?; VegaFusionDataset::from_table_ipc_bytes(&table.to_ipc_bytes()?)? }; @@ -260,7 +263,7 @@ impl PyVegaFusionRuntime { local_tz: String, default_input_tz: Option, row_limit: Option, - inline_datasets: Option<&PyDict>, + inline_datasets: Option<&Bound>, ) -> PyResult { let spec = parse_json_spec(spec)?; let tz_config = TzConfig { @@ -291,13 +294,17 @@ impl PyVegaFusionRuntime { }) } - pub fn process_request_bytes(&self, py: Python, request_bytes: &PyBytes) -> PyResult { + pub fn process_request_bytes( + &self, + py: Python, + request_bytes: &Bound, + ) -> PyResult { let request_bytes = request_bytes.as_bytes(); let response_bytes = py.allow_threads(|| { self.tokio_runtime_connection .block_on(self.runtime.query_request_bytes(request_bytes)) })?; - Ok(PyBytes::new(py, &response_bytes).into()) + Ok(PyBytes::new_bound(py, &response_bytes).into()) } #[allow(clippy::too_many_arguments)] @@ -348,7 +355,7 @@ impl PyVegaFusionRuntime { default_input_tz: Option, row_limit: Option, preserve_interactivity: Option, - inline_datasets: Option<&PyDict>, + inline_datasets: Option<&Bound>, keep_signals: Option)>>, keep_datasets: Option)>>, ) -> PyResult<(PyObject, PyObject)> { @@ -407,7 +414,7 @@ impl PyVegaFusionRuntime { local_tz: String, default_input_tz: Option, row_limit: Option, - inline_datasets: Option<&PyDict>, + inline_datasets: Option<&Bound>, ) -> PyResult<(PyObject, PyObject)> { let (inline_datasets, any_main_thread_sources) = self.process_inline_datasets(inline_datasets)?; @@ -457,7 +464,7 @@ impl PyVegaFusionRuntime { .collect(); Python::with_gil(|py| -> PyResult<(PyObject, PyObject)> { - let py_response_list = PyList::empty(py); + let py_response_list = PyList::empty_bound(py); for value in values { let pytable: PyObject = if let TaskValue::Table(table) = value { table.to_pyarrow(py)? @@ -484,7 +491,7 @@ impl PyVegaFusionRuntime { preserve_interactivity: Option, extract_threshold: Option, extracted_format: Option, - inline_datasets: Option<&PyDict>, + inline_datasets: Option<&Bound>, keep_signals: Option)>>, keep_datasets: Option)>>, ) -> PyResult<(PyObject, Vec, PyObject)> { @@ -545,7 +552,7 @@ impl PyVegaFusionRuntime { let table = match extracted_format.as_str() { "pyarrow" => table.to_pyarrow(py)?, "arrow-ipc" => { - PyBytes::new(py, table.to_ipc_bytes()?.as_slice()).to_object(py) + PyBytes::new_bound(py, table.to_ipc_bytes()?.as_slice()).to_object(py) } "arrow-ipc-base64" => table.to_ipc_base64()?.into_py(py), _ => { @@ -556,7 +563,8 @@ impl PyVegaFusionRuntime { } }; - let dataset: PyObject = PyTuple::new(py, &[name, scope, table]).into_py(py); + let dataset: PyObject = + PyTuple::new_bound(py, &[name, scope, table]).into_py(py); Ok(dataset) }) .collect::>>()?; @@ -614,7 +622,7 @@ impl PyVegaFusionRuntime { /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. #[pymodule] -fn vegafusion_embed(_py: Python, m: &PyModule) -> PyResult<()> { +fn vegafusion_embed(_py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -631,8 +639,8 @@ fn parse_json_spec(chart_spec: PyObject) -> PyResult { "Failed to parse chart_spec string as Vega: {err}" ))), } - } else if let Ok(chart_spec) = chart_spec.downcast::(py) { - match depythonize(chart_spec) { + } else if let Ok(chart_spec) = chart_spec.downcast_bound::(py) { + match depythonize_bound::(chart_spec.clone()) { Ok(chart_spec) => Ok(chart_spec), Err(err) => Err(PyValueError::new_err(format!( "Failed to parse chart_spec dict as Vega: {err}" diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index e3a38909..9b28a3ae 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -28,7 +28,7 @@ impl PyDatasource { pub fn try_new(py_datasource: PyObject) -> Result { Python::with_gil(|py| -> Result<_, PyErr> { let table_schema_obj = py_datasource.call_method0(py, "schema")?; - let schema = Arc::new(Schema::from_pyarrow(table_schema_obj.as_ref(py))?); + let schema = Arc::new(Schema::from_pyarrow_bound(table_schema_obj.bind(py))?); Ok(Self { py_datasource, schema, @@ -141,9 +141,9 @@ impl ExecutionPlan for PyDatasourceExec { .iter() .map(|field| field.name().clone()) .collect::>(); - let args = PyTuple::new(py, vec![column_names.into_py(py)]); + let args = PyTuple::new_bound(py, vec![column_names.into_py(py)]); let pa_table = self.db.py_datasource.call_method1(py, "fetch", args)?; - let table = VegaFusionTable::from_pyarrow(py, pa_table.as_ref(py))?; + let table = VegaFusionTable::from_pyarrow(pa_table.bind(py))?; Ok(table) }) .map_err(|err| DataFusionError::Execution(err.to_string()))?; diff --git a/vegafusion-sql/tests/test_aggregate.rs b/vegafusion-sql/tests/test_aggregate.rs index e61df58d..18c42326 100644 --- a/vegafusion-sql/tests/test_aggregate.rs +++ b/vegafusion-sql/tests/test_aggregate.rs @@ -77,6 +77,7 @@ mod test_median_agg { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion::functions_aggregate::median::median_udaf; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); @@ -98,7 +99,7 @@ mod test_median_agg { vec![ count(flat_col("a")).alias("count_a"), Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Median), + func_def: AggregateFunctionDefinition::UDF(median_udaf()), args: vec![flat_col("a")], distinct: false, filter: None, @@ -126,6 +127,7 @@ mod test_median_agg { #[cfg(test)] mod test_variance_aggs { use crate::*; + use datafusion::functions_aggregate::variance::var_samp_udaf; use datafusion_expr::expr::AggregateFunctionDefinition; use vegafusion_common::column::flat_col; @@ -176,7 +178,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_pop_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Variance), + func_def: AggregateFunctionDefinition::UDF(var_samp_udaf()), args: vec![flat_col("a")], distinct: false, filter: None, diff --git a/vegafusion-sql/tests/test_select.rs b/vegafusion-sql/tests/test_select.rs index 1d5311e7..de44cd15 100644 --- a/vegafusion-sql/tests/test_select.rs +++ b/vegafusion-sql/tests/test_select.rs @@ -565,7 +565,7 @@ mod test_is_finite { use arrow::array::{Float64Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; @@ -602,12 +602,12 @@ mod test_is_finite { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ISFINITE_UDF.clone())), + func: Arc::new(ISFINITE_UDF.clone()), args: vec![flat_col("a")], }) .alias("f1"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(ISFINITE_UDF.clone())), + func: Arc::new(ISFINITE_UDF.clone()), args: vec![flat_col("b")], }) .alias("f2"), @@ -638,7 +638,7 @@ mod test_is_finite { #[cfg(test)] mod test_str_to_utc_timestamp { use crate::*; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -663,9 +663,7 @@ mod test_str_to_utc_timestamp { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - STR_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -702,7 +700,7 @@ mod test_str_to_utc_timestamp { #[cfg(test)] mod test_date_part_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_part_tz::DATE_PART_TZ_UDF; @@ -728,9 +726,7 @@ mod test_date_part_tz { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - STR_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -743,17 +739,17 @@ mod test_date_part_tz { flat_col("b"), flat_col("b_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("hour"), flat_col("b_utc"), lit("UTC")], }) .alias("hours_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("hour"), flat_col("b_utc"), lit("America/Los_Angeles")], }) .alias("hours_la"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_PART_TZ_UDF.clone())), + func: Arc::new(DATE_PART_TZ_UDF.clone()), args: vec![lit("hour"), flat_col("b_utc"), lit("America/New_York")], }) .alias("hours_nyc"), @@ -793,7 +789,7 @@ mod test_date_part_tz { #[cfg(test)] mod test_date_trunc_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_trunc_tz::DATE_TRUNC_TZ_UDF; @@ -819,9 +815,7 @@ mod test_date_trunc_tz { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - STR_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -834,17 +828,17 @@ mod test_date_trunc_tz { flat_col("b"), flat_col("b_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), + func: Arc::new(DATE_TRUNC_TZ_UDF.clone()), args: vec![lit("day"), flat_col("b_utc"), lit("UTC")], }) .alias("day_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), + func: Arc::new(DATE_TRUNC_TZ_UDF.clone()), args: vec![lit("day"), flat_col("b_utc"), lit("America/Los_Angeles")], }) .alias("day_la"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_TRUNC_TZ_UDF.clone())), + func: Arc::new(DATE_TRUNC_TZ_UDF.clone()), args: vec![lit("day"), flat_col("b_utc"), lit("America/New_York")], }) .alias("day_nyc"), @@ -884,7 +878,7 @@ mod test_date_trunc_tz { #[cfg(test)] mod test_make_timestamp_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::make_utc_timestamp::MAKE_UTC_TIMESTAMP; @@ -907,7 +901,7 @@ mod test_make_timestamp_tz { .select(vec![ flat_col("a"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), + func: Arc::new(MAKE_UTC_TIMESTAMP.clone()), args: vec![ flat_col("Y"), flat_col("M"), @@ -921,7 +915,7 @@ mod test_make_timestamp_tz { }) .alias("ts_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), + func: Arc::new(MAKE_UTC_TIMESTAMP.clone()), args: vec![ flat_col("Y"), flat_col("M"), @@ -935,7 +929,7 @@ mod test_make_timestamp_tz { }) .alias("ts_nyc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(MAKE_UTC_TIMESTAMP.clone())), + func: Arc::new(MAKE_UTC_TIMESTAMP.clone()), args: vec![ flat_col("Y"), flat_col("M"), @@ -981,7 +975,7 @@ mod test_make_timestamp_tz { #[cfg(test)] mod test_epoch_to_utc_timestamp { use crate::*; - use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -1004,9 +998,7 @@ mod test_epoch_to_utc_timestamp { flat_col("a"), flat_col("t"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("t")], }) .alias("t_utc"), @@ -1043,7 +1035,7 @@ mod test_epoch_to_utc_timestamp { #[cfg(test)] mod test_utc_timestamp_to_epoch_ms { use crate::*; - use datafusion_expr::{expr, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::epoch_to_utc_timestamp::EPOCH_MS_TO_UTC_TIMESTAMP_UDF; @@ -1068,9 +1060,7 @@ mod test_utc_timestamp_to_epoch_ms { flat_col("a"), flat_col("t"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(EPOCH_MS_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("t")], }) .alias("t_utc"), @@ -1083,9 +1073,7 @@ mod test_utc_timestamp_to_epoch_ms { flat_col("t"), flat_col("t_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - UTC_TIMESTAMP_TO_EPOCH_MS.clone(), - )), + func: Arc::new(UTC_TIMESTAMP_TO_EPOCH_MS.clone()), args: vec![flat_col("t_utc")], }) .alias("epoch_millis"), @@ -1125,7 +1113,7 @@ mod test_utc_timestamp_to_epoch_ms { #[cfg(test)] mod test_date_add_tz { use crate::*; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -1150,9 +1138,7 @@ mod test_date_add_tz { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - STR_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("UTC")], }) .alias("b_utc"), @@ -1165,12 +1151,12 @@ mod test_date_add_tz { flat_col("b"), flat_col("b_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_ADD_TZ_UDF.clone())), + func: Arc::new(DATE_ADD_TZ_UDF.clone()), args: vec![lit("month"), lit(1), flat_col("b_utc"), lit("UTC")], }) .alias("month_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(DATE_ADD_TZ_UDF.clone())), + func: Arc::new(DATE_ADD_TZ_UDF.clone()), args: vec![ lit("month"), lit(1), @@ -1215,7 +1201,7 @@ mod test_date_add_tz { #[cfg(test)] mod test_utc_timestamp_to_str { use crate::*; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::str_to_utc_timestamp::STR_TO_UTC_TIMESTAMP_UDF; @@ -1241,9 +1227,7 @@ mod test_utc_timestamp_to_str { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - STR_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(STR_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("UTC")], }) .alias("b_utc"), @@ -1256,16 +1240,12 @@ mod test_utc_timestamp_to_str { flat_col("b"), flat_col("b_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - UTC_TIMESTAMP_TO_STR_UDF.clone(), - )), + func: Arc::new(UTC_TIMESTAMP_TO_STR_UDF.clone()), args: vec![flat_col("b_utc"), lit("UTC")], }) .alias("str_utc"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - UTC_TIMESTAMP_TO_STR_UDF.clone(), - )), + func: Arc::new(UTC_TIMESTAMP_TO_STR_UDF.clone()), args: vec![flat_col("b_utc"), lit("America/New_York")], }) .alias("str_nyc"), @@ -1308,7 +1288,7 @@ mod test_date_to_utc_timestamp { use arrow::array::{ArrayRef, Date32Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::date_to_utc_timestamp::DATE_TO_UTC_TIMESTAMP_UDF; @@ -1341,9 +1321,7 @@ mod test_date_to_utc_timestamp { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new( - DATE_TO_UTC_TIMESTAMP_UDF.clone(), - )), + func: Arc::new(DATE_TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), @@ -1383,7 +1361,7 @@ mod test_timestamp_to_utc_timestamp { use arrow::array::{ArrayRef, Int32Array, TimestampMillisecondArray}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use arrow::record_batch::RecordBatch; - use datafusion_expr::{expr, lit, Expr, ScalarFunctionDefinition}; + use datafusion_expr::{expr, lit, Expr}; use std::sync::Arc; use vegafusion_common::column::flat_col; use vegafusion_datafusion_udfs::udfs::datetime::to_utc_timestamp::TO_UTC_TIMESTAMP_UDF; @@ -1415,7 +1393,7 @@ mod test_timestamp_to_utc_timestamp { flat_col("a"), flat_col("b"), Expr::ScalarFunction(expr::ScalarFunction { - func_def: ScalarFunctionDefinition::UDF(Arc::new(TO_UTC_TIMESTAMP_UDF.clone())), + func: Arc::new(TO_UTC_TIMESTAMP_UDF.clone()), args: vec![flat_col("b"), lit("America/New_York")], }) .alias("b_utc"), From 10ca5982df5b01aa00f123b264f3eaf6a7990e87 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 11 Aug 2024 07:12:31 -0400 Subject: [PATCH 04/28] empty array now Int64 --- vegafusion-runtime/src/expression/compiler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 909ae81e..ef0ef02f 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -407,7 +407,7 @@ mod test_compile { let result_value = result_expr.eval_to_scalar().unwrap(); let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array( - &DataType::Null, + &DataType::Int64, )))); println!("value: {result_value:?}"); From 6b049ffb78dff589d47c48cdbcaf21a046a84894 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 11 Aug 2024 08:15:03 -0400 Subject: [PATCH 05/28] Fix timezone handling --- Cargo.lock | 47 ++++++++----------- Cargo.toml | 3 +- vegafusion-datafusion-udfs/Cargo.toml | 4 +- vegafusion-runtime/Cargo.toml | 4 +- .../src/dialect/transforms/date_add_tz.rs | 8 +--- 5 files changed, 29 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a6356f8..432f1659 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,7 +202,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "chrono-tz 0.9.0", + "chrono-tz", "half", "hashbrown 0.14.3", "num", @@ -885,9 +885,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -898,17 +898,6 @@ dependencies = [ "windows-targets 0.52.3", ] -[[package]] -name = "chrono-tz" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" -dependencies = [ - "chrono", - "chrono-tz-build 0.2.1", - "phf", -] - [[package]] name = "chrono-tz" version = "0.9.0" @@ -916,19 +905,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono", - "chrono-tz-build 0.3.0", - "phf", -] - -[[package]] -name = "chrono-tz-build" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" -dependencies = [ - "parse-zoneinfo", + "chrono-tz-build", "phf", - "phf_codegen", + "uncased", ] [[package]] @@ -940,6 +919,8 @@ dependencies = [ "parse-zoneinfo", "phf", "phf_codegen", + "regex", + "uncased", ] [[package]] @@ -3107,6 +3088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", + "uncased", ] [[package]] @@ -4755,6 +4737,15 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.7.0" @@ -4909,7 +4900,7 @@ name = "vegafusion-datafusion-udfs" version = "1.6.9" dependencies = [ "chrono", - "chrono-tz 0.8.6", + "chrono-tz", "datafusion-functions", "datafusion-physical-expr", "lazy_static", @@ -4965,7 +4956,7 @@ dependencies = [ "base64 0.21.7", "bytes", "chrono", - "chrono-tz 0.8.6", + "chrono-tz", "criterion", "datafusion-common", "datafusion-expr", diff --git a/Cargo.toml b/Cargo.toml index 2e60f7be..20e90548 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,8 @@ members = [ [workspace.dependencies] arrow = { version = "52.0.0", default-features = false } sqlparser = { version = "0.47.0" } -chrono = { version = "0.4.31", default-features = false } +chrono = { version = "0.4.35", default-features = false } +chrono-tz = {version = "0.9.0", features=["case-insensitive", "filter-by-regex"] } reqwest = { version = "0.11.22", default-features = false } tokio = { version = "1.36.0" } pyo3 = { version = "0.21.1" } diff --git a/vegafusion-datafusion-udfs/Cargo.toml b/vegafusion-datafusion-udfs/Cargo.toml index db54cbd8..5867c8d8 100644 --- a/vegafusion-datafusion-udfs/Cargo.toml +++ b/vegafusion-datafusion-udfs/Cargo.toml @@ -8,12 +8,14 @@ description = "Custom DataFusion UDFs used by VegaFusion" [dependencies] ordered-float = "3.6.0" lazy_static = "^1.4.0" -chrono-tz = "0.8.1" regex = "^1.5.5" [dependencies.chrono] workspace = true +[dependencies.chrono-tz] +workspace = true + [dependencies.vegafusion-common] path = "../vegafusion-common" version = "1.6.9" diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index 07f3c3fc..874d9d97 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -17,7 +17,6 @@ protobuf-src = [ "vegafusion-core/protobuf-src",] regex = "^1.5.5" lazy_static = "^1.4.0" serde_json = "1.0.91" -chrono-tz = "0.8.1" num-traits = "0.2.15" itertools = "0.11.0" float-cmp = "0.9.0" @@ -53,6 +52,9 @@ features = [ "aws",] [dependencies.chrono] workspace = true +[dependencies.chrono-tz] +workspace = true + [dependencies.prost] workspace = true diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index 24aa868e..dc4b7ff9 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -69,9 +69,7 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { args: FunctionArguments::List(FunctionArgumentList { args: vec![ SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.to_string()), - ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), ], duplicate_treatment: None, clauses: vec![], @@ -97,9 +95,7 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { args: FunctionArguments::List(FunctionArgumentList { args: vec![ SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(time_zone.to_string()), - ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_zone.clone())), ], duplicate_treatment: None, clauses: vec![], From 92c071556b9ded843facd3b5afbed4c4d44a9469 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 11 Aug 2024 16:43:56 -0400 Subject: [PATCH 06/28] fix deprecations --- vegafusion-common/src/data/json_writer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index 6138e29e..d731405e 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -164,7 +164,7 @@ macro_rules! set_temporal_column_as_millis_by_array_type { .for_each(|(i, row)| { if !arr.is_null(i) { if let Some(v) = arr.$cast_fn(i) { - row.insert($col_name.to_string(), v.timestamp_millis().into()); + row.insert($col_name.to_string(), v.and_utc().timestamp_millis().into()); } else { row.insert($col_name.to_string(), Value::Null); } @@ -811,6 +811,7 @@ mod tests { let ts_millis = ts_string .parse::() .unwrap() + .and_utc() .timestamp_millis(); let arr_date32 = Date32Array::from(vec![ From 0491d5e7abab44a64e132946e2fce63bc6464798 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 11 Aug 2024 17:09:06 -0400 Subject: [PATCH 07/28] fix deprecations --- vegafusion-common/src/data/json_writer.rs | 1 + .../src/udfs/datetime/date_to_utc_timestamp.rs | 7 ++++--- .../src/udfs/datetime/format_timestamp.rs | 10 +++++----- .../src/udfs/datetime/from_utc_timestamp.rs | 9 ++++----- .../src/udfs/datetime/to_utc_timestamp.rs | 6 +++--- .../src/udfs/datetime/utc_timestamp_to_str.rs | 12 +++++------- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/vegafusion-common/src/data/json_writer.rs b/vegafusion-common/src/data/json_writer.rs index d731405e..264972c3 100644 --- a/vegafusion-common/src/data/json_writer.rs +++ b/vegafusion-common/src/data/json_writer.rs @@ -758,6 +758,7 @@ mod tests { let ts_nanos = ts_string .parse::() .unwrap() + .and_utc() .timestamp_nanos_opt() .unwrap(); let ts_micros = ts_nanos / 1000; diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs index 28b98a3d..e08df159 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs @@ -1,4 +1,4 @@ -use chrono::{NaiveDateTime, TimeZone}; +use chrono::{DateTime, TimeZone}; use std::any::Any; use std::str::FromStr; use std::sync::Arc; @@ -78,8 +78,9 @@ impl ScalarUDFImpl for DateToUtcTimestampUDF { // Build naive datetime for time let seconds = (v as i64) * s_per_day; let nanoseconds = 0_u32; - let naive_local_datetime = NaiveDateTime::from_timestamp_opt(seconds, nanoseconds) - .expect("invalid or out-of-range datetime"); + let naive_local_datetime = DateTime::from_timestamp(seconds, nanoseconds) + .expect("invalid or out-of-range datetime") + .naive_utc(); // Compute UTC date time when naive date time is interpreted in the provided timezone let local_datetime = tz diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs index e020f521..f1d591c7 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs @@ -1,5 +1,5 @@ use crate::udfs::datetime::to_utc_timestamp::to_timestamp_ms; -use chrono::NaiveDateTime; +use chrono::DateTime; use std::any::Any; use std::sync::Arc; use vegafusion_common::datafusion_expr::ScalarUDFImpl; @@ -100,16 +100,16 @@ impl ScalarUDFImpl for FormatTimestampUDF { let formatted = Arc::new(StringArray::from_iter(utc_millis_array.iter().map( |utc_millis| { - utc_millis.map(|utc_millis| { + utc_millis.and_then(|utc_millis| { // Load as UTC datetime let utc_seconds = utc_millis / 1_000; let utc_nanos = (utc_millis % 1_000 * 1_000_000) as u32; - let naive_datetime = NaiveDateTime::from_timestamp_opt(utc_seconds, utc_nanos) - .expect("invalid or out-of-range datetime"); + let naive_datetime = + DateTime::from_timestamp(utc_seconds, utc_nanos)?.naive_utc(); // Format as string let formatted = naive_datetime.format(&format_str); - formatted.to_string() + Some(formatted.to_string()) }) }, ))) as ArrayRef; diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs index 974e1902..ca154c10 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs @@ -1,4 +1,4 @@ -use chrono::NaiveDateTime; +use chrono::DateTime; use chrono::TimeZone; use chrono_tz::Tz; use std::any::Any; @@ -113,20 +113,19 @@ pub fn from_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result>>(), diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs index def4af37..01c74a5d 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs @@ -1,5 +1,5 @@ -use chrono::TimeZone; -use chrono::{NaiveDateTime, Timelike}; +use chrono::Timelike; +use chrono::{DateTime, TimeZone}; use chrono_tz::Tz; use std::any::Any; use std::str::FromStr; @@ -107,7 +107,7 @@ pub fn to_utc_timestamp(timestamp_array: ArrayRef, tz: Tz) -> Result Date: Sun, 11 Aug 2024 17:09:25 -0400 Subject: [PATCH 08/28] Don't expect pandas to be imported under altair --- .../tests/altair_mocks/bar/diverging_stacked/mock.py | 3 ++- .../tests/altair_mocks/casestudy/falkensee/mock.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/vegafusion/tests/altair_mocks/bar/diverging_stacked/mock.py b/python/vegafusion/tests/altair_mocks/bar/diverging_stacked/mock.py index 8fbb61bd..7e1bd976 100644 --- a/python/vegafusion/tests/altair_mocks/bar/diverging_stacked/mock.py +++ b/python/vegafusion/tests/altair_mocks/bar/diverging_stacked/mock.py @@ -1,8 +1,9 @@ # https://altair-viz.github.io/gallery/diverging_stacked_bar_chart.html import altair as alt +import pandas as pd -source = alt.pd.DataFrame([ +source = pd.DataFrame([ { "question": "Question 1", "type": "Strongly disagree", diff --git a/python/vegafusion/tests/altair_mocks/casestudy/falkensee/mock.py b/python/vegafusion/tests/altair_mocks/casestudy/falkensee/mock.py index 13c72703..1dc85856 100644 --- a/python/vegafusion/tests/altair_mocks/casestudy/falkensee/mock.py +++ b/python/vegafusion/tests/altair_mocks/casestudy/falkensee/mock.py @@ -1,6 +1,7 @@ # https://altair-viz.github.io/gallery/falkensee.html import altair as alt +import pandas as pd source = [ {"year": "1875", "population": 1309}, @@ -55,8 +56,8 @@ }] -source = alt.pd.DataFrame(source) -source2 = alt.pd.DataFrame(source2) +source = pd.DataFrame(source) +source2 = pd.DataFrame(source2) line = alt.Chart(source).mark_line(color='#333').encode( From 035de06e0472b58d43f52ce56ad76b7ea27efb76 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 11 Aug 2024 18:49:00 -0400 Subject: [PATCH 09/28] update to datafusion 40 --- Cargo.lock | 134 ++++++++---------- Cargo.toml | 22 +-- vegafusion-common/src/data/scalar.rs | 4 +- vegafusion-common/src/data/table.rs | 8 +- vegafusion-datafusion-udfs/src/udafs/mod.rs | 2 +- .../data/vl_selection_resolve.rs | 1 + .../src/expression/compiler/mod.rs | 36 +++-- vegafusion-runtime/src/transform/aggregate.rs | 20 ++- vegafusion-runtime/src/transform/bin.rs | 1 + vegafusion-runtime/src/transform/extent.rs | 1 + vegafusion-runtime/src/transform/window.rs | 30 ++-- .../tests/test_vegajs_runtime.rs | 3 +- vegafusion-sql/Cargo.toml | 3 + vegafusion-sql/src/compile/function_arg.rs | 14 +- vegafusion-sql/src/compile/scalar.rs | 9 ++ .../connection/datafusion_py_datasource.rs | 4 + vegafusion-sql/src/dataframe/mod.rs | 12 +- vegafusion-sql/tests/test_aggregate.rs | 19 ++- vegafusion-sql/tests/test_joinaggregate.rs | 5 +- vegafusion-sql/tests/test_window.rs | 36 +++-- 20 files changed, 193 insertions(+), 171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 432f1659..a9662702 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,7 +204,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "num", ] @@ -1230,7 +1230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core 0.9.9", @@ -1238,9 +1238,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" +checksum = "ab9d55a9cd2634818953809f75ebe5248b00dd43c3227efb2a51a2d5feaad54e" dependencies = [ "ahash", "arrow", @@ -1269,12 +1269,12 @@ dependencies = [ "futures", "glob", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indexmap 2.2.3", "itertools 0.12.1", "log", "num_cpus", - "object_store 0.10.2", + "object_store", "parking_lot 0.12.1", "parquet", "paste", @@ -1292,9 +1292,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" +checksum = "def66b642959e7f96f5d2da22e1f43d3bd35598f821e5ce351a0553e0f1b7367" dependencies = [ "ahash", "arrow", @@ -1303,11 +1303,11 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "instant", "libc", "num_cpus", - "object_store 0.10.2", + "object_store", "parquet", "pyo3", "sqlparser", @@ -1315,18 +1315,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" +checksum = "f104bb9cb44c06c9badf8a0d7e0855e5f7fa5e395b887d7f835e8a9457dc1352" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" +checksum = "2ac0fd8b5d80bbca3fc3b6f40da4e9f6907354824ec3b18bbd83fee8cf5c3c3e" dependencies = [ "arrow", "chrono", @@ -1334,9 +1334,9 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "log", - "object_store 0.10.2", + "object_store", "parking_lot 0.12.1", "rand", "tempfile", @@ -1345,9 +1345,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" +checksum = "2103d2cc16fb11ef1fa993a6cac57ed5cb028601db4b97566c90e5fa77aa1e68" dependencies = [ "ahash", "arrow", @@ -1364,9 +1364,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" +checksum = "a369332afd0ef5bd565f6db2139fb9f1dfdd0afa75a7f70f000b74208d76994f" dependencies = [ "arrow", "base64 0.22.1", @@ -1376,8 +1376,7 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "hex", "itertools 0.12.1", "log", @@ -1391,9 +1390,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" +checksum = "92718db1aff70c47e5abf9fc975768530097059e5db7c7b78cd64b5e9a11fc77" dependencies = [ "ahash", "arrow", @@ -1409,9 +1408,9 @@ dependencies = [ [[package]] name = "datafusion-functions-array" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" +checksum = "30bb80f46ff3dcf4bb4510209c2ba9b8ce1b716ac8b7bf70c6bf7dca6260c831" dependencies = [ "arrow", "arrow-array", @@ -1422,6 +1421,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "itertools 0.12.1", "log", "paste", @@ -1429,9 +1429,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" +checksum = "82f34692011bec4fdd6fc18c264bf8037b8625d801e6dd8f5111af15cb6d71d3" dependencies = [ "arrow", "async-trait", @@ -1439,18 +1439,19 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indexmap 2.2.3", "itertools 0.12.1", "log", + "paste", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" +checksum = "45538630defedb553771434a437f7ca8f04b9b3e834344aafacecb27dc65d5e5" dependencies = [ "ahash", "arrow", @@ -1464,10 +1465,9 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", "datafusion-physical-expr-common", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "hex", "indexmap 2.2.3", "itertools 0.12.1", @@ -1479,21 +1479,23 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" +checksum = "9d8a72b0ca908e074aaeca52c14ddf5c28d22361e9cb6bc79bb733cd6661b536" dependencies = [ + "ahash", "arrow", "datafusion-common", "datafusion-expr", + "hashbrown 0.14.5", "rand", ] [[package]] name = "datafusion-physical-plan" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" +checksum = "b504eae6107a342775e22e323e9103f7f42db593ec6103b28605b7b7b1405c4a" dependencies = [ "ahash", "arrow", @@ -1512,7 +1514,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indexmap 2.2.3", "itertools 0.12.1", "log", @@ -1525,9 +1527,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8eb8d706c73f01c0e2630c64ffc61c33831b064a813ec08a3e094dc3190c0f" +checksum = "6a38d1e3d26fcb5a7de58b068d4f6a2eff20663a6d10ad1b45c6222505409003" dependencies = [ "arrow", "chrono", @@ -1535,28 +1537,28 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-proto-common", - "object_store 0.10.2", + "object_store", "prost", ] [[package]] name = "datafusion-proto-common" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7115772d326eeb78a1c77c974e7753b1538e1747675cb6b428058b654b31c" +checksum = "cad4529d59ebcc88f9d717b3b83cab01b7b6adee3f9944deab966be1886414a3" dependencies = [ "arrow", "chrono", "datafusion-common", - "object_store 0.9.1", + "object_store", "prost", ] [[package]] name = "datafusion-sql" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" +checksum = "e5db33f323f41b95ae201318ba654a9bf11113e58a51a1dff977b1a836d3d889" dependencies = [ "arrow", "arrow-array", @@ -2020,9 +2022,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -2321,7 +2323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -2611,7 +2613,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -2838,27 +2840,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object_store" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "humantime", - "itertools 0.12.1", - "parking_lot 0.12.1", - "percent-encoding", - "snafu", - "tokio", - "tracing", - "url", - "walkdir", -] - [[package]] name = "object_store" version = "0.10.2" @@ -3006,11 +2987,11 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lz4_flex", "num", "num-bigint", - "object_store 0.10.2", + "object_store", "paste", "seq-macro", "snap", @@ -4845,7 +4826,7 @@ dependencies = [ "datafusion-proto", "datafusion-proto-common", "jni", - "object_store 0.10.2", + "object_store", "pyo3", "serde_json", "sqlparser", @@ -4976,7 +4957,7 @@ dependencies = [ "log", "lru", "num-traits", - "object_store 0.10.2", + "object_store", "ordered-float 3.9.2", "pixelmatch", "prost", @@ -5037,10 +5018,11 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "deterministic-hash", "lazy_static", "log", - "object_store 0.10.2", + "object_store", "pyo3", "regex", "reqwest 0.11.24", diff --git a/Cargo.toml b/Cargo.toml index 20e90548..20f2d718 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ ] [workspace.dependencies] -arrow = { version = "52.0.0", default-features = false } +arrow = { version = "52.1.0", default-features = false } sqlparser = { version = "0.47.0" } chrono = { version = "0.4.35", default-features = false } chrono-tz = {version = "0.9.0", features=["case-insensitive", "filter-by-regex"] } @@ -27,34 +27,34 @@ prost-types = { version = "0.12.3" } object_store = { version= "0.10.1" } [workspace.dependencies.datafusion] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-common] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-expr] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-proto] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-proto-common] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-physical-expr] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-optimizer] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-functions] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-functions-array] -version = "39.0.0" +version = "40.0.0" [workspace.dependencies.datafusion-functions-aggregate] -version = "39.0.0" +version = "40.0.0" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 11bd551a..1d87d2d4 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -68,14 +68,14 @@ impl ScalarValueHelpers for ScalarValue { } Value::Array(elements) => { let array: ListArray = if elements.is_empty() { - array_into_list_array(Arc::new(new_empty_array(&DataType::Float64))) + array_into_list_array(Arc::new(new_empty_array(&DataType::Float64)), true) } else { let elements: Vec<_> = elements .iter() .map(ScalarValue::from_json) .collect::>>()?; - array_into_list_array(ScalarValue::iter_to_array(elements)?) + array_into_list_array(ScalarValue::iter_to_array(elements)?, true) }; ScalarValue::List(Arc::new(array)) diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index 8a0ce9b4..bdab2141 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -185,10 +185,14 @@ impl VegaFusionTable { if self.num_rows() == 0 { // Return empty list with (arbitrary) Float64 type let array = Arc::new(new_empty_array(&DataType::Float64)); - return Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))); + return Ok(ScalarValue::List(Arc::new(array_into_list_array( + array, true, + )))); } let array = Arc::new(StructArray::from(self.to_record_batch()?)) as ArrayRef; - Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))) + Ok(ScalarValue::List(Arc::new(array_into_list_array( + array, true, + )))) } #[cfg(feature = "json")] diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index f9dbfe0d..2c3b915d 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -19,7 +19,7 @@ pub(crate) struct PercentileContAccumulator { impl Accumulator for PercentileContAccumulator { fn state(&mut self) -> Result, DataFusionError> { - let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type); + let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type, true); Ok(vec![ScalarValue::List(state)]) } diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs index cdfb6143..c6e4a1b1 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs @@ -119,6 +119,7 @@ pub fn vl_selection_resolve_fn( // Turn values into a scalar list let values = ScalarValue::List(Arc::new(array_into_list_array( ScalarValue::iter_to_array(values)?, + true, ))); Ok((name, values)) }) diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index ef0ef02f..04e3d0b3 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -388,9 +388,10 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List(Arc::new(array_into_list_array(Arc::new( - Float64Array::from(vec![1.0, 2.0, 3.0]), - )))); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array( + Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])), + true, + ))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -406,9 +407,10 @@ mod test_compile { assert_eq!(result_expr, expected_expr); let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array( - &DataType::Int64, - )))); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array( + new_empty_array(&DataType::Int64), + true, + ))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -432,17 +434,21 @@ mod test_compile { let result_value = result_expr.eval_to_scalar().unwrap(); let expected_value = ScalarValue::List(Arc::new(array_into_list_array( ScalarValue::iter_to_array(vec![ - ScalarValue::List(Arc::new(array_into_list_array(Arc::new( - Float64Array::from(vec![1.0, 2.0]), - )))), - ScalarValue::List(Arc::new(array_into_list_array(Arc::new( - Float64Array::from(vec![3.0, 4.0]), - )))), - ScalarValue::List(Arc::new(array_into_list_array(Arc::new( - Float64Array::from(vec![5.0, 6.0]), - )))), + ScalarValue::List(Arc::new(array_into_list_array( + Arc::new(Float64Array::from(vec![1.0, 2.0])), + true, + ))), + ScalarValue::List(Arc::new(array_into_list_array( + Arc::new(Float64Array::from(vec![3.0, 4.0])), + true, + ))), + ScalarValue::List(Arc::new(array_into_list_array( + Arc::new(Float64Array::from(vec![5.0, 6.0])), + true, + ))), ]) .unwrap(), + true, ))); println!("value: {result_value:?}"); diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index 346101af..bce4d48c 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -1,15 +1,17 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; -use datafusion_expr::{avg, count, count_distinct, lit, max, min, sum, Expr}; +use datafusion_expr::{lit, max, min, Expr}; use datafusion_functions_aggregate::median::median_udaf; -use datafusion_functions_aggregate::variance::var_samp_udaf; +use datafusion_functions_aggregate::variance::{var_pop_udaf, var_samp_udaf}; use sqlparser::ast::NullTreatment; use std::collections::HashMap; use async_trait::async_trait; +use datafusion_expr::expr; use datafusion_expr::expr::AggregateFunctionDefinition; -use datafusion_expr::{aggregate_function, expr}; +use datafusion_functions_aggregate::expr_fn::{avg, count, count_distinct, sum}; +use datafusion_functions_aggregate::stddev::{stddev_pop_udaf, stddev_udaf}; use std::sync::Arc; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; @@ -182,9 +184,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Variancep => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - aggregate_function::AggregateFunction::VariancePop, - ), + func_def: AggregateFunctionDefinition::UDF(var_pop_udaf()), distinct: false, args: vec![numeric_column()?], filter: None, @@ -192,9 +192,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Stdev => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - aggregate_function::AggregateFunction::Stddev, - ), + func_def: AggregateFunctionDefinition::UDF(stddev_udaf()), distinct: false, args: vec![numeric_column()?], filter: None, @@ -202,9 +200,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Stdevp => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - aggregate_function::AggregateFunction::StddevPop, - ), + func_def: AggregateFunctionDefinition::UDF(stddev_pop_udaf()), distinct: false, args: vec![numeric_column()?], filter: None, diff --git a/vegafusion-runtime/src/transform/bin.rs b/vegafusion-runtime/src/transform/bin.rs index 834e56a3..55ee6708 100644 --- a/vegafusion-runtime/src/transform/bin.rs +++ b/vegafusion-runtime/src/transform/bin.rs @@ -126,6 +126,7 @@ fn compute_output_value(bin_tx: &Bin, start: f64, stop: f64, step: f64) -> Optio fname.insert_str(0, "bin_"); let fields = ScalarValue::List(Arc::new(array_into_list_array( ScalarValue::iter_to_array(vec![ScalarValue::from(bin_tx.field.as_str())]).ok()?, + true, ))); if bin_tx.signal.is_some() { diff --git a/vegafusion-runtime/src/transform/extent.rs b/vegafusion-runtime/src/transform/extent.rs index 2458afdf..bfd9fc77 100644 --- a/vegafusion-runtime/src/transform/extent.rs +++ b/vegafusion-runtime/src/transform/extent.rs @@ -63,6 +63,7 @@ fn extract_extent_list(table: &VegaFusionTable) -> Result { // Build two-element list of the extents let extent_list = TaskValue::Scalar(ScalarValue::List(Arc::new(array_into_list_array( ScalarValue::iter_to_array(vec![min_val_scalar, max_val_scalar])?, + true, )))); Ok(extent_list) } diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index 45db29c3..35a7b85d 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use datafusion_common::ScalarValue; use datafusion_expr::{aggregate_function, expr, lit, Expr, WindowFrame, WindowFunctionDefinition}; -use datafusion_functions_aggregate::variance::var_samp_udaf; +use datafusion_functions_aggregate::variance::{var_pop_udaf, var_samp_udaf}; use sqlparser::ast::NullTreatment; use std::sync::Arc; use vegafusion_core::error::Result; @@ -13,7 +13,11 @@ use vegafusion_core::proto::gen::transforms::{ }; use vegafusion_core::task_graph::task_value::TaskValue; +use datafusion_expr::test::function_stub::count_udaf; use datafusion_expr::{BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits}; +use datafusion_functions_aggregate::average::avg_udaf; +use datafusion_functions_aggregate::stddev::{stddev_pop_udaf, stddev_udaf}; +use datafusion_functions_aggregate::sum::sum_udaf; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::ORDER_COL; use vegafusion_common::datatypes::to_numeric; @@ -113,21 +117,15 @@ impl TransformTrait for Window { use AggregateOp::*; match op { Count => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::Count, - ), + WindowFunctionDefinition::AggregateUDF(count_udaf()), vec![lit(true)], ), Sum => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::Sum, - ), + WindowFunctionDefinition::AggregateUDF(sum_udaf()), vec![numeric_field()?], ), Mean | Average => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::Avg, - ), + WindowFunctionDefinition::AggregateUDF(avg_udaf()), vec![numeric_field()?], ), Min => ( @@ -147,21 +145,15 @@ impl TransformTrait for Window { vec![numeric_field()?], ), Variancep => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::VariancePop, - ), + WindowFunctionDefinition::AggregateUDF(var_pop_udaf()), vec![numeric_field()?], ), Stdev => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::Stddev, - ), + WindowFunctionDefinition::AggregateUDF(stddev_udaf()), vec![numeric_field()?], ), Stdevp => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::StddevPop, - ), + WindowFunctionDefinition::AggregateUDF(stddev_pop_udaf()), vec![numeric_field()?], ), // ArrayAgg only available on master right now diff --git a/vegafusion-runtime/tests/test_vegajs_runtime.rs b/vegafusion-runtime/tests/test_vegajs_runtime.rs index 85b79284..6af28166 100644 --- a/vegafusion-runtime/tests/test_vegajs_runtime.rs +++ b/vegafusion-runtime/tests/test_vegajs_runtime.rs @@ -124,7 +124,8 @@ fn test_evaluate_filter_transform() { assert_eq!( result_signals, vec![ScalarValue::List(Arc::new(array_into_list_array( - Arc::new(Float64Array::from(vec![6.0, 10.0])) + Arc::new(Float64Array::from(vec![6.0, 10.0])), + true )))] ); diff --git a/vegafusion-sql/Cargo.toml b/vegafusion-sql/Cargo.toml index f17a3a5c..a40fdfff 100644 --- a/vegafusion-sql/Cargo.toml +++ b/vegafusion-sql/Cargo.toml @@ -55,6 +55,9 @@ workspace = true [dependencies.datafusion-functions] workspace = true +[dependencies.datafusion-functions-aggregate] +workspace = true + [dependencies.datafusion] workspace = true optional = true diff --git a/vegafusion-sql/src/compile/function_arg.rs b/vegafusion-sql/src/compile/function_arg.rs index e257f866..d4040e29 100644 --- a/vegafusion-sql/src/compile/function_arg.rs +++ b/vegafusion-sql/src/compile/function_arg.rs @@ -23,10 +23,16 @@ impl ToSqlFunctionArg for Expr { Expr::Wildcard { qualifier: None } => SqlFunctionArgExpr::Wildcard, Expr::Wildcard { qualifier: Some(qualifier), - } => SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { - value: qualifier.clone(), - quote_style: None, - }])), + } => SqlFunctionArgExpr::QualifiedWildcard(ObjectName( + qualifier + .to_vec() + .into_iter() + .map(|value| Ident { + value, + quote_style: None, + }) + .collect(), + )), expr => SqlFunctionArgExpr::Expr(expr.to_sql(dialect, schema)?), }) } diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 7b20d20e..0d612b1e 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -297,6 +297,15 @@ impl ToSqlScalar for ScalarValue { ScalarValue::Union(_, _, _) => Err(VegaFusionError::internal( "Union cannot be converted to SQL", )), + ScalarValue::Utf8View(_) => Err(VegaFusionError::internal( + "Utf8View cannot be converted to SQL", + )), + ScalarValue::BinaryView(_) => Err(VegaFusionError::internal( + "BinaryView cannot be converted to SQL", + )), + ScalarValue::Map(_) => Err(VegaFusionError::internal( + "BinaryView cannot be converted to SQL", + )), } } } diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index 9b28a3ae..f8348c1f 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -162,4 +162,8 @@ impl ExecutionPlan for PyDatasourceExec { fn properties(&self) -> &PlanProperties { &self.plan_properties } + + fn name(&self) -> &str { + "py_datasource" + } } diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index 10224d0f..e0e75d53 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -9,10 +9,12 @@ use async_trait::async_trait; use datafusion_common::{Column, DFSchema, ScalarValue, TableReference}; use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{ - expr, is_null, lit, max, min, when, AggregateFunction, BuiltInWindowFunction, Expr, - ExprSchemable, WindowFrame, WindowFunctionDefinition, + expr, is_null, lit, max, min, when, BuiltInWindowFunction, Expr, ExprSchemable, WindowFrame, + WindowFunctionDefinition, }; use datafusion_functions::expr_fn::{abs, coalesce}; + +use datafusion_functions_aggregate::sum::sum_udaf; use sqlparser::ast::{ Cte, Expr as SqlExpr, GroupByExpr, Ident, NullTreatment, Query, Select, SelectItem, SetExpr, Statement, TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, @@ -908,7 +910,7 @@ impl SqlDataFrame { // Build window function to compute stacked value let window_expr = Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateUDF(sum_udaf()), args: vec![numeric_field.clone()], partition_by, order_by: orderby, @@ -983,7 +985,7 @@ impl SqlDataFrame { // Create aggregate for total of stack value let total_agg = Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Sum), + func_def: AggregateFunctionDefinition::UDF(sum_udaf()), args: vec![flat_col(stack_col_name)], distinct: false, filter: None, @@ -1034,7 +1036,7 @@ impl SqlDataFrame { // Build window function to compute cumulative sum of stack column let cumulative_field = "_cumulative"; - let fun = WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum); + let fun = WindowFunctionDefinition::AggregateUDF(sum_udaf()); let window_expr = Expr::WindowFunction(expr::WindowFunction { fun, args: vec![flat_col(stack_col_name)], diff --git a/vegafusion-sql/tests/test_aggregate.rs b/vegafusion-sql/tests/test_aggregate.rs index 18c42326..28a8f630 100644 --- a/vegafusion-sql/tests/test_aggregate.rs +++ b/vegafusion-sql/tests/test_aggregate.rs @@ -2,7 +2,7 @@ extern crate lazy_static; mod utils; -use datafusion_expr::{avg, count, expr, lit, max, min, sum, AggregateFunction, Expr}; +use datafusion_expr::{expr, lit, max, min, Expr}; use rstest::rstest; use rstest_reuse::{self, *}; use serde_json::json; @@ -14,6 +14,7 @@ use vegafusion_sql::dataframe::SqlDataFrame; #[cfg(test)] mod test_simple_aggs { use crate::*; + use datafusion_functions_aggregate::expr_fn::{avg, count, sum}; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -73,11 +74,12 @@ mod test_simple_aggs { mod test_median_agg { use crate::*; use datafusion_expr::expr::AggregateFunctionDefinition; + use datafusion_functions_aggregate::expr_fn::count; use vegafusion_common::column::flat_col; #[apply(dialect_names)] async fn test(dialect_name: &str) { - use datafusion::functions_aggregate::median::median_udaf; + use datafusion_functions_aggregate::median::median_udaf; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); @@ -127,8 +129,9 @@ mod test_median_agg { #[cfg(test)] mod test_variance_aggs { use crate::*; - use datafusion::functions_aggregate::variance::var_samp_udaf; use datafusion_expr::expr::AggregateFunctionDefinition; + use datafusion_functions_aggregate::stddev::{stddev_pop_udaf, stddev_udaf}; + use datafusion_functions_aggregate::variance::{var_pop_udaf, var_samp_udaf}; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -154,7 +157,7 @@ mod test_variance_aggs { vec![flat_col("b")], vec![ round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Stddev), + func_def: AggregateFunctionDefinition::UDF(stddev_udaf()), args: vec![flat_col("a")], distinct: false, filter: None, @@ -165,9 +168,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - AggregateFunction::StddevPop, - ), + func_def: AggregateFunctionDefinition::UDF(stddev_pop_udaf()), args: vec![flat_col("a")], distinct: false, filter: None, @@ -189,9 +190,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("var_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - AggregateFunction::VariancePop, - ), + func_def: AggregateFunctionDefinition::UDF(var_pop_udaf()), args: vec![flat_col("a")], distinct: false, filter: None, diff --git a/vegafusion-sql/tests/test_joinaggregate.rs b/vegafusion-sql/tests/test_joinaggregate.rs index 30512910..3ae18708 100644 --- a/vegafusion-sql/tests/test_joinaggregate.rs +++ b/vegafusion-sql/tests/test_joinaggregate.rs @@ -2,7 +2,7 @@ extern crate lazy_static; mod utils; -use datafusion_expr::{avg, count, expr, max, min, sum, Expr}; +use datafusion_expr::{expr, max, min, Expr}; use rstest::rstest; use rstest_reuse::{self, *}; use serde_json::json; @@ -13,6 +13,7 @@ use vegafusion_sql::dataframe::SqlDataFrame; #[cfg(test)] mod test_simple_aggs { use crate::*; + use datafusion_functions_aggregate::expr_fn::{avg, count, sum}; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -74,6 +75,8 @@ mod test_simple_aggs { #[cfg(test)] mod test_simple_aggs_no_grouping { use crate::*; + use datafusion_functions_aggregate::expr_fn::{avg, count}; + use datafusion_functions_aggregate::sum::sum; use vegafusion_common::column::flat_col; #[apply(dialect_names)] diff --git a/vegafusion-sql/tests/test_window.rs b/vegafusion-sql/tests/test_window.rs index 278c55d6..5a3f9b90 100644 --- a/vegafusion-sql/tests/test_window.rs +++ b/vegafusion-sql/tests/test_window.rs @@ -18,6 +18,9 @@ use vegafusion_sql::dataframe::SqlDataFrame; mod test_simple_aggs_unbounded { use crate::*; use datafusion_expr::WindowFunctionDefinition; + use datafusion_functions_aggregate::average::avg_udaf; + use datafusion_functions_aggregate::count::count_udaf; + use datafusion_functions_aggregate::sum::sum_udaf; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -49,7 +52,7 @@ mod test_simple_aggs_unbounded { flat_col("b"), flat_col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateUDF(sum_udaf()), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -58,7 +61,7 @@ mod test_simple_aggs_unbounded { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), + fun: WindowFunctionDefinition::AggregateUDF(count_udaf()), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -67,7 +70,7 @@ mod test_simple_aggs_unbounded { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateUDF(avg_udaf()), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -117,7 +120,10 @@ mod test_simple_aggs_unbounded { #[cfg(test)] mod test_simple_aggs_unbounded_groups { use crate::*; + use datafusion_expr::test::function_stub::avg_udaf; use datafusion_expr::WindowFunctionDefinition; + use datafusion_functions_aggregate::count::count_udaf; + use datafusion_functions_aggregate::sum::sum_udaf; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -153,7 +159,7 @@ mod test_simple_aggs_unbounded_groups { flat_col("b"), flat_col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateUDF(sum_udaf()), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -162,7 +168,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), + fun: WindowFunctionDefinition::AggregateUDF(count_udaf()), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -171,7 +177,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateUDF(avg_udaf()), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -223,6 +229,9 @@ mod test_simple_aggs_unbounded_groups { mod test_simple_aggs_bounded { use crate::*; use datafusion_expr::WindowFunctionDefinition; + use datafusion_functions_aggregate::average::avg_udaf; + use datafusion_functions_aggregate::count::count_udaf; + use datafusion_functions_aggregate::sum::sum_udaf; use vegafusion_common::column::flat_col; #[apply(dialect_names)] @@ -258,7 +267,7 @@ mod test_simple_aggs_bounded { flat_col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateUDF(sum_udaf()), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -267,7 +276,7 @@ mod test_simple_aggs_bounded { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), + fun: WindowFunctionDefinition::AggregateUDF(count_udaf()), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -276,7 +285,7 @@ mod test_simple_aggs_bounded { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateUDF(avg_udaf()), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -328,6 +337,9 @@ mod test_simple_aggs_bounded { mod test_simple_aggs_bounded_groups { use crate::*; use datafusion_expr::WindowFunctionDefinition; + use datafusion_functions_aggregate::average::avg_udaf; + use datafusion_functions_aggregate::count::count_udaf; + use datafusion_functions_aggregate::sum::sum_udaf; #[apply(dialect_names)] async fn test(dialect_name: &str) { @@ -362,7 +374,7 @@ mod test_simple_aggs_bounded_groups { col("b"), col("c"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Sum), + fun: WindowFunctionDefinition::AggregateUDF(sum_udaf()), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -371,7 +383,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("sum_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Count), + fun: WindowFunctionDefinition::AggregateUDF(count_udaf()), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -380,7 +392,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("count_part_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Avg), + fun: WindowFunctionDefinition::AggregateUDF(avg_udaf()), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), From 9ad1ec643e686c2589714364265b36723b15388b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Aug 2024 06:56:43 -0400 Subject: [PATCH 10/28] update to datafusion 41 --- Cargo.lock | 105 ++++++++++++------ Cargo.toml | 26 ++--- vegafusion-runtime/Cargo.toml | 4 +- .../src/expression/compiler/array.rs | 2 +- .../builtin_functions/array/length.rs | 2 +- .../src/expression/compiler/mod.rs | 2 +- vegafusion-runtime/src/transform/aggregate.rs | 19 ++-- vegafusion-runtime/src/transform/extent.rs | 3 +- vegafusion-runtime/src/transform/pivot.rs | 3 +- vegafusion-runtime/src/transform/window.rs | 11 +- vegafusion-sql/src/compile/expr.rs | 29 ++++- vegafusion-sql/src/compile/order.rs | 1 + vegafusion-sql/src/compile/scalar.rs | 5 + .../src/connection/datafusion_conn.rs | 18 +-- .../connection/datafusion_py_datasource.rs | 3 +- vegafusion-sql/src/dataframe/mod.rs | 26 +++-- vegafusion-sql/src/dialect/mod.rs | 35 ++++++ .../src/dialect/transforms/date_add_tz.rs | 45 ++++++++ .../src/dialect/transforms/date_part_tz.rs | 40 +++++++ .../transforms/date_to_utc_timestamp.rs | 35 ++++++ .../src/dialect/transforms/date_trunc_tz.rs | 50 +++++++++ .../transforms/epoch_ms_to_utc_timestamp.rs | 40 +++++++ .../dialect/transforms/make_utc_timestamp.rs | 60 ++++++++++ .../transforms/str_to_utc_timestamp.rs | 35 ++++++ .../dialect/transforms/to_utc_timestamp.rs | 15 +++ .../transforms/utc_timestamp_to_epoch_ms.rs | 30 +++++ .../transforms/utc_timestamp_to_str.rs | 45 ++++++++ vegafusion-sql/tests/test_aggregate.rs | 16 +-- vegafusion-sql/tests/test_joinaggregate.rs | 6 +- vegafusion-sql/tests/test_window.rs | 23 ++-- 30 files changed, 612 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9662702..1c41afa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,11 +1225,12 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.3" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" dependencies = [ "cfg-if", + "crossbeam-utils", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -1238,9 +1239,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab9d55a9cd2634818953809f75ebe5248b00dd43c3227efb2a51a2d5feaad54e" +checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb" dependencies = [ "ahash", "arrow", @@ -1253,16 +1254,18 @@ dependencies = [ "bzip2", "chrono", "dashmap", + "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", - "datafusion-functions-array", + "datafusion-functions-nested", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-sql", "flate2", @@ -1290,11 +1293,25 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-catalog" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", +] + [[package]] name = "datafusion-common" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "def66b642959e7f96f5d2da22e1f43d3bd35598f821e5ce351a0553e0f1b7367" +checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c" dependencies = [ "ahash", "arrow", @@ -1315,18 +1332,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f104bb9cb44c06c9badf8a0d7e0855e5f7fa5e395b887d7f835e8a9457dc1352" +checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac0fd8b5d80bbca3fc3b6f40da4e9f6907354824ec3b18bbd83fee8cf5c3c3e" +checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" dependencies = [ "arrow", "chrono", @@ -1345,9 +1362,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2103d2cc16fb11ef1fa993a6cac57ed5cb028601db4b97566c90e5fa77aa1e68" +checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2" dependencies = [ "ahash", "arrow", @@ -1364,11 +1381,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a369332afd0ef5bd565f6db2139fb9f1dfdd0afa75a7f70f000b74208d76994f" +checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb" dependencies = [ "arrow", + "arrow-buffer", "base64 0.22.1", "blake2", "blake3", @@ -1390,9 +1408,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92718db1aff70c47e5abf9fc975768530097059e5db7c7b78cd64b5e9a11fc77" +checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb" dependencies = [ "ahash", "arrow", @@ -1407,10 +1425,10 @@ dependencies = [ ] [[package]] -name = "datafusion-functions-array" -version = "40.0.0" +name = "datafusion-functions-nested" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bb80f46ff3dcf4bb4510209c2ba9b8ce1b716ac8b7bf70c6bf7dca6260c831" +checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d" dependencies = [ "arrow", "arrow-array", @@ -1425,13 +1443,14 @@ dependencies = [ "itertools 0.12.1", "log", "paste", + "rand", ] [[package]] name = "datafusion-optimizer" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f34692011bec4fdd6fc18c264bf8037b8625d801e6dd8f5111af15cb6d71d3" +checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906" dependencies = [ "arrow", "async-trait", @@ -1449,9 +1468,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45538630defedb553771434a437f7ca8f04b9b3e834344aafacecb27dc65d5e5" +checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4" dependencies = [ "ahash", "arrow", @@ -1479,9 +1498,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d8a72b0ca908e074aaeca52c14ddf5c28d22361e9cb6bc79bb733cd6661b536" +checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6" dependencies = [ "ahash", "arrow", @@ -1491,11 +1510,23 @@ dependencies = [ "rand", ] +[[package]] +name = "datafusion-physical-optimizer" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0" +dependencies = [ + "datafusion-common", + "datafusion-execution", + "datafusion-physical-expr", + "datafusion-physical-plan", +] + [[package]] name = "datafusion-physical-plan" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b504eae6107a342775e22e323e9103f7f42db593ec6103b28605b7b7b1405c4a" +checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe" dependencies = [ "ahash", "arrow", @@ -1527,9 +1558,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a38d1e3d26fcb5a7de58b068d4f6a2eff20663a6d10ad1b45c6222505409003" +checksum = "cf1d25864c18178d0e51438648f5e0fa08417dbbc39b642c1752cbbb1013abf0" dependencies = [ "arrow", "chrono", @@ -1543,9 +1574,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad4529d59ebcc88f9d717b3b83cab01b7b6adee3f9944deab966be1886414a3" +checksum = "96a683253732334526b1cc5314a73a0f786803831f7e189ed3fe387ac50d7222" dependencies = [ "arrow", "chrono", @@ -1556,9 +1587,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5db33f323f41b95ae201318ba654a9bf11113e58a51a1dff977b1a836d3d889" +checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0" dependencies = [ "arrow", "arrow-array", @@ -4135,9 +4166,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.47.0" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" dependencies = [ "log", "sqlparser_derive", @@ -4943,7 +4974,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", - "datafusion-functions-array", + "datafusion-functions-nested", "datafusion-optimizer", "datafusion-physical-expr", "deterministic-hash", diff --git a/Cargo.toml b/Cargo.toml index 20f2d718..8b9ba406 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ members = [ ] [workspace.dependencies] -arrow = { version = "52.1.0", default-features = false } -sqlparser = { version = "0.47.0" } +arrow = { version = "52.2.0", default-features = false } +sqlparser = { version = "0.49.0" } chrono = { version = "0.4.35", default-features = false } chrono-tz = {version = "0.9.0", features=["case-insensitive", "filter-by-regex"] } reqwest = { version = "0.11.22", default-features = false } @@ -27,34 +27,34 @@ prost-types = { version = "0.12.3" } object_store = { version= "0.10.1" } [workspace.dependencies.datafusion] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-common] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-expr] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-proto] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-proto-common] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-physical-expr] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-optimizer] -version = "40.0.0" +version = "41.0.0" [workspace.dependencies.datafusion-functions] -version = "40.0.0" +version = "41.0.0" -[workspace.dependencies.datafusion-functions-array] -version = "40.0.0" +[workspace.dependencies.datafusion-functions-nested] +version = "41.0.0" [workspace.dependencies.datafusion-functions-aggregate] -version = "40.0.0" +version = "41.0.0" [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index 874d9d97..33e68ca6 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -105,10 +105,10 @@ workspace = true [dependencies.datafusion-functions] workspace = true -[dependencies.datafusion-functions-array] +[dependencies.datafusion-functions-aggregate] workspace = true -[dependencies.datafusion-functions-aggregate] +[dependencies.datafusion-functions-nested] workspace = true [dependencies.tokio] diff --git a/vegafusion-runtime/src/expression/compiler/array.rs b/vegafusion-runtime/src/expression/compiler/array.rs index 8766c2d4..0d3315ed 100644 --- a/vegafusion-runtime/src/expression/compiler/array.rs +++ b/vegafusion-runtime/src/expression/compiler/array.rs @@ -1,6 +1,6 @@ use crate::expression::compiler::{compile, config::CompilationConfig}; use datafusion_expr::Expr; -use datafusion_functions_array::expr_fn::make_array; +use datafusion_functions_nested::expr_fn::make_array; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_core::error::Result; use vegafusion_core::proto::gen::expression::ArrayExpression; diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs index 694104e8..1b029a2a 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/array/length.rs @@ -2,7 +2,7 @@ use datafusion_common::DFSchema; use datafusion_expr::{expr, Expr, ExprSchemable}; use datafusion_functions::unicode::expr_fn::character_length; -use datafusion_functions_array::length::array_length; +use datafusion_functions_nested::length::array_length; use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::error::{ResultWithContext, VegaFusionError}; diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 04e3d0b3..3f8a435e 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -61,7 +61,7 @@ mod test_compile { use crate::expression::compiler::config::CompilationConfig; use crate::expression::compiler::utils::ExprHelpers; use datafusion_functions::expr_fn::{coalesce, concat}; - use datafusion_functions_array::expr_fn::make_array; + use datafusion_functions_nested::expr_fn::make_array; use vegafusion_core::expression::parser::parse; use crate::task_graph::timezone::RuntimeTzConfig; diff --git a/vegafusion-runtime/src/transform/aggregate.rs b/vegafusion-runtime/src/transform/aggregate.rs index bce4d48c..7c278aa6 100644 --- a/vegafusion-runtime/src/transform/aggregate.rs +++ b/vegafusion-runtime/src/transform/aggregate.rs @@ -1,7 +1,7 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; -use datafusion_expr::{lit, max, min, Expr}; +use datafusion_expr::{lit, Expr}; use datafusion_functions_aggregate::median::median_udaf; use datafusion_functions_aggregate::variance::{var_pop_udaf, var_samp_udaf}; use sqlparser::ast::NullTreatment; @@ -9,8 +9,7 @@ use std::collections::HashMap; use async_trait::async_trait; use datafusion_expr::expr; -use datafusion_expr::expr::AggregateFunctionDefinition; -use datafusion_functions_aggregate::expr_fn::{avg, count, count_distinct, sum}; +use datafusion_functions_aggregate::expr_fn::{avg, count, count_distinct, max, min, sum}; use datafusion_functions_aggregate::stddev::{stddev_pop_udaf, stddev_udaf}; use std::sync::Arc; use vegafusion_common::column::{flat_col, unescaped_col}; @@ -168,7 +167,7 @@ pub fn make_agg_expr_for_col_expr( AggregateOp::Max => max(column), AggregateOp::Sum => sum(numeric_column()?), AggregateOp::Median => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(median_udaf()), + func: median_udaf(), distinct: false, args: vec![numeric_column()?], filter: None, @@ -176,7 +175,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Variance => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(var_samp_udaf()), + func: var_samp_udaf(), distinct: false, args: vec![numeric_column()?], filter: None, @@ -184,7 +183,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Variancep => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(var_pop_udaf()), + func: var_pop_udaf(), distinct: false, args: vec![numeric_column()?], filter: None, @@ -192,7 +191,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Stdev => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(stddev_udaf()), + func: stddev_udaf(), distinct: false, args: vec![numeric_column()?], filter: None, @@ -200,7 +199,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Stdevp => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(stddev_pop_udaf()), + func: stddev_pop_udaf(), distinct: false, args: vec![numeric_column()?], filter: None, @@ -230,7 +229,7 @@ pub fn make_agg_expr_for_col_expr( count_distinct(column) + max(missing) } AggregateOp::Q1 => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(Arc::new((*Q1_UDF).clone())), + func: Arc::new((*Q1_UDF).clone()), args: vec![numeric_column()?], distinct: false, filter: None, @@ -238,7 +237,7 @@ pub fn make_agg_expr_for_col_expr( null_treatment: Some(NullTreatment::IgnoreNulls), }), AggregateOp::Q3 => Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(Arc::new((*Q3_UDF).clone())), + func: Arc::new((*Q3_UDF).clone()), args: vec![numeric_column()?], distinct: false, filter: None, diff --git a/vegafusion-runtime/src/transform/extent.rs b/vegafusion-runtime/src/transform/extent.rs index bfd9fc77..c471b214 100644 --- a/vegafusion-runtime/src/transform/extent.rs +++ b/vegafusion-runtime/src/transform/extent.rs @@ -4,7 +4,8 @@ use async_trait::async_trait; use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::{max, min, Expr}; +use datafusion_expr::Expr; +use datafusion_functions_aggregate::expr_fn::{max, min}; use std::sync::Arc; use vegafusion_common::column::unescaped_col; use vegafusion_common::data::table::VegaFusionTable; diff --git a/vegafusion-runtime/src/transform/pivot.rs b/vegafusion-runtime/src/transform/pivot.rs index c53d098e..086112dd 100644 --- a/vegafusion-runtime/src/transform/pivot.rs +++ b/vegafusion-runtime/src/transform/pivot.rs @@ -2,8 +2,9 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::aggregate::make_agg_expr_for_col_expr; use crate::transform::TransformTrait; use async_trait::async_trait; -use datafusion_expr::{expr::Sort, lit, min, when, Expr}; +use datafusion_expr::{expr::Sort, lit, when, Expr}; use datafusion_functions::expr_fn::coalesce; +use datafusion_functions_aggregate::expr_fn::min; use std::sync::Arc; use vegafusion_common::arrow::array::StringArray; use vegafusion_common::arrow::datatypes::DataType; diff --git a/vegafusion-runtime/src/transform/window.rs b/vegafusion-runtime/src/transform/window.rs index 35a7b85d..4dc81c9e 100644 --- a/vegafusion-runtime/src/transform/window.rs +++ b/vegafusion-runtime/src/transform/window.rs @@ -3,7 +3,7 @@ use crate::transform::TransformTrait; use async_trait::async_trait; use datafusion_common::ScalarValue; -use datafusion_expr::{aggregate_function, expr, lit, Expr, WindowFrame, WindowFunctionDefinition}; +use datafusion_expr::{expr, lit, Expr, WindowFrame, WindowFunctionDefinition}; use datafusion_functions_aggregate::variance::{var_pop_udaf, var_samp_udaf}; use sqlparser::ast::NullTreatment; use std::sync::Arc; @@ -16,6 +16,7 @@ use vegafusion_core::task_graph::task_value::TaskValue; use datafusion_expr::test::function_stub::count_udaf; use datafusion_expr::{BuiltInWindowFunction, WindowFrameBound, WindowFrameUnits}; use datafusion_functions_aggregate::average::avg_udaf; +use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; use datafusion_functions_aggregate::stddev::{stddev_pop_udaf, stddev_udaf}; use datafusion_functions_aggregate::sum::sum_udaf; use vegafusion_common::column::{flat_col, unescaped_col}; @@ -129,15 +130,11 @@ impl TransformTrait for Window { vec![numeric_field()?], ), Min => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::Min, - ), + WindowFunctionDefinition::AggregateUDF(min_udaf()), vec![numeric_field()?], ), Max => ( - WindowFunctionDefinition::AggregateFunction( - aggregate_function::AggregateFunction::Max, - ), + WindowFunctionDefinition::AggregateUDF(max_udaf()), vec![numeric_field()?], ), Variance => ( diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 7431f74f..c39e9a2e 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -304,12 +304,12 @@ impl ToSqlExpr for Expr { translate_scalar_function(&fun_name, &fun.args, dialect, schema) } Expr::AggregateFunction(expr::AggregateFunction { - func_def, + func, args, distinct, .. }) => translate_aggregate_function( - &func_def.name().to_ascii_lowercase(), + &func.name().to_ascii_lowercase(), args.as_slice(), *distinct, dialect, @@ -325,7 +325,7 @@ impl ToSqlExpr for Expr { }) => { // Extract function name let (fun_name, supports_frame) = match fun { - WindowFunctionDefinition::AggregateFunction(agg) => { + WindowFunctionDefinition::AggregateUDF(agg) => { (agg.name().to_ascii_lowercase(), true) } WindowFunctionDefinition::BuiltInWindowFunction(win_fn) => { @@ -347,9 +347,6 @@ impl ToSqlExpr for Expr { (win_fn.to_string().to_ascii_lowercase(), supports_frame) } - WindowFunctionDefinition::AggregateUDF(udf) => { - (udf.name().to_ascii_lowercase(), true) - } WindowFunctionDefinition::WindowUDF(udf) => { (udf.name().to_ascii_lowercase(), true) } @@ -370,6 +367,11 @@ impl ToSqlExpr for Expr { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })); } UnorderedRowNumberMode::OrderByConstant => { @@ -461,6 +463,11 @@ impl ToSqlExpr for Expr { null_treatment: None, over: Some(over), within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }; Ok(SqlExpr::Function(sql_fun)) @@ -563,6 +570,11 @@ fn translate_scalar_function( null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } else if let Some(transformer) = dialect.scalar_transformers.get(fun_name) { // Supported through AST transformation @@ -600,6 +612,11 @@ fn translate_aggregate_function( null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } else if let Some(transformer) = dialect.aggregate_transformers.get(fun_name) { // Supported through AST transformation diff --git a/vegafusion-sql/src/compile/order.rs b/vegafusion-sql/src/compile/order.rs index d1c6f762..152a60fe 100644 --- a/vegafusion-sql/src/compile/order.rs +++ b/vegafusion-sql/src/compile/order.rs @@ -38,6 +38,7 @@ impl ToSqlOrderByExpr for Expr { })?, asc: Some(*asc), nulls_first, + with_fill: None, }) } _ => Err(VegaFusionError::internal( diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 0d612b1e..3314674a 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -209,6 +209,11 @@ impl ToSqlScalar for ScalarValue { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } ScalarValue::Date32(v) => date32_to_date(v, dialect), diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index 73554839..1ff8400e 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -7,11 +7,9 @@ use arrow::record_batch::RecordBatch; use datafusion::config::TableOptions; use datafusion::datasource::listing::ListingTableUrl; use datafusion::datasource::MemTable; -use datafusion::execution::context::SessionState; use datafusion::execution::options::{ArrowReadOptions, ReadOptions}; use datafusion::execution::runtime_env::RuntimeEnv; -use datafusion::optimizer::analyzer::inline_table_scan::InlineTableScan; -use datafusion::optimizer::analyzer::type_coercion::TypeCoercion; +use datafusion::execution::session_state::SessionStateBuilder; use datafusion::prelude::{ CsvReadOptions as DfCsvReadOptions, ParquetReadOptions, SessionConfig, SessionContext, }; @@ -483,20 +481,14 @@ pub fn make_request_client() -> ClientWithMiddleware { } pub fn make_datafusion_context() -> SessionContext { - // Work around issues: - // - https://github.com/apache/arrow-datafusion/issues/6386 - // - https://github.com/apache/arrow-datafusion/issues/6447 let mut config = SessionConfig::new(); let options = config.options_mut(); options.optimizer.skip_failed_rules = true; let runtime = Arc::new(RuntimeEnv::default()); - let session_state = SessionState::new_with_config_rt(config, runtime); - let session_state = session_state.with_analyzer_rules(vec![ - Arc::new(InlineTableScan::new()), - Arc::new(TypeCoercion::new()), - // Intentionally exclude the CountWildcardRule - // Arc::new(CountWildcardRule::new()), - ]); + let session_state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(runtime) + .build(); let ctx = SessionContext::new_with_state(session_state); diff --git a/vegafusion-sql/src/connection/datafusion_py_datasource.rs b/vegafusion-sql/src/connection/datafusion_py_datasource.rs index f8348c1f..496bdbef 100644 --- a/vegafusion-sql/src/connection/datafusion_py_datasource.rs +++ b/vegafusion-sql/src/connection/datafusion_py_datasource.rs @@ -2,7 +2,6 @@ use arrow::datatypes::{Schema, SchemaRef}; use arrow::pyarrow::FromPyArrow; use async_trait::async_trait; use datafusion::datasource::TableProvider; -use datafusion::execution::context::SessionState; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::memory::MemoryStream; @@ -65,7 +64,7 @@ impl TableProvider for PyDatasource { async fn scan( &self, - _state: &SessionState, + _state: &(dyn datafusion::catalog::Session), projection: Option<&Vec>, _filters: &[Expr], _limit: Option, diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index e0e75d53..9a5519ef 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -7,17 +7,18 @@ use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion_common::{Column, DFSchema, ScalarValue, TableReference}; -use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{ - expr, is_null, lit, max, min, when, BuiltInWindowFunction, Expr, ExprSchemable, WindowFrame, + expr, is_null, lit, when, BuiltInWindowFunction, Expr, ExprSchemable, WindowFrame, WindowFunctionDefinition, }; use datafusion_functions::expr_fn::{abs, coalesce}; +use datafusion_functions_aggregate::min_max::{max, min}; use datafusion_functions_aggregate::sum::sum_udaf; use sqlparser::ast::{ - Cte, Expr as SqlExpr, GroupByExpr, Ident, NullTreatment, Query, Select, SelectItem, SetExpr, - Statement, TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, + Cte, Expr as SqlExpr, GroupByExpr, Ident, NullTreatment, OrderBy, Query, Select, SelectItem, + SetExpr, Statement, TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, + With, }; use sqlparser::parser::Parser; use std::any::Any; @@ -284,13 +285,14 @@ impl SqlDataFrame { selection: None, from: Default::default(), lateral_views: Default::default(), - group_by: GroupByExpr::Expressions(Vec::new()), + group_by: GroupByExpr::Expressions(Vec::new(), Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), having: None, named_window: Default::default(), qualify: None, + prewhere: None, }); } @@ -331,6 +333,8 @@ impl SqlDataFrame { fetch: None, locks: Default::default(), for_clause: None, + settings: None, + format_clause: None, }; let (projection, table_alias) = if let ValuesMode::ValuesWithSelectColumnAliases { @@ -394,11 +398,12 @@ impl SqlDataFrame { }], lateral_views: Default::default(), selection: None, - group_by: GroupByExpr::Expressions(Vec::new()), + group_by: GroupByExpr::Expressions(Vec::new(), Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), having: None, + prewhere: None, qualify: None, named_window: Default::default(), value_table_mode: None, @@ -415,6 +420,8 @@ impl SqlDataFrame { fetch: None, locks: Default::default(), for_clause: None, + settings: None, + format_clause: None, } } }; @@ -473,7 +480,10 @@ impl SqlDataFrame { .iter() .map(|expr| expr.to_sql_order(self.dialect(), &self.schema_df()?)) .collect::>>()?; - query.order_by = sql_exprs; + query.order_by = Some(OrderBy { + exprs: sql_exprs, + interpolate: None, + }); if let Some(limit) = limit { query.limit = Some( lit(limit) @@ -985,7 +995,7 @@ impl SqlDataFrame { // Create aggregate for total of stack value let total_agg = Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(sum_udaf()), + func: sum_udaf(), args: vec![flat_col(stack_col_name)], distinct: false, filter: None, diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index 9ca6d67a..7af10aef 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -1760,6 +1760,11 @@ impl BinaryOperatorTransformer for ModulusOpToFunction { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -1807,6 +1812,11 @@ impl FunctionTransformer for RenameFunctionTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -1843,6 +1853,11 @@ impl FunctionTransformer for ExpWithPowFunctionTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -1888,6 +1903,11 @@ impl FunctionTransformer for CastArgsFunctionTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -1930,6 +1950,11 @@ impl FunctionTransformer for LogBaseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -1976,6 +2001,11 @@ impl FunctionTransformer for LogBaseWithLnTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let base_arg = FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Value( @@ -1995,6 +2025,11 @@ impl FunctionTransformer for LogBaseWithLnTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(SqlExpr::BinaryOp { diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index dc4b7ff9..1215ff4a 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -78,6 +78,11 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) } } @@ -104,6 +109,11 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) } } @@ -142,6 +152,11 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let date_time_field = part_to_date_time_field(&part)?; @@ -170,6 +185,11 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let timestamp_expr = SqlExpr::Function(SqlFunction { @@ -189,6 +209,11 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(timestamp_expr) @@ -277,6 +302,11 @@ impl FunctionTransformer for DateAddTzDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(maybe_to_utc(shifted_tz_expr, &time_zone)) @@ -369,6 +399,11 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -397,6 +432,11 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let date_add_in_utc = if time_zone == utc { @@ -420,6 +460,11 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index e163434e..c563eb2e 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -118,6 +118,11 @@ impl FunctionTransformer for DatePartTzWithDatePartAndAtTimezoneTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -181,6 +186,11 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let part_function = match part.to_ascii_lowercase().as_str() { @@ -215,6 +225,11 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -256,6 +271,11 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -278,6 +298,11 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -316,6 +341,11 @@ impl FunctionTransformer for DatePartTzMySqlTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -361,6 +391,11 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -383,6 +418,11 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } diff --git a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs index 23b7acf6..b27db001 100644 --- a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs @@ -122,6 +122,11 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(convert_tz_expr) @@ -165,6 +170,11 @@ impl FunctionTransformer for DateToUtcTimestampWithFunctionTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -221,6 +231,11 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); if self.at_timezone_utc { @@ -266,6 +281,11 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let to_time_zone_expr = SqlExpr::Function(SqlFunction { @@ -287,6 +307,11 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(to_time_zone_expr) @@ -324,6 +349,11 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); if time_zone == utc { @@ -350,6 +380,11 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs index 8822f2d6..0c3f104b 100644 --- a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs @@ -76,6 +76,11 @@ impl FunctionTransformer for DateTruncTzWithDateTruncAndAtTimezoneTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); @@ -135,6 +140,11 @@ impl FunctionTransformer for DateTruncTzWithTimestampTruncTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -185,6 +195,11 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); @@ -208,6 +223,11 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -253,6 +273,11 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -275,6 +300,11 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let date_trunc_in_utc = if time_zone == utc { @@ -297,6 +327,11 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -341,6 +376,11 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; let date_trunc_in_tz = SqlExpr::Function(SqlFunction { @@ -362,6 +402,11 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let date_trunc_in_utc = if time_zone == utc { @@ -385,6 +430,11 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; diff --git a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs index 6ad5f446..19474cec 100644 --- a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs @@ -52,6 +52,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(ts_millis_expr) @@ -100,6 +105,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let from_unix_time_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -117,6 +127,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let dateadd_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -141,6 +156,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(dateadd_expr) @@ -176,6 +196,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampDuckDbTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(epoch_ms_expr) @@ -224,6 +249,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let to_timestamp_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -241,6 +271,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let to_timestamp_at_utc_expr = SqlExpr::AtTimeZone { @@ -308,6 +343,11 @@ impl FunctionTransformer for EpochMsToUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(to_timestamp_expr) diff --git a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs index 4b0ff7c2..d2e87230 100644 --- a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs @@ -107,6 +107,11 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let time_expr = SqlExpr::Function(SqlFunction { @@ -127,6 +132,11 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let time_with_ms_expr = SqlExpr::Function(SqlFunction { @@ -154,6 +164,11 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let datetime_expr = SqlExpr::Function(SqlFunction { @@ -173,6 +188,11 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let timestamp_expr = SqlExpr::Function(SqlFunction { @@ -192,6 +212,11 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(timestamp_expr) @@ -236,6 +261,11 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let date_add_expr = SqlExpr::Function(SqlFunction { @@ -261,6 +291,11 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); @@ -284,6 +319,11 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -329,6 +369,11 @@ impl FunctionTransformer for MakeUtcTimestampDuckDbTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); @@ -385,6 +430,11 @@ impl FunctionTransformer for MakeUtcTimestampPostgresTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let timestamp_in_utc_expr = SqlExpr::AtTimeZone { @@ -443,6 +493,11 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let utc = make_utc_expr(); @@ -471,6 +526,11 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; diff --git a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs index 7be4c7d7..6723c7c8 100644 --- a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs @@ -121,6 +121,11 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); if self.at_timezone_utc { Ok(SqlExpr::AtTimeZone { @@ -169,6 +174,11 @@ impl FunctionTransformer for StrToUtcTimestampWithFunctionTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -205,6 +215,11 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let to_time_zone_expr = SqlExpr::Function(SqlFunction { @@ -226,6 +241,11 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(to_time_zone_expr) @@ -264,6 +284,11 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); if time_zone == utc { @@ -290,6 +315,11 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(convert_tz_expr) @@ -350,6 +380,11 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs index fd8146db..fe3be098 100644 --- a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs @@ -118,6 +118,11 @@ impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(convert_tz_expr) @@ -160,6 +165,11 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let convert_tz_expr = SqlExpr::Function(SqlFunction { @@ -179,6 +189,11 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { filter: None, null_treatment: None, over: None, + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs index 2ff85c48..724ac0cd 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs @@ -54,6 +54,11 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let lhs = SqlExpr::BinaryOp { left: Box::new(unix_timestamp_expr), @@ -80,6 +85,11 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let mod_1_expr = SqlExpr::Nested(Box::new(SqlExpr::BinaryOp { left: Box::new(date_part_expr), @@ -132,6 +142,11 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let lhs = SqlExpr::BinaryOp { left: Box::new(unix_timestamp_expr), @@ -158,6 +173,11 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); let rhs = SqlExpr::BinaryOp { left: Box::new(date_part_expr), @@ -216,6 +236,11 @@ impl FunctionTransformer for UtcTimestampToEpochMsPostgresTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(epoch_expr) @@ -257,6 +282,11 @@ impl FunctionTransformer for UtcTimestampToEpochMsSnowflakeTransform { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(date_part_expr) diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs index 45e292a1..0f7e1cc9 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs @@ -58,6 +58,11 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(SqlExpr::Function(SqlFunction { @@ -79,6 +84,11 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), })) } } @@ -121,6 +131,11 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -143,6 +158,11 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); // There should be a better way to do this, but including the "T" directly in the format @@ -169,6 +189,11 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(replace_expr) @@ -223,6 +248,11 @@ impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(strftime_expr) @@ -278,6 +308,11 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(strftime_expr) @@ -323,6 +358,11 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }) }; @@ -345,6 +385,11 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], + parameters: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), }); Ok(date_format_expr) diff --git a/vegafusion-sql/tests/test_aggregate.rs b/vegafusion-sql/tests/test_aggregate.rs index 28a8f630..f9f61f0e 100644 --- a/vegafusion-sql/tests/test_aggregate.rs +++ b/vegafusion-sql/tests/test_aggregate.rs @@ -2,7 +2,7 @@ extern crate lazy_static; mod utils; -use datafusion_expr::{expr, lit, max, min, Expr}; +use datafusion_expr::{expr, lit, Expr}; use rstest::rstest; use rstest_reuse::{self, *}; use serde_json::json; @@ -19,6 +19,8 @@ mod test_simple_aggs { #[apply(dialect_names)] async fn test(#[case] dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max, min}; + println!("{dialect_name}"); let (conn, evaluable) = TOKIO_RUNTIME.block_on(make_connection(dialect_name)); @@ -73,7 +75,6 @@ mod test_simple_aggs { #[cfg(test)] mod test_median_agg { use crate::*; - use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_functions_aggregate::expr_fn::count; use vegafusion_common::column::flat_col; @@ -101,7 +102,7 @@ mod test_median_agg { vec![ count(flat_col("a")).alias("count_a"), Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(median_udaf()), + func: median_udaf(), args: vec![flat_col("a")], distinct: false, filter: None, @@ -129,7 +130,6 @@ mod test_median_agg { #[cfg(test)] mod test_variance_aggs { use crate::*; - use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_functions_aggregate::stddev::{stddev_pop_udaf, stddev_udaf}; use datafusion_functions_aggregate::variance::{var_pop_udaf, var_samp_udaf}; use vegafusion_common::column::flat_col; @@ -157,7 +157,7 @@ mod test_variance_aggs { vec![flat_col("b")], vec![ round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(stddev_udaf()), + func: stddev_udaf(), args: vec![flat_col("a")], distinct: false, filter: None, @@ -168,7 +168,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(stddev_pop_udaf()), + func: stddev_pop_udaf(), args: vec![flat_col("a")], distinct: false, filter: None, @@ -179,7 +179,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("stddev_pop_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(var_samp_udaf()), + func: var_samp_udaf(), args: vec![flat_col("a")], distinct: false, filter: None, @@ -190,7 +190,7 @@ mod test_variance_aggs { .div(lit(100)) .alias("var_a"), round(vec![Expr::AggregateFunction(expr::AggregateFunction { - func_def: AggregateFunctionDefinition::UDF(var_pop_udaf()), + func: var_pop_udaf(), args: vec![flat_col("a")], distinct: false, filter: None, diff --git a/vegafusion-sql/tests/test_joinaggregate.rs b/vegafusion-sql/tests/test_joinaggregate.rs index 3ae18708..033e7e77 100644 --- a/vegafusion-sql/tests/test_joinaggregate.rs +++ b/vegafusion-sql/tests/test_joinaggregate.rs @@ -2,7 +2,7 @@ extern crate lazy_static; mod utils; -use datafusion_expr::{expr, max, min, Expr}; +use datafusion_expr::{expr, Expr}; use rstest::rstest; use rstest_reuse::{self, *}; use serde_json::json; @@ -18,6 +18,8 @@ mod test_simple_aggs { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max, min}; + println!("{dialect_name}"); let (conn, evaluable) = TOKIO_RUNTIME.block_on(make_connection(dialect_name)); @@ -81,6 +83,8 @@ mod test_simple_aggs_no_grouping { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max, min}; + println!("{dialect_name}"); let (conn, evaluable) = TOKIO_RUNTIME.block_on(make_connection(dialect_name)); diff --git a/vegafusion-sql/tests/test_window.rs b/vegafusion-sql/tests/test_window.rs index 5a3f9b90..401c71f4 100644 --- a/vegafusion-sql/tests/test_window.rs +++ b/vegafusion-sql/tests/test_window.rs @@ -4,8 +4,7 @@ extern crate lazy_static; mod utils; use datafusion_common::ScalarValue; use datafusion_expr::{ - col, expr, lit, AggregateFunction, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, - WindowFrameUnits, + col, expr, lit, BuiltInWindowFunction, Expr, WindowFrame, WindowFrameBound, WindowFrameUnits, }; use rstest::rstest; use rstest_reuse::{self, *}; @@ -25,6 +24,7 @@ mod test_simple_aggs_unbounded { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); @@ -79,7 +79,7 @@ mod test_simple_aggs_unbounded { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateUDF(min_udaf()), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -88,7 +88,7 @@ mod test_simple_aggs_unbounded { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateUDF(max_udaf()), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -128,6 +128,7 @@ mod test_simple_aggs_unbounded_groups { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); @@ -186,7 +187,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateUDF(min_udaf()), args: vec![flat_col("b")], partition_by: vec![flat_col("c")], order_by: order_by.clone(), @@ -195,7 +196,7 @@ mod test_simple_aggs_unbounded_groups { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateUDF(max_udaf()), args: vec![flat_col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -236,6 +237,7 @@ mod test_simple_aggs_bounded { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); @@ -294,7 +296,7 @@ mod test_simple_aggs_bounded { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateUDF(min_udaf()), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -303,7 +305,7 @@ mod test_simple_aggs_bounded { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateUDF(max_udaf()), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), @@ -343,6 +345,7 @@ mod test_simple_aggs_bounded_groups { #[apply(dialect_names)] async fn test(dialect_name: &str) { + use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; use sqlparser::ast::NullTreatment; println!("{dialect_name}"); @@ -401,7 +404,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("cume_mean_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Min), + fun: WindowFunctionDefinition::AggregateUDF(min_udaf()), args: vec![col("b")], partition_by: vec![col("c")], order_by: order_by.clone(), @@ -410,7 +413,7 @@ mod test_simple_aggs_bounded_groups { }) .alias("min_b"), Expr::WindowFunction(expr::WindowFunction { - fun: WindowFunctionDefinition::AggregateFunction(AggregateFunction::Max), + fun: WindowFunctionDefinition::AggregateUDF(max_udaf()), args: vec![col("b")], partition_by: vec![], order_by: order_by.clone(), From 84fad1e19f216d23c47a6b699334e9cbfa957a6a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Aug 2024 07:10:33 -0400 Subject: [PATCH 11/28] Update macos images --- .github/workflows/build_test.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 8785e907..c1d43f59 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -35,7 +35,7 @@ jobs: os: - ubuntu-20.04 - windows-2022 - - macos-11 + - macos-12 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -179,7 +179,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-64: - runs-on: macos-11 + runs-on: macos-12 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -206,7 +206,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-arm64: - runs-on: macos-11 + runs-on: macos-latest steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -221,9 +221,8 @@ jobs: ~/.cargo target .pixi - - name: Build vegafusion-server + - name: Build vegafusion-python-embed run: | - pixi run python automation/download_rust_target.py aarch64-apple-darwin pixi run build-py-embed --target aarch64-apple-darwin - name: Upload artifacts uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 @@ -328,7 +327,7 @@ jobs: run: pytest test-vegafusion-python-osx-64: - runs-on: macos-11 + runs-on: macos-12 needs: [build-vegafusion-python-embed-osx-64, build-vegafusion-packages] steps: - name: Check out repository code @@ -592,7 +591,7 @@ jobs: vegafusion-server-* build-vegafusion-server-osx-64: - runs-on: macos-11 + runs-on: macos-12 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -623,7 +622,7 @@ jobs: vegafusion-server-* build-vegafusion-server-osx-arm64: - runs-on: macos-11 + runs-on: macos-latest steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -640,7 +639,6 @@ jobs: .pixi - name: Build vegafusion-server run: | - pixi run python automation/download_rust_target.py aarch64-apple-darwin pixi run build-rs-vegafusion-server --target aarch64-apple-darwin - name: zip executable uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 From 8cd51870d1900c31bff21a0b7e196dbd21e07bfc Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Aug 2024 07:40:05 -0400 Subject: [PATCH 12/28] DataFusion 41 fixes --- vegafusion-sql/src/compile/expr.rs | 49 +++++++------------ vegafusion-sql/src/compile/scalar.rs | 6 +-- .../src/connection/datafusion_conn.rs | 1 + 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index c39e9a2e..d3856a07 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -3,11 +3,11 @@ use crate::compile::scalar::ToSqlScalar; use arrow::datatypes::DataType; use datafusion_common::{DFSchema, ScalarValue}; use sqlparser::ast::{ - BinaryOperator as SqlBinaryOperator, CastKind, Expr as SqlExpr, Function as SqlFunction, - FunctionArg as SqlFunctionArg, FunctionArgumentList, FunctionArguments, Ident, - ObjectName as SqlObjectName, UnaryOperator as SqlUnaryOperator, WindowFrame as SqlWindowFrame, - WindowFrameBound as SqlWindowBound, WindowFrameUnits as SqlWindowFrameUnits, - WindowSpec as SqlWindowSpec, WindowType, + BinaryOperator as SqlBinaryOperator, CastKind, DuplicateTreatment, Expr as SqlExpr, + Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgumentList, + FunctionArguments, Ident, ObjectName as SqlObjectName, UnaryOperator as SqlUnaryOperator, + WindowFrame as SqlWindowFrame, WindowFrameBound as SqlWindowBound, + WindowFrameUnits as SqlWindowFrameUnits, WindowSpec as SqlWindowSpec, WindowType, }; use datafusion_expr::expr::{BinaryExpr, Case, Cast, Sort}; @@ -367,11 +367,7 @@ impl ToSqlExpr for Expr { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })); } UnorderedRowNumberMode::OrderByConstant => { @@ -463,11 +459,7 @@ impl ToSqlExpr for Expr { null_treatment: None, over: Some(over), within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }; Ok(SqlExpr::Function(sql_fun)) @@ -570,11 +562,7 @@ fn translate_scalar_function( null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } else if let Some(transformer) = dialect.scalar_transformers.get(fun_name) { // Supported through AST transformation @@ -590,7 +578,7 @@ fn translate_scalar_function( fn translate_aggregate_function( fun_name: &str, args: &[Expr], - _distinct: bool, // Where should distinct go now? + distinct: bool, dialect: &Dialect, schema: &DFSchema, ) -> Result { @@ -600,24 +588,25 @@ fn translate_aggregate_function( quote_style: None, }; let args = translate_function_args(args, dialect, schema)?; - - Ok(SqlExpr::Function(SqlFunction { + let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args: FunctionArguments::List(FunctionArgumentList { args, - duplicate_treatment: None, + duplicate_treatment: if distinct { + Some(DuplicateTreatment::Distinct) + } else { + None + }, clauses: vec![], }), filter: None, null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), - })) + parameters: FunctionArguments::None, + }); + println!("{}", fn_expr.to_string()); + Ok(fn_expr) } else if let Some(transformer) = dialect.aggregate_transformers.get(fun_name) { // Supported through AST transformation transformer.transform(args, dialect, schema) diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index 3314674a..92887fc7 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -209,11 +209,7 @@ impl ToSqlScalar for ScalarValue { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } ScalarValue::Date32(v) => date32_to_date(v, dialect), diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index 1ff8400e..1b65de52 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -488,6 +488,7 @@ pub fn make_datafusion_context() -> SessionContext { let session_state = SessionStateBuilder::new() .with_config(config) .with_runtime_env(runtime) + .with_default_features() .build(); let ctx = SessionContext::new_with_state(session_state); From f1b43b24eed1ee5f8839f7201e21421f8e28dd7f Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Aug 2024 07:48:55 -0400 Subject: [PATCH 13/28] macos-13 --- .github/workflows/build_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index c1d43f59..9babb978 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -327,7 +327,7 @@ jobs: run: pytest test-vegafusion-python-osx-64: - runs-on: macos-12 + runs-on: macos-13 needs: [build-vegafusion-python-embed-osx-64, build-vegafusion-packages] steps: - name: Check out repository code @@ -591,7 +591,7 @@ jobs: vegafusion-server-* build-vegafusion-server-osx-64: - runs-on: macos-12 + runs-on: macos-13 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 From 7f4ffdd4a0f8c0752ec3952f77c1da98570e1bc2 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Aug 2024 10:18:24 -0400 Subject: [PATCH 14/28] Remove coalesce usage from pivot transform --- vegafusion-runtime/src/transform/pivot.rs | 15 ++++++--------- vegafusion-sql/src/compile/expr.rs | 1 - 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/vegafusion-runtime/src/transform/pivot.rs b/vegafusion-runtime/src/transform/pivot.rs index 086112dd..ea1b720e 100644 --- a/vegafusion-runtime/src/transform/pivot.rs +++ b/vegafusion-runtime/src/transform/pivot.rs @@ -3,7 +3,6 @@ use crate::transform::aggregate::make_agg_expr_for_col_expr; use crate::transform::TransformTrait; use async_trait::async_trait; use datafusion_expr::{expr::Sort, lit, when, Expr}; -use datafusion_functions::expr_fn::coalesce; use datafusion_functions_aggregate::expr_fn::min; use std::sync::Arc; use vegafusion_common::arrow::array::StringArray; @@ -170,17 +169,15 @@ async fn pivot_case( for pivot_val in pivot_vec.iter() { let predicate_expr = unescaped_col(&tx.field).eq(lit(pivot_val.as_str())); let value_expr = unescaped_col(tx.value.as_str()); - let agg_col = when(predicate_expr, value_expr).otherwise(lit(ScalarValue::Null))?; + let agg_col = when(predicate_expr, value_expr).otherwise(if fill_zero { + // Replace null with zero for certain aggregates + lit(0) + } else { + lit(ScalarValue::Null) + })?; let agg_expr = make_agg_expr_for_col_expr(agg_col, &agg_op, &dataframe.schema_df()?)?; - // Replace null with zero for certain aggregates - let agg_expr = if fill_zero { - coalesce(vec![agg_expr, lit(0.0)]) - } else { - agg_expr - }; - // Compute pivot column name, replacing null placeholder with "null" let col_name = if pivot_val == NULL_PLACEHOLDER_NAME { NULL_NAME diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index d3856a07..c1b3a537 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -605,7 +605,6 @@ fn translate_aggregate_function( within_group: vec![], parameters: FunctionArguments::None, }); - println!("{}", fn_expr.to_string()); Ok(fn_expr) } else if let Some(transformer) = dialect.aggregate_transformers.get(fun_name) { // Supported through AST transformation From 6d1dbd56083c08aff564ba462332e2306e7fa4c4 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 12 Aug 2024 19:54:19 -0400 Subject: [PATCH 15/28] fixes --- .../tests/test_pre_transform_values.rs | 24 ++--- vegafusion-sql/src/dataframe/mod.rs | 9 +- vegafusion-sql/src/dialect/mod.rs | 42 ++------ .../src/dialect/transforms/date_add_tz.rs | 54 ++-------- .../src/dialect/transforms/date_part_tz.rs | 48 ++------- .../transforms/date_to_utc_timestamp.rs | 42 ++------ .../src/dialect/transforms/date_trunc_tz.rs | 60 ++--------- .../transforms/epoch_ms_to_utc_timestamp.rs | 48 ++------- .../dialect/transforms/make_utc_timestamp.rs | 76 +++---------- .../transforms/str_to_utc_timestamp.rs | 42 ++------ .../dialect/transforms/to_utc_timestamp.rs | 18 +--- .../transforms/utc_timestamp_to_epoch_ms.rs | 36 ++----- .../transforms/utc_timestamp_to_str.rs | 54 ++-------- vegafusion-sql/tests/expected/stack.toml | 100 +++++++++--------- 14 files changed, 154 insertions(+), 499 deletions(-) diff --git a/vegafusion-runtime/tests/test_pre_transform_values.rs b/vegafusion-runtime/tests/test_pre_transform_values.rs index f9ffa628..7595b60a 100644 --- a/vegafusion-runtime/tests/test_pre_transform_values.rs +++ b/vegafusion-runtime/tests/test_pre_transform_values.rs @@ -342,8 +342,8 @@ mod tests { +---------------------+---------------------+---------+---------+---------------+-------------+ | yearmonth_date | yearmonth_date_end | weather | __count | __count_start | __count_end | +---------------------+---------------------+---------+---------+---------------+-------------+ -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12.0 | 27.0 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0.0 | 16.0 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12 | 27 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0 | 16 | +---------------------+---------------------+---------+---------+---------------+-------------+"; assert_eq!(click_selected.pretty_format(None).unwrap(), expected); @@ -355,16 +355,16 @@ mod tests { +---------------------+---------------------+---------+---------+---------------+-------------+ | yearmonth_date | yearmonth_date_end | weather | __count | __count_start | __count_end | +---------------------+---------------------+---------+---------+---------------+-------------+ -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | sun | 12 | 0.0 | 12.0 | -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12.0 | 27.0 | -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | fog | 2 | 27.0 | 29.0 | -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | drizzle | 1 | 29.0 | 30.0 | -| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | sun | 17 | 0.0 | 17.0 | -| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | snow | 1 | 17.0 | 18.0 | -| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | rain | 13 | 18.0 | 31.0 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0.0 | 16.0 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | rain | 13 | 16.0 | 29.0 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | fog | 2 | 29.0 | 31.0 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | sun | 12 | 0 | 12 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12 | 27 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | fog | 2 | 27 | 29 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | drizzle | 1 | 29 | 30 | +| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | sun | 17 | 0 | 17 | +| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | snow | 1 | 17 | 18 | +| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | rain | 13 | 18 | 31 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0 | 16 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | rain | 13 | 16 | 29 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | fog | 2 | 29 | 31 | +---------------------+---------------------+---------+---------+---------------+-------------+"; assert_eq!(drag_selected.pretty_format(None).unwrap(), expected); } diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index 9a5519ef..e28fb3ec 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -912,7 +912,7 @@ impl SqlDataFrame { let partition_by: Vec<_> = groupby.iter().map(|group| flat_col(group)).collect(); let numeric_field = coalesce(vec![ to_numeric(flat_col(field), &self.schema_df()?)?, - lit(0.0), + lit(0), ]); if let StackMode::Zero = mode { @@ -1117,9 +1117,9 @@ impl SqlDataFrame { dataframe } StackMode::Normalize => { - let total_zero = flat_col("__total").eq(lit(0.0)); + let total_zero = flat_col("__total").eq(lit(0)); - let start_col = when(total_zero.clone(), lit(0.0)) + let start_col = when(total_zero.clone(), lit(0)) .otherwise( flat_col(cumulative_field) .sub(flat_col(stack_col_name)) @@ -1129,7 +1129,7 @@ impl SqlDataFrame { final_selection.push(start_col); - let stop_col = when(total_zero, lit(0.0)) + let stop_col = when(total_zero, lit(0)) .otherwise(flat_col(cumulative_field).div(flat_col("__total")))? .alias(stop_field); @@ -1490,6 +1490,7 @@ fn query_chain_to_cte(queries: &[Query], prefix: &str) -> Query { } fn parse_sql_query(query: &str, dialect: &Dialect) -> Result { + println!("{}", query); let statements: Vec = Parser::parse_sql(dialect.parser_dialect().as_ref(), query)?; if let Some(statement) = statements.first() { if let Statement::Query(box_query) = statement { diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index 7af10aef..9fd0eca6 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -1760,11 +1760,7 @@ impl BinaryOperatorTransformer for ModulusOpToFunction { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -1812,11 +1808,7 @@ impl FunctionTransformer for RenameFunctionTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -1853,11 +1845,7 @@ impl FunctionTransformer for ExpWithPowFunctionTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -1903,11 +1891,7 @@ impl FunctionTransformer for CastArgsFunctionTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -1950,11 +1934,7 @@ impl FunctionTransformer for LogBaseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -2001,11 +1981,7 @@ impl FunctionTransformer for LogBaseWithLnTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let base_arg = FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Value( @@ -2025,11 +2001,7 @@ impl FunctionTransformer for LogBaseWithLnTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(SqlExpr::BinaryOp { diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index 1215ff4a..6de76acc 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -78,11 +78,7 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) } } @@ -109,11 +105,7 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &SqlExpr) -> SqlExpr { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) } } @@ -152,11 +144,7 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let date_time_field = part_to_date_time_field(&part)?; @@ -185,11 +173,7 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let timestamp_expr = SqlExpr::Function(SqlFunction { @@ -209,11 +193,7 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(timestamp_expr) @@ -302,11 +282,7 @@ impl FunctionTransformer for DateAddTzDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(maybe_to_utc(shifted_tz_expr, &time_zone)) @@ -399,11 +375,7 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -432,11 +404,7 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let date_add_in_utc = if time_zone == utc { @@ -460,11 +428,7 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index c563eb2e..59488a1b 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -118,11 +118,7 @@ impl FunctionTransformer for DatePartTzWithDatePartAndAtTimezoneTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -186,11 +182,7 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let part_function = match part.to_ascii_lowercase().as_str() { @@ -225,11 +217,7 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -271,11 +259,7 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -298,11 +282,7 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -341,11 +321,7 @@ impl FunctionTransformer for DatePartTzMySqlTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -391,11 +367,7 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -418,11 +390,7 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } diff --git a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs index b27db001..f5efdc73 100644 --- a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs @@ -122,11 +122,7 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(convert_tz_expr) @@ -170,11 +166,7 @@ impl FunctionTransformer for DateToUtcTimestampWithFunctionTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -231,11 +223,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); if self.at_timezone_utc { @@ -281,11 +269,7 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let to_time_zone_expr = SqlExpr::Function(SqlFunction { @@ -307,11 +291,7 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(to_time_zone_expr) @@ -349,11 +329,7 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); if time_zone == utc { @@ -380,11 +356,7 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs index 0c3f104b..38691461 100644 --- a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs @@ -76,11 +76,7 @@ impl FunctionTransformer for DateTruncTzWithDateTruncAndAtTimezoneTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); @@ -140,11 +136,7 @@ impl FunctionTransformer for DateTruncTzWithTimestampTruncTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -195,11 +187,7 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); @@ -223,11 +211,7 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -273,11 +257,7 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -300,11 +280,7 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let date_trunc_in_utc = if time_zone == utc { @@ -327,11 +303,7 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -376,11 +348,7 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; let date_trunc_in_tz = SqlExpr::Function(SqlFunction { @@ -402,11 +370,7 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let date_trunc_in_utc = if time_zone == utc { @@ -430,11 +394,7 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; diff --git a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs index 19474cec..35599454 100644 --- a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs @@ -52,11 +52,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(ts_millis_expr) @@ -105,11 +101,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let from_unix_time_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -127,11 +119,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let dateadd_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -156,11 +144,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(dateadd_expr) @@ -196,11 +180,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampDuckDbTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(epoch_ms_expr) @@ -249,11 +229,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let to_timestamp_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -271,11 +247,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let to_timestamp_at_utc_expr = SqlExpr::AtTimeZone { @@ -343,11 +315,7 @@ impl FunctionTransformer for EpochMsToUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(to_timestamp_expr) diff --git a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs index d2e87230..ee4b56cb 100644 --- a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs @@ -107,11 +107,7 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let time_expr = SqlExpr::Function(SqlFunction { @@ -132,11 +128,7 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let time_with_ms_expr = SqlExpr::Function(SqlFunction { @@ -164,11 +156,7 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let datetime_expr = SqlExpr::Function(SqlFunction { @@ -188,11 +176,7 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let timestamp_expr = SqlExpr::Function(SqlFunction { @@ -212,11 +196,7 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(timestamp_expr) @@ -261,11 +241,7 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let date_add_expr = SqlExpr::Function(SqlFunction { @@ -291,11 +267,7 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); @@ -319,11 +291,7 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -369,11 +337,7 @@ impl FunctionTransformer for MakeUtcTimestampDuckDbTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); @@ -430,11 +394,7 @@ impl FunctionTransformer for MakeUtcTimestampPostgresTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let timestamp_in_utc_expr = SqlExpr::AtTimeZone { @@ -493,11 +453,7 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let utc = make_utc_expr(); @@ -511,9 +467,7 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { }]), args: FunctionArguments::List(FunctionArgumentList { args: vec![ - SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( - SqlValue::SingleQuotedString(sql_args.time_zone.to_string()), - ))), + SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.time_zone)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(SqlExpr::Value( SqlValue::SingleQuotedString("UTC".to_string()), ))), @@ -526,11 +480,7 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; diff --git a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs index 6723c7c8..87302a6d 100644 --- a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs @@ -121,11 +121,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); if self.at_timezone_utc { Ok(SqlExpr::AtTimeZone { @@ -174,11 +170,7 @@ impl FunctionTransformer for StrToUtcTimestampWithFunctionTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -215,11 +207,7 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let to_time_zone_expr = SqlExpr::Function(SqlFunction { @@ -241,11 +229,7 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(to_time_zone_expr) @@ -284,11 +268,7 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); if time_zone == utc { @@ -315,11 +295,7 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(convert_tz_expr) @@ -380,11 +356,7 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs index fe3be098..4dcf17e3 100644 --- a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs @@ -118,11 +118,7 @@ impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(convert_tz_expr) @@ -165,11 +161,7 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let convert_tz_expr = SqlExpr::Function(SqlFunction { @@ -189,11 +181,7 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { filter: None, null_treatment: None, over: None, - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(convert_tz_expr) diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs index 724ac0cd..98b4614f 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs @@ -54,11 +54,7 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let lhs = SqlExpr::BinaryOp { left: Box::new(unix_timestamp_expr), @@ -85,11 +81,7 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let mod_1_expr = SqlExpr::Nested(Box::new(SqlExpr::BinaryOp { left: Box::new(date_part_expr), @@ -142,11 +134,7 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let lhs = SqlExpr::BinaryOp { left: Box::new(unix_timestamp_expr), @@ -173,11 +161,7 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); let rhs = SqlExpr::BinaryOp { left: Box::new(date_part_expr), @@ -236,11 +220,7 @@ impl FunctionTransformer for UtcTimestampToEpochMsPostgresTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(epoch_expr) @@ -282,11 +262,7 @@ impl FunctionTransformer for UtcTimestampToEpochMsSnowflakeTransform { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(date_part_expr) diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs index 0f7e1cc9..b2ff94d1 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs @@ -58,11 +58,7 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(SqlExpr::Function(SqlFunction { @@ -84,11 +80,7 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, })) } } @@ -131,11 +123,7 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -158,11 +146,7 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); // There should be a better way to do this, but including the "T" directly in the format @@ -189,11 +173,7 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(replace_expr) @@ -248,11 +228,7 @@ impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(strftime_expr) @@ -308,11 +284,7 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(strftime_expr) @@ -358,11 +330,7 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }) }; @@ -385,11 +353,7 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { null_treatment: None, over: None, within_group: vec![], - parameters: FunctionArguments::List(FunctionArgumentList { - args: vec![], - duplicate_treatment: None, - clauses: vec![], - }), + parameters: FunctionArguments::None, }); Ok(date_format_expr) diff --git a/vegafusion-sql/tests/expected/stack.toml b/vegafusion-sql/tests/expected/stack.toml index a0a06e6c..d5d7f367 100644 --- a/vegafusion-sql/tests/expected/stack.toml +++ b/vegafusion-sql/tests/expected/stack.toml @@ -2,71 +2,71 @@ athena = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ bigquery = """ WITH \ values0 AS (SELECT 1 AS `a`, 9 AS `b`, 'A' AS `c` UNION ALL SELECT -3 AS `a`, 8 AS `b`, 'BB' AS `c` UNION ALL SELECT 5 AS `a`, 7 AS `b`, 'A' AS `c` UNION ALL SELECT -7 AS `a`, 6 AS `b`, 'BB' AS `c` UNION ALL SELECT 9 AS `a`, 5 AS `b`, 'BB' AS `c` UNION ALL SELECT -11 AS `a`, 4 AS `b`, 'A' AS `c` UNION ALL SELECT 13 AS `a`, 3 AS `b`, 'BB' AS `c`), \ -values1 AS (SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) < 0), \ -values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0.0)) AS `start`, `end` FROM values1) \ +values1 AS (SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) < 0), \ +values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0)) AS `start`, `end` FROM values1) \ SELECT * FROM values2 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST """ clickhouse = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ databricks = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) < 0), \ -values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0.0)) AS `start`, `end` FROM values1) \ +values1 AS (SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) < 0), \ +values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0)) AS `start`, `end` FROM values1) \ SELECT * FROM values2 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST """ datafusion = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ duckdb = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ mysql = """ WITH \ values0 AS (SELECT * FROM (VALUES ROW(1, 9, 'A'), ROW(-3, 8, 'BB'), ROW(5, 7, 'A'), ROW(-7, 6, 'BB'), ROW(9, 5, 'BB'), ROW(-11, 4, 'A'), ROW(13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) < 0), \ -values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0.0)) AS `start`, `end` FROM values1) \ +values1 AS (SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) < 0), \ +values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0)) AS `start`, `end` FROM values1) \ SELECT * FROM values2 ORDER BY `c` ASC, `end` ASC """ postgres = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ redshift = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ snowflake = """ WITH \ values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB'))), \ -values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ result = ''' @@ -87,7 +87,7 @@ result = ''' athena = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -97,7 +97,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST bigquery = """ WITH \ values0 AS (SELECT 1 AS `a`, 9 AS `b`, 'A' AS `c` UNION ALL SELECT -3 AS `a`, 8 AS `b`, 'BB' AS `c` UNION ALL SELECT 5 AS `a`, 7 AS `b`, 'A' AS `c` UNION ALL SELECT -7 AS `a`, 6 AS `b`, 'BB' AS `c` UNION ALL SELECT 9 AS `a`, 5 AS `b`, 'BB' AS `c` UNION ALL SELECT -11 AS `a`, 4 AS `b`, 'A' AS `c` UNION ALL SELECT 13 AS `a`, 3 AS `b`, 'BB' AS `c`), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max(`__total`) AS `__max_total` FROM values3) AS _cross), \ @@ -107,7 +107,7 @@ SELECT * FROM values5 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST clickhouse = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -117,7 +117,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST databricks = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max(`__total`) AS `__max_total` FROM values3) AS _cross), \ @@ -127,7 +127,7 @@ SELECT * FROM values5 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST datafusion = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -137,7 +137,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST duckdb = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -147,7 +147,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST mysql = """ WITH \ values0 AS (SELECT * FROM (VALUES ROW(1, 9, 'A'), ROW(-3, 8, 'BB'), ROW(5, 7, 'A'), ROW(-7, 6, 'BB'), ROW(9, 5, 'BB'), ROW(-11, 4, 'A'), ROW(13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max(`__total`) AS `__max_total` FROM values3) AS _cross), \ @@ -157,7 +157,7 @@ SELECT * FROM values5 ORDER BY `c` ASC, `end` ASC postgres = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -167,7 +167,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST redshift = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -177,7 +177,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST snowflake = """ WITH \ values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB'))), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -202,100 +202,100 @@ result = ''' athena = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ bigquery = """ WITH \ values0 AS (SELECT 1 AS `a`, 9 AS `b`, 'A' AS `c` UNION ALL SELECT -3 AS `a`, 8 AS `b`, 'BB' AS `c` UNION ALL SELECT 5 AS `a`, 7 AS `b`, 'A' AS `c` UNION ALL SELECT -7 AS `a`, 6 AS `b`, 'BB' AS `c` UNION ALL SELECT 9 AS `a`, 5 AS `b`, 'BB' AS `c` UNION ALL SELECT -11 AS `a`, 4 AS `b`, 'A' AS `c` UNION ALL SELECT 13 AS `a`, 3 AS `b`, 'BB' AS `c`), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ -values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ +values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0) THEN 0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0) THEN 0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST) \ SELECT `a`, `b`, `c`, (round((`start` * 100)) / 100) AS `trunc_start`, (round((`end` * 100)) / 100) AS `trunc_end` FROM values5 """ clickhouse = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ databricks = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ -values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ +values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0) THEN 0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0) THEN 0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST) \ SELECT `a`, `b`, `c`, (round((`start` * 100)) / 100) AS `trunc_start`, (round((`end` * 100)) / 100) AS `trunc_end` FROM values5 """ datafusion = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ duckdb = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ mysql = """ WITH \ values0 AS (SELECT * FROM (VALUES ROW(1, 9, 'A'), ROW(-3, 8, 'BB'), ROW(5, 7, 'A'), ROW(-7, 6, 'BB'), ROW(9, 5, 'BB'), ROW(-11, 4, 'A'), ROW(13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ -values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ +values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0) THEN 0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0) THEN 0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY `c` ASC, `end` ASC) \ SELECT `a`, `b`, `c`, (round((`start` * 100)) / 100) AS `trunc_start`, (round((`end` * 100)) / 100) AS `trunc_end` FROM values5 """ postgres = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ redshift = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ snowflake = """ WITH \ values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB'))), \ -values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ From b9099ef8145ab0371c9f0123837c36fc09a31aaa Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Aug 2024 07:08:50 -0400 Subject: [PATCH 16/28] Fix stack --- vegafusion-sql/src/dataframe/mod.rs | 9 +- vegafusion-sql/tests/expected/stack.toml | 100 +++++++++++------------ 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index e28fb3ec..9a5519ef 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -912,7 +912,7 @@ impl SqlDataFrame { let partition_by: Vec<_> = groupby.iter().map(|group| flat_col(group)).collect(); let numeric_field = coalesce(vec![ to_numeric(flat_col(field), &self.schema_df()?)?, - lit(0), + lit(0.0), ]); if let StackMode::Zero = mode { @@ -1117,9 +1117,9 @@ impl SqlDataFrame { dataframe } StackMode::Normalize => { - let total_zero = flat_col("__total").eq(lit(0)); + let total_zero = flat_col("__total").eq(lit(0.0)); - let start_col = when(total_zero.clone(), lit(0)) + let start_col = when(total_zero.clone(), lit(0.0)) .otherwise( flat_col(cumulative_field) .sub(flat_col(stack_col_name)) @@ -1129,7 +1129,7 @@ impl SqlDataFrame { final_selection.push(start_col); - let stop_col = when(total_zero, lit(0)) + let stop_col = when(total_zero, lit(0.0)) .otherwise(flat_col(cumulative_field).div(flat_col("__total")))? .alias(stop_field); @@ -1490,7 +1490,6 @@ fn query_chain_to_cte(queries: &[Query], prefix: &str) -> Query { } fn parse_sql_query(query: &str, dialect: &Dialect) -> Result { - println!("{}", query); let statements: Vec = Parser::parse_sql(dialect.parser_dialect().as_ref(), query)?; if let Some(statement) = statements.first() { if let Statement::Query(box_query) = statement { diff --git a/vegafusion-sql/tests/expected/stack.toml b/vegafusion-sql/tests/expected/stack.toml index d5d7f367..a0a06e6c 100644 --- a/vegafusion-sql/tests/expected/stack.toml +++ b/vegafusion-sql/tests/expected/stack.toml @@ -2,71 +2,71 @@ athena = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ bigquery = """ WITH \ values0 AS (SELECT 1 AS `a`, 9 AS `b`, 'A' AS `c` UNION ALL SELECT -3 AS `a`, 8 AS `b`, 'BB' AS `c` UNION ALL SELECT 5 AS `a`, 7 AS `b`, 'A' AS `c` UNION ALL SELECT -7 AS `a`, 6 AS `b`, 'BB' AS `c` UNION ALL SELECT 9 AS `a`, 5 AS `b`, 'BB' AS `c` UNION ALL SELECT -11 AS `a`, 4 AS `b`, 'A' AS `c` UNION ALL SELECT 13 AS `a`, 3 AS `b`, 'BB' AS `c`), \ -values1 AS (SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) < 0), \ -values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0)) AS `start`, `end` FROM values1) \ +values1 AS (SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) < 0), \ +values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0.0)) AS `start`, `end` FROM values1) \ SELECT * FROM values2 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST """ clickhouse = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ databricks = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) < 0), \ -values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0)) AS `start`, `end` FROM values1) \ +values1 AS (SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) < 0), \ +values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0.0)) AS `start`, `end` FROM values1) \ SELECT * FROM values2 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST """ datafusion = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ duckdb = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ mysql = """ WITH \ values0 AS (SELECT * FROM (VALUES ROW(1, 9, 'A'), ROW(-3, 8, 'BB'), ROW(5, 7, 'A'), ROW(-7, 6, 'BB'), ROW(9, 5, 'BB'), ROW(-11, 4, 'A'), ROW(13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0) < 0), \ -values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0)) AS `start`, `end` FROM values1) \ +values1 AS (SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) >= 0 UNION ALL SELECT *, sum(coalesce(`a`, 0.0)) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `end` FROM values0 WHERE coalesce(`a`, 0.0) < 0), \ +values2 AS (SELECT `a`, `b`, `c`, (`end` - coalesce(`a`, 0.0)) AS `start`, `end` FROM values1) \ SELECT * FROM values2 ORDER BY `c` ASC, `end` ASC """ postgres = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ redshift = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ snowflake = """ WITH \ values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB'))), \ -values1 AS (SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0) < 0), \ -values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0)) AS "start", "end" FROM values1) \ +values1 AS (SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) >= 0 UNION ALL SELECT *, sum(coalesce("a", 0.0)) OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "end" FROM values0 WHERE coalesce("a", 0.0) < 0), \ +values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" FROM values1) \ SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ result = ''' @@ -87,7 +87,7 @@ result = ''' athena = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -97,7 +97,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST bigquery = """ WITH \ values0 AS (SELECT 1 AS `a`, 9 AS `b`, 'A' AS `c` UNION ALL SELECT -3 AS `a`, 8 AS `b`, 'BB' AS `c` UNION ALL SELECT 5 AS `a`, 7 AS `b`, 'A' AS `c` UNION ALL SELECT -7 AS `a`, 6 AS `b`, 'BB' AS `c` UNION ALL SELECT 9 AS `a`, 5 AS `b`, 'BB' AS `c` UNION ALL SELECT -11 AS `a`, 4 AS `b`, 'A' AS `c` UNION ALL SELECT 13 AS `a`, 3 AS `b`, 'BB' AS `c`), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max(`__total`) AS `__max_total` FROM values3) AS _cross), \ @@ -107,7 +107,7 @@ SELECT * FROM values5 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST clickhouse = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -117,7 +117,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST databricks = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max(`__total`) AS `__max_total` FROM values3) AS _cross), \ @@ -127,7 +127,7 @@ SELECT * FROM values5 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST datafusion = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -137,7 +137,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST duckdb = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -147,7 +147,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST mysql = """ WITH \ values0 AS (SELECT * FROM (VALUES ROW(1, 9, 'A'), ROW(-3, 8, 'BB'), ROW(5, 7, 'A'), ROW(-7, 6, 'BB'), ROW(9, 5, 'BB'), ROW(-11, 4, 'A'), ROW(13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max(`__total`) AS `__max_total` FROM values3) AS _cross), \ @@ -157,7 +157,7 @@ SELECT * FROM values5 ORDER BY `c` ASC, `end` ASC postgres = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -167,7 +167,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST redshift = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -177,7 +177,7 @@ SELECT * FROM values5 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST snowflake = """ WITH \ values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB'))), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ values4 AS (SELECT * FROM values3 CROSS JOIN (SELECT max("__total") AS "__max_total" FROM values3) AS _cross), \ @@ -202,100 +202,100 @@ result = ''' athena = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ bigquery = """ WITH \ values0 AS (SELECT 1 AS `a`, 9 AS `b`, 'A' AS `c` UNION ALL SELECT -3 AS `a`, 8 AS `b`, 'BB' AS `c` UNION ALL SELECT 5 AS `a`, 7 AS `b`, 'A' AS `c` UNION ALL SELECT -7 AS `a`, 6 AS `b`, 'BB' AS `c` UNION ALL SELECT 9 AS `a`, 5 AS `b`, 'BB' AS `c` UNION ALL SELECT -11 AS `a`, 4 AS `b`, 'A' AS `c` UNION ALL SELECT 13 AS `a`, 3 AS `b`, 'BB' AS `c`), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ -values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0) THEN 0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0) THEN 0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ +values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST) \ SELECT `a`, `b`, `c`, (round((`start` * 100)) / 100) AS `trunc_start`, (round((`end` * 100)) / 100) AS `trunc_end` FROM values5 """ clickhouse = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ databricks = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ -values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0) THEN 0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0) THEN 0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ +values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY `c` ASC NULLS FIRST, `end` ASC NULLS FIRST) \ SELECT `a`, `b`, `c`, (round((`start` * 100)) / 100) AS `trunc_start`, (round((`end` * 100)) / 100) AS `trunc_end` FROM values5 """ datafusion = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ duckdb = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ mysql = """ WITH \ values0 AS (SELECT * FROM (VALUES ROW(1, 9, 'A'), ROW(-3, 8, 'BB'), ROW(5, 7, 'A'), ROW(-7, 6, 'BB'), ROW(9, 5, 'BB'), ROW(-11, 4, 'A'), ROW(13, 3, 'BB')) AS `_values` (`a`, `b`, `c`)), \ -values1 AS (SELECT *, abs(coalesce(`a`, 0)) AS `__stack` FROM values0), \ +values1 AS (SELECT *, abs(coalesce(`a`, 0.0)) AS `__stack` FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT `c`, sum(`__stack`) AS `__total` FROM values1 GROUP BY `c`) AS __inner USING(`c`)), \ values3 AS (SELECT *, sum(`__stack`) OVER (PARTITION BY `c` ORDER BY `b` ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `_cumulative` FROM values2), \ -values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0) THEN 0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0) THEN 0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ +values4 AS (SELECT `a`, `b`, `c`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE ((`_cumulative` - `__stack`) / `__total`) END AS `start`, CASE WHEN (`__total` = 0.0) THEN 0.0 ELSE (`_cumulative` / `__total`) END AS `end` FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY `c` ASC, `end` ASC) \ SELECT `a`, `b`, `c`, (round((`start` * 100)) / 100) AS `trunc_start`, (round((`end` * 100)) / 100) AS `trunc_end` FROM values5 """ postgres = """ WITH \ values0 AS (SELECT * FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB')) AS "_values" ("a", "b", "c")), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ redshift = """ WITH \ values0 AS (SELECT 1 AS "a", 9 AS "b", 'A' AS "c" UNION ALL SELECT -3 AS "a", 8 AS "b", 'BB' AS "c" UNION ALL SELECT 5 AS "a", 7 AS "b", 'A' AS "c" UNION ALL SELECT -7 AS "a", 6 AS "b", 'BB' AS "c" UNION ALL SELECT 9 AS "a", 5 AS "b", 'BB' AS "c" UNION ALL SELECT -11 AS "a", 4 AS "b", 'A' AS "c" UNION ALL SELECT 13 AS "a", 3 AS "b", 'BB' AS "c"), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ snowflake = """ WITH \ values0 AS (SELECT "COLUMN1" AS "a", "COLUMN2" AS "b", "COLUMN3" AS "c" FROM (VALUES (1, 9, 'A'), (-3, 8, 'BB'), (5, 7, 'A'), (-7, 6, 'BB'), (9, 5, 'BB'), (-11, 4, 'A'), (13, 3, 'BB'))), \ -values1 AS (SELECT *, abs(coalesce("a", 0)) AS "__stack" FROM values0), \ +values1 AS (SELECT *, abs(coalesce("a", 0.0)) AS "__stack" FROM values0), \ values2 AS (SELECT * FROM values1 JOIN (SELECT "c", sum("__stack") AS "__total" FROM values1 GROUP BY "c") AS __inner USING("c")), \ values3 AS (SELECT *, sum("__stack") OVER (PARTITION BY "c" ORDER BY "b" ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS "_cumulative" FROM values2), \ -values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0) THEN 0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0) THEN 0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ +values4 AS (SELECT "a", "b", "c", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE (("_cumulative" - "__stack") / "__total") END AS "start", CASE WHEN ("__total" = 0.0) THEN 0.0 ELSE ("_cumulative" / "__total") END AS "end" FROM values3), \ values5 AS (SELECT * FROM values4 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST) \ SELECT "a", "b", "c", (round(("start" * 100)) / 100) AS "trunc_start", (round(("end" * 100)) / 100) AS "trunc_end" FROM values5 """ From cea50d4fdaab70e301365b195ac24ccafe6698aa Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Aug 2024 07:12:35 -0400 Subject: [PATCH 17/28] macos-13 --- .github/workflows/build_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 9babb978..e10eaa05 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -35,7 +35,7 @@ jobs: os: - ubuntu-20.04 - windows-2022 - - macos-12 + - macos-13 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -179,7 +179,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-64: - runs-on: macos-12 + runs-on: macos-13 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -206,7 +206,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-arm64: - runs-on: macos-latest + runs-on: macos-13-arm64 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -622,7 +622,7 @@ jobs: vegafusion-server-* build-vegafusion-server-osx-arm64: - runs-on: macos-latest + runs-on: macos-13-arm64 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 From 6a4b6eeb012dc9415718641f231985e7a93e5e17 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Aug 2024 07:45:01 -0400 Subject: [PATCH 18/28] back to macos-latest --- .github/workflows/build_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index e10eaa05..eb0d1bd0 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -206,7 +206,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-arm64: - runs-on: macos-13-arm64 + runs-on: macos-latest steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -622,7 +622,7 @@ jobs: vegafusion-server-* build-vegafusion-server-osx-arm64: - runs-on: macos-13-arm64 + runs-on: macos-latest steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 From 92c750200efe84482b5e16032d06476416319a16 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Aug 2024 07:47:40 -0400 Subject: [PATCH 19/28] fix test --- .../tests/test_pre_transform_values.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/vegafusion-runtime/tests/test_pre_transform_values.rs b/vegafusion-runtime/tests/test_pre_transform_values.rs index 7595b60a..721691f0 100644 --- a/vegafusion-runtime/tests/test_pre_transform_values.rs +++ b/vegafusion-runtime/tests/test_pre_transform_values.rs @@ -342,8 +342,8 @@ mod tests { +---------------------+---------------------+---------+---------+---------------+-------------+ | yearmonth_date | yearmonth_date_end | weather | __count | __count_start | __count_end | +---------------------+---------------------+---------+---------+---------------+-------------+ -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12 | 27 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0 | 16 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12.0 | 27 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0.0 | 16 | +---------------------+---------------------+---------+---------+---------------+-------------+"; assert_eq!(click_selected.pretty_format(None).unwrap(), expected); @@ -355,16 +355,16 @@ mod tests { +---------------------+---------------------+---------+---------+---------------+-------------+ | yearmonth_date | yearmonth_date_end | weather | __count | __count_start | __count_end | +---------------------+---------------------+---------+---------+---------------+-------------+ -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | sun | 12 | 0 | 12 | -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12 | 27 | -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | fog | 2 | 27 | 29 | -| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | drizzle | 1 | 29 | 30 | -| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | sun | 17 | 0 | 17 | -| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | snow | 1 | 17 | 18 | -| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | rain | 13 | 18 | 31 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0 | 16 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | rain | 13 | 16 | 29 | -| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | fog | 2 | 29 | 31 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | sun | 12 | 0.0 | 12 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | rain | 15 | 12.0 | 27 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | fog | 2 | 27.0 | 29 | +| 2013-11-01T00:00:00 | 2013-12-01T00:00:00 | drizzle | 1 | 29.0 | 30 | +| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | sun | 17 | 0.0 | 17 | +| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | snow | 1 | 17.0 | 18 | +| 2013-12-01T00:00:00 | 2014-01-01T00:00:00 | rain | 13 | 18.0 | 31 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | sun | 16 | 0.0 | 16 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | rain | 13 | 16.0 | 29 | +| 2014-01-01T00:00:00 | 2014-02-01T00:00:00 | fog | 2 | 29.0 | 31 | +---------------------+---------------------+---------+---------+---------------+-------------+"; assert_eq!(drag_selected.pretty_format(None).unwrap(), expected); } From a9e08ad2cfef202f2f4d0ddb069b0d5411109adb Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Aug 2024 19:04:14 -0400 Subject: [PATCH 20/28] fix pivot --- vegafusion-runtime/src/transform/pivot.rs | 2 +- vegafusion-runtime/tests/test_transform_pivot.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/vegafusion-runtime/src/transform/pivot.rs b/vegafusion-runtime/src/transform/pivot.rs index ea1b720e..778cc209 100644 --- a/vegafusion-runtime/src/transform/pivot.rs +++ b/vegafusion-runtime/src/transform/pivot.rs @@ -201,5 +201,5 @@ async fn pivot_case( /// Test whether null values should be replaced by zero for the specified aggregation fn should_fill_zero(op: &AggregateOp) -> bool { - matches!(op, AggregateOp::Count | AggregateOp::Sum) + matches!(op, AggregateOp::Sum) } diff --git a/vegafusion-runtime/tests/test_transform_pivot.rs b/vegafusion-runtime/tests/test_transform_pivot.rs index 8afdf418..dccf067a 100644 --- a/vegafusion-runtime/tests/test_transform_pivot.rs +++ b/vegafusion-runtime/tests/test_transform_pivot.rs @@ -219,13 +219,13 @@ mod test_pivot_with_empty_string { assert_eq!( result_data.pretty_format(None).unwrap(), "\ -+-----+------+-------+-----+---+ -| | blue | green | red | A | -+-----+------+-------+-----+---+ -| 3.0 | 0.0 | 2.0 | 1.0 | 1 | -| 0.0 | 5.0 | 0.0 | 4.0 | 2 | -| 8.0 | 9.0 | 6.0 | 7.0 | 3 | -+-----+------+-------+-----+---+" ++---+------+-------+-----+---+ +| | blue | green | red | A | ++---+------+-------+-----+---+ +| 3 | 0 | 2 | 1 | 1 | +| 0 | 5 | 0 | 4 | 2 | +| 8 | 9 | 6 | 7 | 3 | ++---+------+-------+-----+---+" ); // Check that first column is a single space (" "), rather than an empty string. From 98cc72dac0aa5dc995dace1a0641db53cb19c2ca Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 13 Aug 2024 19:45:02 -0400 Subject: [PATCH 21/28] Cast pivot value to type of columns --- vegafusion-runtime/src/transform/pivot.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vegafusion-runtime/src/transform/pivot.rs b/vegafusion-runtime/src/transform/pivot.rs index 778cc209..2e608c73 100644 --- a/vegafusion-runtime/src/transform/pivot.rs +++ b/vegafusion-runtime/src/transform/pivot.rs @@ -10,7 +10,7 @@ use vegafusion_common::arrow::datatypes::DataType; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::scalar::ScalarValue; use vegafusion_common::data::ORDER_COL; -use vegafusion_common::datatypes::{cast_to, data_type, is_string_datatype}; +use vegafusion_common::datatypes::{cast_to, data_type, is_string_datatype, to_numeric}; use vegafusion_common::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_common::escape::unescape_field; use vegafusion_core::proto::gen::transforms::{AggregateOp, Pivot}; @@ -156,6 +156,8 @@ async fn pivot_case( return Err(VegaFusionError::internal("Unexpected empty pivot dataset")); } + let schema = dataframe.schema_df()?; + // Process aggregate operation let agg_op: AggregateOp = tx .op @@ -168,7 +170,7 @@ async fn pivot_case( for pivot_val in pivot_vec.iter() { let predicate_expr = unescaped_col(&tx.field).eq(lit(pivot_val.as_str())); - let value_expr = unescaped_col(tx.value.as_str()); + let value_expr = to_numeric(unescaped_col(tx.value.as_str()), &schema)?; let agg_col = when(predicate_expr, value_expr).otherwise(if fill_zero { // Replace null with zero for certain aggregates lit(0) @@ -176,7 +178,7 @@ async fn pivot_case( lit(ScalarValue::Null) })?; - let agg_expr = make_agg_expr_for_col_expr(agg_col, &agg_op, &dataframe.schema_df()?)?; + let agg_expr = make_agg_expr_for_col_expr(agg_col, &agg_op, &schema)?; // Compute pivot column name, replacing null placeholder with "null" let col_name = if pivot_val == NULL_PLACEHOLDER_NAME { From c8c1fcab2080234c8b92903f37d78a09e7feeb7d Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 06:26:05 -0400 Subject: [PATCH 22/28] fix stack expected --- vegafusion-sql/tests/expected/stack.toml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/vegafusion-sql/tests/expected/stack.toml b/vegafusion-sql/tests/expected/stack.toml index a0a06e6c..92e430ea 100644 --- a/vegafusion-sql/tests/expected/stack.toml +++ b/vegafusion-sql/tests/expected/stack.toml @@ -70,17 +70,17 @@ values2 AS (SELECT "a", "b", "c", ("end" - coalesce("a", 0.0)) AS "start", "end" SELECT * FROM values2 ORDER BY "c" ASC NULLS FIRST, "end" ASC NULLS FIRST """ result = ''' -+-----+---+----+-------+-------+ -| a | b | c | start | end | -+-----+---+----+-------+-------+ -| -11 | 4 | A | 0.0 | -11.0 | -| 5 | 7 | A | 0.0 | 5.0 | -| 1 | 9 | A | 5.0 | 6.0 | -| -3 | 8 | BB | -7.0 | -10.0 | -| -7 | 6 | BB | 0.0 | -7.0 | -| 13 | 3 | BB | 0.0 | 13.0 | -| 9 | 5 | BB | 13.0 | 22.0 | -+-----+---+----+-------+-------+ ++-----+---+----+-------+-----+ +| a | b | c | start | end | ++-----+---+----+-------+-----+ +| -11 | 4 | A | 0.0 | -11 | +| 5 | 7 | A | 0.0 | 5 | +| 1 | 9 | A | 5.0 | 6 | +| -3 | 8 | BB | -7.0 | -10 | +| -7 | 6 | BB | 0.0 | -7 | +| 13 | 3 | BB | 0.0 | 13 | +| 9 | 5 | BB | 13.0 | 22 | ++-----+---+----+-------+-----+ ''' [mode_center] From 78f02037ddbf83ac710b712b89d4c5c5023a623a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 09:01:48 -0400 Subject: [PATCH 23/28] Use arm image for macos rust tests, intel image for python tests --- .github/workflows/build_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index eb0d1bd0..f9a3d58a 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -35,7 +35,7 @@ jobs: os: - ubuntu-20.04 - windows-2022 - - macos-13 + - macos-14 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -179,7 +179,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-64: - runs-on: macos-13 + runs-on: macos-12 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -206,7 +206,7 @@ jobs: target/wheels/*.whl build-vegafusion-python-embed-osx-arm64: - runs-on: macos-latest + runs-on: macos-14 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 @@ -327,7 +327,7 @@ jobs: run: pytest test-vegafusion-python-osx-64: - runs-on: macos-13 + runs-on: macos-12 needs: [build-vegafusion-python-embed-osx-64, build-vegafusion-packages] steps: - name: Check out repository code @@ -622,7 +622,7 @@ jobs: vegafusion-server-* build-vegafusion-server-osx-arm64: - runs-on: macos-latest + runs-on: macos-14 steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 From b593e95f2a035131aa9dd57045ac47a7bdaff6a6 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 09:11:19 -0400 Subject: [PATCH 24/28] Try macos requirement in pixi.toml --- pixi.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pixi.toml b/pixi.toml index a5f686df..c1b14a62 100644 --- a/pixi.toml +++ b/pixi.toml @@ -6,6 +6,9 @@ authors = ["Jon Mease "] channels = ["conda-forge"] platforms = ["osx-arm64", "osx-64", "linux-64", "win-64"] +[system-requirements] +macos = "12.0" + [tasks] check-rs-fmt = "cargo fmt --all -- --check" check-rs-warnings = "export RUSTFLAGS=\"-D warnings\" && cargo check --tests" From 974c0441c1610ae47264ef17069efcd6068e4a76 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 09:12:25 -0400 Subject: [PATCH 25/28] break cache --- .github/workflows/build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index f9a3d58a..2cad7dd7 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -45,7 +45,7 @@ jobs: - name: Cache uses: actions/cache@v3 with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2 path: | ~/.cargo target From cb11fcfb29ed256a44db7c5d3dd014ada87a8950 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 09:36:24 -0400 Subject: [PATCH 26/28] debug --- .github/workflows/build_test.yml | 977 +++++++++++++++---------------- 1 file changed, 488 insertions(+), 489 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 2cad7dd7..1f0412cf 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -3,154 +3,154 @@ on: pull_request: types: [ opened, synchronize ] jobs: - check-rust: - runs-on: ubuntu-20.04 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - cache: true - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust - path: | - ~/.cargo - target - .pixi - - name: Check fmt - run: pixi run check-rs-fmt - - name: Check warnings - run: pixi run check-rs-warnings - - name: Check clippy - run: pixi run check-rs-clippy - - test-rust: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-20.04 - - windows-2022 - - macos-14 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2 - path: | - ~/.cargo - target - .pixi - - name: start minio server in the background - run: pixi run start-minio & - - name: Test rust - run: | - pixi run test-rs --release - - name: Upload test artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - if: always() - with: - name: vegafusion-rt-test-images - path: | - vegafusion-runtime/tests/output - - build-vegafusion-wasm: - runs-on: ubuntu-20.04 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm - path: | - ~/.cargo - target - .pixi - - name: Build package - run: | - pixi run pack-wasm - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-wasm-packages - path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz - - # Use maturin action to build linux wheels within proper manylinux compatible containers - # (This is why we don't use the pixi "build-py-embed" action) - build-vegafusion-python-embed-linux-64: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - name: Install latest stable Rust toolchain - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 - with: - toolchain: stable - override: true - - name: Cache rust dependencies - uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 - with: - cache-on-failure: True - - name: Build wheels (Linux) - uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 - with: - command: build - manylinux: 2014 - rust-toolchain: stable - args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-python-embed-wheels - path: | - target/wheels/*.tar.gz - target/wheels/*.whl - - build-vegafusion-python-embed-linux-arm64: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - name: Install latest stable Rust toolchain - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 - with: - toolchain: stable - override: true - - name: Cache rust dependencies - uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 - with: - cache-on-failure: True - - name: Download arm64 toolchain - run: | - rustup target add aarch64-unknown-linux-gnu - - name: Build arm64 wheels - uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 - with: - command: build - manylinux: 2_28 - rust-toolchain: stable - args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-python-embed-wheels - path: | - target/wheels/*.tar.gz - target/wheels/*.whl - +# check-rust: +# runs-on: ubuntu-20.04 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# cache: true +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust +# path: | +# ~/.cargo +# target +# .pixi +# - name: Check fmt +# run: pixi run check-rs-fmt +# - name: Check warnings +# run: pixi run check-rs-warnings +# - name: Check clippy +# run: pixi run check-rs-clippy +# +# test-rust: +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: +# - ubuntu-20.04 +# - windows-2022 +# - macos-14 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2 +# path: | +# ~/.cargo +# target +# .pixi +# - name: start minio server in the background +# run: pixi run start-minio & +# - name: Test rust +# run: | +# pixi run test-rs --release +# - name: Upload test artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# if: always() +# with: +# name: vegafusion-rt-test-images +# path: | +# vegafusion-runtime/tests/output +# +# build-vegafusion-wasm: +# runs-on: ubuntu-20.04 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm +# path: | +# ~/.cargo +# target +# .pixi +# - name: Build package +# run: | +# pixi run pack-wasm +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-wasm-packages +# path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz +# +# # Use maturin action to build linux wheels within proper manylinux compatible containers +# # (This is why we don't use the pixi "build-py-embed" action) +# build-vegafusion-python-embed-linux-64: +# runs-on: ubuntu-latest +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - name: Install latest stable Rust toolchain +# uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 +# with: +# toolchain: stable +# override: true +# - name: Cache rust dependencies +# uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 +# with: +# cache-on-failure: True +# - name: Build wheels (Linux) +# uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 +# with: +# command: build +# manylinux: 2014 +# rust-toolchain: stable +# args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-python-embed-wheels +# path: | +# target/wheels/*.tar.gz +# target/wheels/*.whl +# +# build-vegafusion-python-embed-linux-arm64: +# runs-on: ubuntu-latest +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - name: Install latest stable Rust toolchain +# uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 +# with: +# toolchain: stable +# override: true +# - name: Cache rust dependencies +# uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 +# with: +# cache-on-failure: True +# - name: Download arm64 toolchain +# run: | +# rustup target add aarch64-unknown-linux-gnu +# - name: Build arm64 wheels +# uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 +# with: +# command: build +# manylinux: 2_28 +# rust-toolchain: stable +# args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-python-embed-wheels +# path: | +# target/wheels/*.tar.gz +# target/wheels/*.whl +# build-vegafusion-python-embed-win-64: runs-on: windows-2022 steps: @@ -178,33 +178,33 @@ jobs: target/wheels/*.tar.gz target/wheels/*.whl - build-vegafusion-python-embed-osx-64: - runs-on: macos-12 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-osx-64 - path: | - ~/.cargo - target - .pixi - - name: Build vegafusion-python-embed - run: | - pixi run build-py-embed - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-python-embed-wheels - path: | - target/wheels/*.tar.gz - target/wheels/*.whl - +# build-vegafusion-python-embed-osx-64: +# runs-on: macos-12 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-osx-64 +# path: | +# ~/.cargo +# target +# .pixi +# - name: Build vegafusion-python-embed +# run: | +# pixi run build-py-embed +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-python-embed-wheels +# path: | +# target/wheels/*.tar.gz +# target/wheels/*.whl +# build-vegafusion-python-embed-osx-arm64: runs-on: macos-14 steps: @@ -258,50 +258,84 @@ jobs: path: | python/vegafusion/dist/*.whl python/vegafusion/dist/*.tar.gz +# +# build-vegafusion-jupyter-packages: +# runs-on: ubuntu-20.04 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-jupyter-packages +# path: | +# ~/.cargo +# target +# .pixi +# - name: Build vegafusion-jupyter wheel +# run: | +# echo JupyterLab Version: `pixi run jupyter-lab --version` +# pixi run build-py-jupyter +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-jupyter-packages +# path: | +# python/vegafusion-jupyter/dist/*.whl +# python/vegafusion-jupyter/dist/*.tar.gz +# +# test-vegafusion-python-linux-64: +# runs-on: ubuntu-20.04 +# needs: [build-vegafusion-python-embed-linux-64, build-vegafusion-packages] +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - name: Setup Python +# uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7 +# with: +# python-version: '3.11' +# - name: Install fonts on Linux +# if: runner.os == 'Linux' +# run: | +# echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections +# sudo apt-get install ttf-mscorefonts-installer +# - name: Download vegafusion-python-embed wheel +# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 +# with: +# name: vegafusion-python-embed-wheels +# path: target/wheels/ +# - name: Download vegafusion wheel +# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 +# with: +# name: vegafusion-packages +# path: target/wheels/ +# - name: install wheels +# working-directory: target/wheels/ +# run: | +# ls -la +# python -m pip install vegafusion-*.whl +# python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl +# python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" +# - name: Test lazy imports +# working-directory: python/vegafusion/ +# run: python checks/check_lazy_imports.py +# - name: Test vegafusion +# working-directory: python/vegafusion/ +# run: pytest - build-vegafusion-jupyter-packages: - runs-on: ubuntu-20.04 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-jupyter-packages - path: | - ~/.cargo - target - .pixi - - name: Build vegafusion-jupyter wheel - run: | - echo JupyterLab Version: `pixi run jupyter-lab --version` - pixi run build-py-jupyter - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-jupyter-packages - path: | - python/vegafusion-jupyter/dist/*.whl - python/vegafusion-jupyter/dist/*.tar.gz - - test-vegafusion-python-linux-64: - runs-on: ubuntu-20.04 - needs: [build-vegafusion-python-embed-linux-64, build-vegafusion-packages] + test-vegafusion-python-osx-arm64: + runs-on: macos-12 + needs: [build-vegafusion-python-embed-osx-arm64, build-vegafusion-packages] steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - name: Setup Python uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7 with: - python-version: '3.11' - - name: Install fonts on Linux - if: runner.os == 'Linux' - run: | - echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections - sudo apt-get install ttf-mscorefonts-installer + python-version: '3.10' - name: Download vegafusion-python-embed wheel uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 with: @@ -317,25 +351,22 @@ jobs: run: | ls -la python -m pip install vegafusion-*.whl - python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl - python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" - - name: Test lazy imports - working-directory: python/vegafusion/ - run: python checks/check_lazy_imports.py + python -m pip install vegafusion_python_embed-*macosx_11_*_arm64.whl + python -m pip install pytest vega-datasets polars "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2" - name: Test vegafusion working-directory: python/vegafusion/ run: pytest - test-vegafusion-python-osx-64: - runs-on: macos-12 - needs: [build-vegafusion-python-embed-osx-64, build-vegafusion-packages] + test-vegafusion-python-win-64: + runs-on: windows-2022 + needs: [build-vegafusion-python-embed-win-64, build-vegafusion-packages] steps: - name: Check out repository code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - name: Setup Python uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7 with: - python-version: '3.10' + python-version: '3.8' - name: Download vegafusion-python-embed wheel uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 with: @@ -348,39 +379,7 @@ jobs: path: target/wheels/ - name: install wheels working-directory: target/wheels/ - run: | - ls -la - python -m pip install vegafusion-*.whl - python -m pip install vegafusion_python_embed-*macosx_10_*_x86_64.whl - python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2" - python -m pip install pyarrow==10.0 altair==5.1.2 - - name: Test vegafusion - working-directory: python/vegafusion/ - run: pytest - - test-vegafusion-python-win-64: - runs-on: windows-2022 - needs: [build-vegafusion-python-embed-win-64, build-vegafusion-packages] - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - name: Setup Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7 - with: - python-version: '3.8' - - name: Download vegafusion-python-embed wheel - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 - with: - name: vegafusion-python-embed-wheels - path: target/wheels/ - - name: Download vegafusion wheel - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 - with: - name: vegafusion-packages - path: target/wheels/ - - name: install wheels - working-directory: target/wheels/ - shell: powershell + shell: powershell run: | Get-ChildItem -Force $vegafusion = Get-ChildItem -Name "vegafusion-*.whl" | Select-Object -First 1 @@ -388,123 +387,164 @@ jobs: python -m pip install $vegafusion python -m pip install $vegafusion_python_embed - python -m pip install pytest vega-datasets polars[timezone] "duckdb>=1.0" vl-convert-python scikit-image + python -m pip install pytest pyarrow==10.0 altair==5.1.2 vega-datasets polars[timezone] "duckdb>=1.0" vl-convert-python scikit-image - name: Test vegafusion working-directory: python/vegafusion/ run: pytest - - test-vegafusion-jupyter-linux-64: - runs-on: ubuntu-20.04 - needs: - - build-vegafusion-python-embed-linux-64 - - build-vegafusion-packages - - build-vegafusion-jupyter-packages - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-vegafusion-jupyter-linux-64 - path: | - .pixi - - name: Install Chrome - uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest - with: - chrome-version: stable - - name: Download vegafusion-python-embed wheel - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 - with: - name: vegafusion-python-embed-wheels - path: target/wheels/ - - name: Download vegafusion wheel - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 - with: - name: vegafusion-packages - path: target/wheels/ - - name: Download vegafusion-jupyter wheel - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 - with: - name: vegafusion-jupyter-packages - path: target/wheels/ - - name: install wheels - run: | - ls -la target/wheels/ - pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl - pixi run pip install target/wheels/vegafusion-*.whl - pixi run pip install target/wheels/vegafusion_jupyter-*.whl - pixi run pip install chromedriver-binary-auto - pixi run pip list - - name: Test vegafusion jupyter - run: | - VEGAFUSION_TEST_HEADLESS=1 pixi run pytest python/vegafusion-jupyter/tests - - name: Upload test artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - if: always() - with: - name: vegafusion-jupyter-test-failures - path: | - python/vegafusion-jupyter/tests/failures/* - - build-vegafusion-server-linux-64: - runs-on: ubuntu-20.04 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64 - path: | - ~/.cargo - target - .pixi - - name: Build vegafusion-server - run: | - pixi run build-rs-vegafusion-server - - name: zip executable - uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 - with: - files: target/release/vegafusion-server - dest: vegafusion-server-linux-64.zip - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-server - path: | - vegafusion-server-* - -# # Below is an Attempt to cross compile to linux aarch64 within conda environment. -# # It fails in compiling protobuf-src. So instead we fall back to the direct use of -# # cargo (without involving conda). # -# build-vegafusion-server-linux-arm64: +# test-vegafusion-jupyter-linux-64: +# runs-on: ubuntu-20.04 +# needs: +# - build-vegafusion-python-embed-linux-64 +# - build-vegafusion-packages +# - build-vegafusion-jupyter-packages +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-vegafusion-jupyter-linux-64 +# path: | +# .pixi +# - name: Install Chrome +# uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest +# with: +# chrome-version: stable +# - name: Download vegafusion-python-embed wheel +# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 +# with: +# name: vegafusion-python-embed-wheels +# path: target/wheels/ +# - name: Download vegafusion wheel +# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 +# with: +# name: vegafusion-packages +# path: target/wheels/ +# - name: Download vegafusion-jupyter wheel +# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 +# with: +# name: vegafusion-jupyter-packages +# path: target/wheels/ +# - name: install wheels +# run: | +# ls -la target/wheels/ +# pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl +# pixi run pip install target/wheels/vegafusion-*.whl +# pixi run pip install target/wheels/vegafusion_jupyter-*.whl +# pixi run pip install chromedriver-binary-auto +# pixi run pip list +# - name: Test vegafusion jupyter +# run: | +# VEGAFUSION_TEST_HEADLESS=1 pixi run pytest python/vegafusion-jupyter/tests +# - name: Upload test artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# if: always() +# with: +# name: vegafusion-jupyter-test-failures +# path: | +# python/vegafusion-jupyter/tests/failures/* +# +# build-vegafusion-server-linux-64: # runs-on: ubuntu-20.04 # steps: # - name: Check out repository code # uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - name: Install pixi -# run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 # - name: Cache # uses: actions/cache@v3 # with: -# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64 # path: | # ~/.cargo # target # .pixi # - name: Build vegafusion-server # run: | -# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu -# pixi add gcc_linux-aarch64 -p linux-64 -# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc" -# pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu +# pixi run build-rs-vegafusion-server +# - name: zip executable +# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 +# with: +# files: target/release/vegafusion-server +# dest: vegafusion-server-linux-64.zip +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-server +# path: | +# vegafusion-server-* +# +## # Below is an Attempt to cross compile to linux aarch64 within conda environment. +## # It fails in compiling protobuf-src. So instead we fall back to the direct use of +## # cargo (without involving conda). +## +## build-vegafusion-server-linux-arm64: +## runs-on: ubuntu-20.04 +## steps: +## - name: Check out repository code +## uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +## - name: Install pixi +## run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH +## - name: Cache +## uses: actions/cache@v3 +## with: +## key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server +## path: | +## ~/.cargo +## target +## .pixi +## - name: Build vegafusion-server +## run: | +## pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu +## pixi add gcc_linux-aarch64 -p linux-64 +## export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc" +## pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu +## - name: zip executable +## uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 +## with: +## files: target/aarch64-unknown-linux-gnu/release/vegafusion-server +## dest: vegafusion-server-linux-arm64.zip +## - name: Upload artifacts +## uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +## with: +## name: vegafusion-server +## path: | +## vegafusion-server-* +# +# +# build-vegafusion-server-linux-arm64: +# runs-on: ubuntu-latest +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - name: Install latest stable Rust toolchain +# uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 +# with: +# toolchain: stable +# override: true +# - name: Cache rust dependencies +# uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 +# with: +# cache-on-failure: True +# - name: Download arm64 toolchain +# run: | +# rustup target add aarch64-unknown-linux-gnu +# - name: Download gcc for cross compilation +# run: | +# sudo apt-get update +# sudo apt-get install gcc-aarch64-linux-gnu +# - name: Build vegafusion-server +# uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1 +# env: +# RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc" +# with: +# command: build +# args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu # - name: zip executable # uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 # with: @@ -516,138 +556,97 @@ jobs: # name: vegafusion-server # path: | # vegafusion-server-* - - - build-vegafusion-server-linux-arm64: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - name: Install latest stable Rust toolchain - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 - with: - toolchain: stable - override: true - - name: Cache rust dependencies - uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 - with: - cache-on-failure: True - - name: Download arm64 toolchain - run: | - rustup target add aarch64-unknown-linux-gnu - - name: Download gcc for cross compilation - run: | - sudo apt-get update - sudo apt-get install gcc-aarch64-linux-gnu - - name: Build vegafusion-server - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1 - env: - RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc" - with: - command: build - args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu - - name: zip executable - uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 - with: - files: target/aarch64-unknown-linux-gnu/release/vegafusion-server - dest: vegafusion-server-linux-arm64.zip - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-server - path: | - vegafusion-server-* - - build-vegafusion-server-win-64: - runs-on: windows-2022 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b - path: | - ~/.cargo - target - .pixi - - name: Build vegafusion-server - run: | - pixi install -vvv - pixi run build-rs-vegafusion-server - - name: zip executable - uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 - with: - files: target/release/vegafusion-server.exe - dest: vegafusion-server-win-64.zip - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-server - path: | - vegafusion-server-* - - build-vegafusion-server-osx-64: - runs-on: macos-13 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64 - path: | - ~/.cargo - target - .pixi - - name: Build vegafusion-server - run: | - pixi run build-rs-vegafusion-server - - name: zip executable - uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 - with: - files: target/release/vegafusion-server - dest: vegafusion-server-osx-64.zip - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-server - path: | - vegafusion-server-* - - build-vegafusion-server-osx-arm64: - runs-on: macos-14 - steps: - - name: Check out repository code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.27.1 - - name: Cache - uses: actions/cache@v3 - with: - key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64 - path: | - ~/.cargo - target - .pixi - - name: Build vegafusion-server - run: | - pixi run build-rs-vegafusion-server --target aarch64-apple-darwin - - name: zip executable - uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 - with: - files: target/aarch64-apple-darwin/release/vegafusion-server - dest: vegafusion-server-osx-arm64.zip - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 - with: - name: vegafusion-server - path: | - vegafusion-server-* +# +# build-vegafusion-server-win-64: +# runs-on: windows-2022 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b +# path: | +# ~/.cargo +# target +# .pixi +# - name: Build vegafusion-server +# run: | +# pixi install -vvv +# pixi run build-rs-vegafusion-server +# - name: zip executable +# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 +# with: +# files: target/release/vegafusion-server.exe +# dest: vegafusion-server-win-64.zip +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-server +# path: | +# vegafusion-server-* +# +# build-vegafusion-server-osx-64: +# runs-on: macos-13 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64 +# path: | +# ~/.cargo +# target +# .pixi +# - name: Build vegafusion-server +# run: | +# pixi run build-rs-vegafusion-server +# - name: zip executable +# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 +# with: +# files: target/release/vegafusion-server +# dest: vegafusion-server-osx-64.zip +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-server +# path: | +# vegafusion-server-* +# +# build-vegafusion-server-osx-arm64: +# runs-on: macos-14 +# steps: +# - name: Check out repository code +# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 +# - uses: prefix-dev/setup-pixi@v0.8.1 +# with: +# pixi-version: v0.27.1 +# - name: Cache +# uses: actions/cache@v3 +# with: +# key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64 +# path: | +# ~/.cargo +# target +# .pixi +# - name: Build vegafusion-server +# run: | +# pixi run build-rs-vegafusion-server --target aarch64-apple-darwin +# - name: zip executable +# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 +# with: +# files: target/aarch64-apple-darwin/release/vegafusion-server +# dest: vegafusion-server-osx-arm64.zip +# - name: Upload artifacts +# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 +# with: +# name: vegafusion-server +# path: | +# vegafusion-server-* From c51191abcb22deaa645122344662e3015ccfc011 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 09:41:41 -0400 Subject: [PATCH 27/28] macos-14 --- .github/workflows/build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 1f0412cf..7a33bd07 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -327,7 +327,7 @@ jobs: # run: pytest test-vegafusion-python-osx-arm64: - runs-on: macos-12 + runs-on: macos-14 needs: [build-vegafusion-python-embed-osx-arm64, build-vegafusion-packages] steps: - name: Check out repository code From 8bb76d70eb8071b8489bf24b2c7feee17376d8a4 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 14 Aug 2024 10:11:08 -0400 Subject: [PATCH 28/28] all the tests! --- .github/workflows/build_test.yml | 952 +++++++++++++++---------------- 1 file changed, 476 insertions(+), 476 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 7a33bd07..61a23d30 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -3,154 +3,154 @@ on: pull_request: types: [ opened, synchronize ] jobs: -# check-rust: -# runs-on: ubuntu-20.04 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# cache: true -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust -# path: | -# ~/.cargo -# target -# .pixi -# - name: Check fmt -# run: pixi run check-rs-fmt -# - name: Check warnings -# run: pixi run check-rs-warnings -# - name: Check clippy -# run: pixi run check-rs-clippy -# -# test-rust: -# runs-on: ${{ matrix.os }} -# strategy: -# fail-fast: false -# matrix: -# os: -# - ubuntu-20.04 -# - windows-2022 -# - macos-14 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2 -# path: | -# ~/.cargo -# target -# .pixi -# - name: start minio server in the background -# run: pixi run start-minio & -# - name: Test rust -# run: | -# pixi run test-rs --release -# - name: Upload test artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# if: always() -# with: -# name: vegafusion-rt-test-images -# path: | -# vegafusion-runtime/tests/output -# -# build-vegafusion-wasm: -# runs-on: ubuntu-20.04 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm -# path: | -# ~/.cargo -# target -# .pixi -# - name: Build package -# run: | -# pixi run pack-wasm -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-wasm-packages -# path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz -# -# # Use maturin action to build linux wheels within proper manylinux compatible containers -# # (This is why we don't use the pixi "build-py-embed" action) -# build-vegafusion-python-embed-linux-64: -# runs-on: ubuntu-latest -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - name: Install latest stable Rust toolchain -# uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 -# with: -# toolchain: stable -# override: true -# - name: Cache rust dependencies -# uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 -# with: -# cache-on-failure: True -# - name: Build wheels (Linux) -# uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 -# with: -# command: build -# manylinux: 2014 -# rust-toolchain: stable -# args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-python-embed-wheels -# path: | -# target/wheels/*.tar.gz -# target/wheels/*.whl -# -# build-vegafusion-python-embed-linux-arm64: -# runs-on: ubuntu-latest -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - name: Install latest stable Rust toolchain -# uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 -# with: -# toolchain: stable -# override: true -# - name: Cache rust dependencies -# uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 -# with: -# cache-on-failure: True -# - name: Download arm64 toolchain -# run: | -# rustup target add aarch64-unknown-linux-gnu -# - name: Build arm64 wheels -# uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 -# with: -# command: build -# manylinux: 2_28 -# rust-toolchain: stable -# args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-python-embed-wheels -# path: | -# target/wheels/*.tar.gz -# target/wheels/*.whl -# + check-rust: + runs-on: ubuntu-20.04 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + cache: true + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust + path: | + ~/.cargo + target + .pixi + - name: Check fmt + run: pixi run check-rs-fmt + - name: Check warnings + run: pixi run check-rs-warnings + - name: Check clippy + run: pixi run check-rs-clippy + + test-rust: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-20.04 + - windows-2022 + - macos-14 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2 + path: | + ~/.cargo + target + .pixi + - name: start minio server in the background + run: pixi run start-minio & + - name: Test rust + run: | + pixi run test-rs --release + - name: Upload test artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + if: always() + with: + name: vegafusion-rt-test-images + path: | + vegafusion-runtime/tests/output + + build-vegafusion-wasm: + runs-on: ubuntu-20.04 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm + path: | + ~/.cargo + target + .pixi + - name: Build package + run: | + pixi run pack-wasm + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-wasm-packages + path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz + + # Use maturin action to build linux wheels within proper manylinux compatible containers + # (This is why we don't use the pixi "build-py-embed" action) + build-vegafusion-python-embed-linux-64: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - name: Install latest stable Rust toolchain + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 + with: + toolchain: stable + override: true + - name: Cache rust dependencies + uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 + with: + cache-on-failure: True + - name: Build wheels (Linux) + uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 + with: + command: build + manylinux: 2014 + rust-toolchain: stable + args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-python-embed-wheels + path: | + target/wheels/*.tar.gz + target/wheels/*.whl + + build-vegafusion-python-embed-linux-arm64: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - name: Install latest stable Rust toolchain + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 + with: + toolchain: stable + override: true + - name: Cache rust dependencies + uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 + with: + cache-on-failure: True + - name: Download arm64 toolchain + run: | + rustup target add aarch64-unknown-linux-gnu + - name: Build arm64 wheels + uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1 + with: + command: build + manylinux: 2_28 + rust-toolchain: stable + args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-python-embed-wheels + path: | + target/wheels/*.tar.gz + target/wheels/*.whl + build-vegafusion-python-embed-win-64: runs-on: windows-2022 steps: @@ -178,33 +178,33 @@ jobs: target/wheels/*.tar.gz target/wheels/*.whl -# build-vegafusion-python-embed-osx-64: -# runs-on: macos-12 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-osx-64 -# path: | -# ~/.cargo -# target -# .pixi -# - name: Build vegafusion-python-embed -# run: | -# pixi run build-py-embed -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-python-embed-wheels -# path: | -# target/wheels/*.tar.gz -# target/wheels/*.whl -# + build-vegafusion-python-embed-osx-64: + runs-on: macos-12 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-osx-64 + path: | + ~/.cargo + target + .pixi + - name: Build vegafusion-python-embed + run: | + pixi run build-py-embed + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-python-embed-wheels + path: | + target/wheels/*.tar.gz + target/wheels/*.whl + build-vegafusion-python-embed-osx-arm64: runs-on: macos-14 steps: @@ -258,73 +258,73 @@ jobs: path: | python/vegafusion/dist/*.whl python/vegafusion/dist/*.tar.gz -# -# build-vegafusion-jupyter-packages: -# runs-on: ubuntu-20.04 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-jupyter-packages -# path: | -# ~/.cargo -# target -# .pixi -# - name: Build vegafusion-jupyter wheel -# run: | -# echo JupyterLab Version: `pixi run jupyter-lab --version` -# pixi run build-py-jupyter -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-jupyter-packages -# path: | -# python/vegafusion-jupyter/dist/*.whl -# python/vegafusion-jupyter/dist/*.tar.gz -# -# test-vegafusion-python-linux-64: -# runs-on: ubuntu-20.04 -# needs: [build-vegafusion-python-embed-linux-64, build-vegafusion-packages] -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - name: Setup Python -# uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7 -# with: -# python-version: '3.11' -# - name: Install fonts on Linux -# if: runner.os == 'Linux' -# run: | -# echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections -# sudo apt-get install ttf-mscorefonts-installer -# - name: Download vegafusion-python-embed wheel -# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 -# with: -# name: vegafusion-python-embed-wheels -# path: target/wheels/ -# - name: Download vegafusion wheel -# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 -# with: -# name: vegafusion-packages -# path: target/wheels/ -# - name: install wheels -# working-directory: target/wheels/ -# run: | -# ls -la -# python -m pip install vegafusion-*.whl -# python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl -# python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" -# - name: Test lazy imports -# working-directory: python/vegafusion/ -# run: python checks/check_lazy_imports.py -# - name: Test vegafusion -# working-directory: python/vegafusion/ -# run: pytest + + build-vegafusion-jupyter-packages: + runs-on: ubuntu-20.04 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-jupyter-packages + path: | + ~/.cargo + target + .pixi + - name: Build vegafusion-jupyter wheel + run: | + echo JupyterLab Version: `pixi run jupyter-lab --version` + pixi run build-py-jupyter + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-jupyter-packages + path: | + python/vegafusion-jupyter/dist/*.whl + python/vegafusion-jupyter/dist/*.tar.gz + + test-vegafusion-python-linux-64: + runs-on: ubuntu-20.04 + needs: [build-vegafusion-python-embed-linux-64, build-vegafusion-packages] + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - name: Setup Python + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7 + with: + python-version: '3.11' + - name: Install fonts on Linux + if: runner.os == 'Linux' + run: | + echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections + sudo apt-get install ttf-mscorefonts-installer + - name: Download vegafusion-python-embed wheel + uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 + with: + name: vegafusion-python-embed-wheels + path: target/wheels/ + - name: Download vegafusion wheel + uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 + with: + name: vegafusion-packages + path: target/wheels/ + - name: install wheels + working-directory: target/wheels/ + run: | + ls -la + python -m pip install vegafusion-*.whl + python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl + python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" + - name: Test lazy imports + working-directory: python/vegafusion/ + run: python checks/check_lazy_imports.py + - name: Test vegafusion + working-directory: python/vegafusion/ + run: pytest test-vegafusion-python-osx-arm64: runs-on: macos-14 @@ -391,160 +391,119 @@ jobs: - name: Test vegafusion working-directory: python/vegafusion/ run: pytest + + test-vegafusion-jupyter-linux-64: + runs-on: ubuntu-20.04 + needs: + - build-vegafusion-python-embed-linux-64 + - build-vegafusion-packages + - build-vegafusion-jupyter-packages + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-vegafusion-jupyter-linux-64 + path: | + .pixi + - name: Install Chrome + uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest + with: + chrome-version: stable + - name: Download vegafusion-python-embed wheel + uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 + with: + name: vegafusion-python-embed-wheels + path: target/wheels/ + - name: Download vegafusion wheel + uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 + with: + name: vegafusion-packages + path: target/wheels/ + - name: Download vegafusion-jupyter wheel + uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 + with: + name: vegafusion-jupyter-packages + path: target/wheels/ + - name: install wheels + run: | + ls -la target/wheels/ + pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl + pixi run pip install target/wheels/vegafusion-*.whl + pixi run pip install target/wheels/vegafusion_jupyter-*.whl + pixi run pip install chromedriver-binary-auto + pixi run pip list + - name: Test vegafusion jupyter + run: | + VEGAFUSION_TEST_HEADLESS=1 pixi run pytest python/vegafusion-jupyter/tests + - name: Upload test artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + if: always() + with: + name: vegafusion-jupyter-test-failures + path: | + python/vegafusion-jupyter/tests/failures/* + + build-vegafusion-server-linux-64: + runs-on: ubuntu-20.04 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64 + path: | + ~/.cargo + target + .pixi + - name: Build vegafusion-server + run: | + pixi run build-rs-vegafusion-server + - name: zip executable + uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 + with: + files: target/release/vegafusion-server + dest: vegafusion-server-linux-64.zip + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-server + path: | + vegafusion-server-* + +# # Below is an Attempt to cross compile to linux aarch64 within conda environment. +# # It fails in compiling protobuf-src. So instead we fall back to the direct use of +# # cargo (without involving conda). # -# test-vegafusion-jupyter-linux-64: -# runs-on: ubuntu-20.04 -# needs: -# - build-vegafusion-python-embed-linux-64 -# - build-vegafusion-packages -# - build-vegafusion-jupyter-packages -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-vegafusion-jupyter-linux-64 -# path: | -# .pixi -# - name: Install Chrome -# uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest -# with: -# chrome-version: stable -# - name: Download vegafusion-python-embed wheel -# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 -# with: -# name: vegafusion-python-embed-wheels -# path: target/wheels/ -# - name: Download vegafusion wheel -# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 -# with: -# name: vegafusion-packages -# path: target/wheels/ -# - name: Download vegafusion-jupyter wheel -# uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 -# with: -# name: vegafusion-jupyter-packages -# path: target/wheels/ -# - name: install wheels -# run: | -# ls -la target/wheels/ -# pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl -# pixi run pip install target/wheels/vegafusion-*.whl -# pixi run pip install target/wheels/vegafusion_jupyter-*.whl -# pixi run pip install chromedriver-binary-auto -# pixi run pip list -# - name: Test vegafusion jupyter -# run: | -# VEGAFUSION_TEST_HEADLESS=1 pixi run pytest python/vegafusion-jupyter/tests -# - name: Upload test artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# if: always() -# with: -# name: vegafusion-jupyter-test-failures -# path: | -# python/vegafusion-jupyter/tests/failures/* -# -# build-vegafusion-server-linux-64: +# build-vegafusion-server-linux-arm64: # runs-on: ubuntu-20.04 # steps: # - name: Check out repository code # uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 +# - name: Install pixi +# run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH # - name: Cache # uses: actions/cache@v3 # with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64 +# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server # path: | # ~/.cargo # target # .pixi # - name: Build vegafusion-server # run: | -# pixi run build-rs-vegafusion-server -# - name: zip executable -# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 -# with: -# files: target/release/vegafusion-server -# dest: vegafusion-server-linux-64.zip -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-server -# path: | -# vegafusion-server-* -# -## # Below is an Attempt to cross compile to linux aarch64 within conda environment. -## # It fails in compiling protobuf-src. So instead we fall back to the direct use of -## # cargo (without involving conda). -## -## build-vegafusion-server-linux-arm64: -## runs-on: ubuntu-20.04 -## steps: -## - name: Check out repository code -## uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -## - name: Install pixi -## run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH -## - name: Cache -## uses: actions/cache@v3 -## with: -## key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server -## path: | -## ~/.cargo -## target -## .pixi -## - name: Build vegafusion-server -## run: | -## pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu -## pixi add gcc_linux-aarch64 -p linux-64 -## export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc" -## pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu -## - name: zip executable -## uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 -## with: -## files: target/aarch64-unknown-linux-gnu/release/vegafusion-server -## dest: vegafusion-server-linux-arm64.zip -## - name: Upload artifacts -## uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -## with: -## name: vegafusion-server -## path: | -## vegafusion-server-* -# -# -# build-vegafusion-server-linux-arm64: -# runs-on: ubuntu-latest -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - name: Install latest stable Rust toolchain -# uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 -# with: -# toolchain: stable -# override: true -# - name: Cache rust dependencies -# uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 -# with: -# cache-on-failure: True -# - name: Download arm64 toolchain -# run: | -# rustup target add aarch64-unknown-linux-gnu -# - name: Download gcc for cross compilation -# run: | -# sudo apt-get update -# sudo apt-get install gcc-aarch64-linux-gnu -# - name: Build vegafusion-server -# uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1 -# env: -# RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc" -# with: -# command: build -# args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu +# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu +# pixi add gcc_linux-aarch64 -p linux-64 +# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc" +# pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu # - name: zip executable # uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 # with: @@ -556,97 +515,138 @@ jobs: # name: vegafusion-server # path: | # vegafusion-server-* -# -# build-vegafusion-server-win-64: -# runs-on: windows-2022 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b -# path: | -# ~/.cargo -# target -# .pixi -# - name: Build vegafusion-server -# run: | -# pixi install -vvv -# pixi run build-rs-vegafusion-server -# - name: zip executable -# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 -# with: -# files: target/release/vegafusion-server.exe -# dest: vegafusion-server-win-64.zip -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-server -# path: | -# vegafusion-server-* -# -# build-vegafusion-server-osx-64: -# runs-on: macos-13 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64 -# path: | -# ~/.cargo -# target -# .pixi -# - name: Build vegafusion-server -# run: | -# pixi run build-rs-vegafusion-server -# - name: zip executable -# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 -# with: -# files: target/release/vegafusion-server -# dest: vegafusion-server-osx-64.zip -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-server -# path: | -# vegafusion-server-* -# -# build-vegafusion-server-osx-arm64: -# runs-on: macos-14 -# steps: -# - name: Check out repository code -# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 -# - uses: prefix-dev/setup-pixi@v0.8.1 -# with: -# pixi-version: v0.27.1 -# - name: Cache -# uses: actions/cache@v3 -# with: -# key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64 -# path: | -# ~/.cargo -# target -# .pixi -# - name: Build vegafusion-server -# run: | -# pixi run build-rs-vegafusion-server --target aarch64-apple-darwin -# - name: zip executable -# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 -# with: -# files: target/aarch64-apple-darwin/release/vegafusion-server -# dest: vegafusion-server-osx-arm64.zip -# - name: Upload artifacts -# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 -# with: -# name: vegafusion-server -# path: | -# vegafusion-server-* + + + build-vegafusion-server-linux-arm64: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - name: Install latest stable Rust toolchain + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1 + with: + toolchain: stable + override: true + - name: Cache rust dependencies + uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1 + with: + cache-on-failure: True + - name: Download arm64 toolchain + run: | + rustup target add aarch64-unknown-linux-gnu + - name: Download gcc for cross compilation + run: | + sudo apt-get update + sudo apt-get install gcc-aarch64-linux-gnu + - name: Build vegafusion-server + uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1 + env: + RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc" + with: + command: build + args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu + - name: zip executable + uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 + with: + files: target/aarch64-unknown-linux-gnu/release/vegafusion-server + dest: vegafusion-server-linux-arm64.zip + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-server + path: | + vegafusion-server-* + + build-vegafusion-server-win-64: + runs-on: windows-2022 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b + path: | + ~/.cargo + target + .pixi + - name: Build vegafusion-server + run: | + pixi install -vvv + pixi run build-rs-vegafusion-server + - name: zip executable + uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 + with: + files: target/release/vegafusion-server.exe + dest: vegafusion-server-win-64.zip + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-server + path: | + vegafusion-server-* + + build-vegafusion-server-osx-64: + runs-on: macos-13 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64 + path: | + ~/.cargo + target + .pixi + - name: Build vegafusion-server + run: | + pixi run build-rs-vegafusion-server + - name: zip executable + uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 + with: + files: target/release/vegafusion-server + dest: vegafusion-server-osx-64.zip + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-server + path: | + vegafusion-server-* + + build-vegafusion-server-osx-arm64: + runs-on: macos-14 + steps: + - name: Check out repository code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.27.1 + - name: Cache + uses: actions/cache@v3 + with: + key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64 + path: | + ~/.cargo + target + .pixi + - name: Build vegafusion-server + run: | + pixi run build-rs-vegafusion-server --target aarch64-apple-darwin + - name: zip executable + uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 + with: + files: target/aarch64-apple-darwin/release/vegafusion-server + dest: vegafusion-server-osx-arm64.zip + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: vegafusion-server + path: | + vegafusion-server-*