From f75d1dc96f4793b90ed3311d647d8d03d7175993 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 18 Nov 2023 13:24:19 -0500 Subject: [PATCH] Update to DataFusion 33 --- Cargo.lock | 237 +++++++++--------- Cargo.toml | 21 +- vegafusion-common/src/data/scalar.rs | 79 +++++- vegafusion-common/src/data/table.rs | 29 +-- vegafusion-core/src/task_graph/memory.rs | 12 +- vegafusion-core/src/task_graph/task_value.rs | 2 +- vegafusion-dataframe/src/dataframe.rs | 4 +- vegafusion-datafusion-udfs/src/udafs/mod.rs | 11 +- .../src/udfs/array/indexof.rs | 7 +- .../src/udfs/array/length.rs | 5 +- .../src/udfs/array/span.rs | 21 +- .../src/udfs/datetime/date_part_tz.rs | 2 +- .../udfs/datetime/date_to_utc_timestamp.rs | 2 +- .../udfs/datetime/epoch_to_utc_timestamp.rs | 2 +- .../src/udfs/datetime/format_timestamp.rs | 2 +- .../src/udfs/datetime/from_utc_timestamp.rs | 2 +- .../src/udfs/datetime/make_utc_timestamp.rs | 4 +- .../src/udfs/datetime/str_to_utc_timestamp.rs | 2 +- .../src/udfs/datetime/timeunit.rs | 2 +- .../src/udfs/datetime/to_utc_timestamp.rs | 2 +- .../udfs/datetime/utc_timestamp_to_epoch.rs | 2 +- .../src/udfs/datetime/utc_timestamp_to_str.rs | 2 +- .../src/udfs/member/mod.rs | 13 +- vegafusion-python-embed/Cargo.toml | 2 +- .../data/vl_selection_resolve.rs | 37 ++- .../data/vl_selection_test.rs | 19 +- .../src/expression/compiler/member.rs | 2 +- .../src/expression/compiler/mod.rs | 46 +--- vegafusion-runtime/src/transform/bin.rs | 14 +- vegafusion-runtime/src/transform/extent.rs | 10 +- .../src/transform/identifier.rs | 2 +- .../tests/test_image_comparison.rs | 168 ++++++------- .../tests/test_vegajs_runtime.rs | 8 +- vegafusion-sql/src/compile/expr.rs | 21 +- vegafusion-sql/src/compile/function_arg.rs | 14 +- vegafusion-sql/src/compile/scalar.rs | 21 +- vegafusion-sql/src/compile/select.rs | 13 +- .../src/connection/datafusion_conn.rs | 6 +- vegafusion-sql/src/dataframe/mod.rs | 27 +- vegafusion-sql/src/dialect/mod.rs | 22 +- .../src/dialect/transforms/date_add_tz.rs | 18 ++ .../src/dialect/transforms/date_part_tz.rs | 16 ++ .../transforms/date_to_utc_timestamp.rs | 17 ++ .../src/dialect/transforms/date_trunc_tz.rs | 20 ++ .../transforms/epoch_ms_to_utc_timestamp.rs | 16 ++ .../dialect/transforms/make_utc_timestamp.rs | 24 ++ .../transforms/str_to_utc_timestamp.rs | 17 ++ .../dialect/transforms/to_utc_timestamp.rs | 6 + .../transforms/utc_timestamp_to_epoch_ms.rs | 12 + .../transforms/utc_timestamp_to_str.rs | 18 ++ 50 files changed, 645 insertions(+), 416 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fb3de97d..a03d9bf1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,9 +141,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" +checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378" dependencies = [ "ahash", "arrow-arith", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +checksum = "ef983914f477d4278b068f13b3224b7d19eb2b807ac9048544d3bfebdf2554c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b" dependencies = [ "ahash", "arrow-buffer", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +checksum = "55512d988c6fbd76e514fd3ff537ac50b0a675da5a245e4fdad77ecfd654205f" dependencies = [ "bytes", "half 2.2.1", @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +checksum = "655ee51a2156ba5375931ce21c1b2494b1d9260e6dcdc6d4db9060c37dc3325b" dependencies = [ "arrow-array", "arrow-buffer", @@ -225,9 +225,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" +checksum = "258bb689997ad5b6660b3ce3638bd6b383d668ec555ed41ad7c6559cbb2e4f91" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,9 +244,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +checksum = "6dc2b9fec74763427e2e5575b8cc31ce96ba4c9b4eb05ce40e0616d9fad12461" dependencies = [ "arrow-buffer", "arrow-schema", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +checksum = "6eaa6ab203cc6d89b7eaa1ac781c1dfeef325454c5d5a0419017f95e6bafc03c" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" +checksum = "fb64e30d9b73f66fdc5c52d5f4cf69bbf03d62f64ffeafa0715590a5320baed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +checksum = "f9a818951c0d11c428dda03e908175969c262629dd20bd0850bd6c7a8c3bfe48" dependencies = [ "arrow-array", "arrow-buffer", @@ -305,9 +305,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3" dependencies = [ "ahash", "arrow-array", @@ -320,18 +320,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +checksum = "aaf4d737bba93da59f16129bec21e087aed0be84ff840e74146d4703879436cb" dependencies = [ "bitflags 2.1.0", ] [[package]] name = "arrow-select" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a" dependencies = [ "ahash", "arrow-array", @@ -343,9 +343,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" +checksum = "b15aed5624bb23da09142f58502b59c23f5bea607393298bb81dab1ce60fc769" dependencies = [ "arrow-array", "arrow-buffer", @@ -354,7 +354,7 @@ dependencies = [ "arrow-select", "num", "regex", - "regex-syntax", + "regex-syntax 0.8.2", ] [[package]] @@ -407,8 +407,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd", - "zstd-safe", + "zstd 0.12.3+zstd.1.5.2", + "zstd-safe 6.0.4+zstd.1.5.4", ] [[package]] @@ -1238,9 +1238,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7014432223f4d721cb9786cd88bb89e7464e0ba984d4a7f49db7787f5f268674" +checksum = "676796427e638d85e9eadf13765705212be60b34f8fc5d3934d95184c63ca1b4" dependencies = [ "ahash", "arrow", @@ -1271,24 +1271,23 @@ dependencies = [ "object_store", "parking_lot 0.12.1", "parquet", - "percent-encoding", "pin-project-lite", "rand", - "sqlparser 0.38.0", + "sqlparser", "tempfile", "tokio", "tokio-util", "url", "uuid", "xz2", - "zstd", + "zstd 0.13.0", ] [[package]] name = "datafusion-common" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3903ed8f102892f17b48efa437f3542159241d41c564f0d1e78efdc5e663aa" +checksum = "31e23b3d21a6531259d291bd20ce59282ea794bda1018b0a1e278c13cd52e50c" dependencies = [ "ahash", "arrow", @@ -1301,14 +1300,14 @@ dependencies = [ "object_store", "parquet", "pyo3", - "sqlparser 0.38.0", + "sqlparser", ] [[package]] name = "datafusion-execution" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780b73b2407050e53f51a9781868593f694102c59e622de9a8aafc0343c4f237" +checksum = "4de1fd0d8db0f2b8e4f4121bfa1c7c09d3a5c08a0a65c2229cd849eb65cff855" dependencies = [ "arrow", "chrono", @@ -1327,24 +1326,24 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c382676338d8caba6c027ba0da47260f65ffedab38fda78f6d8043f607557c" +checksum = "18e227fe88bf6730cab378d0cd8fc4c6b2ea42bc7e414a8ea9feba7225932735" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", - "sqlparser 0.38.0", + "sqlparser", "strum 0.25.0", "strum_macros 0.25.1", ] [[package]] name = "datafusion-optimizer" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2904a432f795484fd45e29ded4537152adb60f636c05691db34fcd94c92c96" +checksum = "5c6648e62ea7605b9bfcd87fdc9d67e579c3b9ac563a87734ae5fe6d79ee4547" dependencies = [ "arrow", "async-trait", @@ -1355,19 +1354,20 @@ dependencies = [ "hashbrown 0.14.0", "itertools 0.11.0", "log", - "regex-syntax", + "regex-syntax 0.8.2", ] [[package]] name = "datafusion-physical-expr" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b4968e9a998dc0476c4db7a82f280e2026b25f464e4aa0c3bb9807ee63ddfd" +checksum = "f32b8574add16a32411a9b3fb3844ac1fc09ab4e7be289f86fd56d620e4f2508" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "base64", "blake2", @@ -1394,9 +1394,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd0d1fe54e37a47a2d58a1232c22786f2c28ad35805fdcd08f0253a8b0aaa90" +checksum = "796abd77d5bfecd9e5275a99daf0ec45f5b3a793ec431349ce8211a67826fd22" dependencies = [ "ahash", "arrow", @@ -1425,9 +1425,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca1db9605e1f545b852fa9dd05f03339b333c9f2814a4e33b9ac7e9e586a1014" +checksum = "26de2592417beb20f73f29b131a04d7de14e2a6336c631554d611584b4306236" dependencies = [ "arrow", "chrono", @@ -1440,16 +1440,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "32.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b568d44c87ead99604d704f942e257c8a236ee1bbf890ee3e034ad659dcb2c21" +checksum = "ced70b8a5648ba7b95c61fc512183c33287ffe2c9f22ffe22700619d7d48c84f" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.38.0", + "sqlparser", ] [[package]] @@ -2090,9 +2090,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "instant" @@ -2412,23 +2412,12 @@ dependencies = [ ] [[package]] -name = "lz4" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" -dependencies = [ - "libc", - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" +name = "lz4_flex" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" dependencies = [ - "cc", - "libc", + "twox-hash", ] [[package]] @@ -2637,9 +2626,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d359e231e5451f4f9fa889d56e3ce34f8724f1a61db2107739359717cf2bbf08" +checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" dependencies = [ "async-trait", "base64", @@ -2648,7 +2637,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.10.5", + "itertools 0.11.0", "parking_lot 0.12.1", "percent-encoding", "quick-xml", @@ -2756,9 +2745,9 @@ dependencies = [ [[package]] name = "parquet" -version = "47.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597" dependencies = [ "ahash", "arrow-array", @@ -2775,7 +2764,7 @@ dependencies = [ "flate2", "futures", "hashbrown 0.14.0", - "lz4", + "lz4_flex", "num", "num-bigint", "object_store", @@ -2785,7 +2774,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd", + "zstd 0.13.0", ] [[package]] @@ -3090,9 +3079,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb88ae05f306b4bfcde40ac4a51dc0b05936a9207a4b75b798c7729c4258a59" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", @@ -3107,9 +3096,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554db24f0b3c180a9c0b1268f91287ab3f17c162e15b54caaae5a6b3773396b0" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -3117,9 +3106,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "922ede8759e8600ad4da3195ae41259654b9c55da4f7eec84a0ccc7d067a70a4" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -3127,32 +3116,33 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5caec6a1dd355964a841fcbeeb1b89fe4146c87295573f94228911af3cc5a2" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b78ccbb160db1556cdb6fd96c50334c5d4ec44dc5e0a968d0a1208fa0efa8b" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "pythonize" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e35b716d430ace57e2d1b4afb51c9e5b7c46d2bce72926e07f9be6a98ced03e" +checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850" dependencies = [ "pyo3", "serde", @@ -3166,9 +3156,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", "serde", @@ -3262,7 +3252,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.3.8", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -3279,7 +3269,7 @@ checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -3288,6 +3278,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "relative-path" version = "1.9.0" @@ -3722,18 +3718,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ae05a8250b968a3f7db93155a84d68b2e6cea1583949af5ca5b5170c76c075" -dependencies = [ - "log", -] - -[[package]] -name = "sqlparser" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" +checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" dependencies = [ "log", "sqlparser_derive", @@ -4335,9 +4322,9 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "untrusted" @@ -4391,7 +4378,7 @@ dependencies = [ "object_store", "pyo3", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "thiserror", ] @@ -4418,7 +4405,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "thiserror", "tonic", "tonic-build", @@ -4526,7 +4513,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "tempfile", "test-case", "tokio", @@ -4585,7 +4572,7 @@ dependencies = [ "rstest_reuse", "serde", "serde_json", - "sqlparser 0.37.0", + "sqlparser", "tempfile", "tokio", "toml", @@ -5014,7 +5001,16 @@ version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe", + "zstd-safe 6.0.4+zstd.1.5.4", +] + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe 7.0.0", ] [[package]] @@ -5027,6 +5023,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.7+zstd.1.5.4" diff --git a/Cargo.toml b/Cargo.toml index e69b749f8..1795da463 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,22 +14,23 @@ members = [ ] [workspace.dependencies] -arrow = { version = "47.0.0", default_features = false } -sqlparser = { version = "0.37.0" } +arrow = { version = "48.0.1", default_features = false } +sqlparser = { version = "0.39.0" } chrono = { version = "0.4.31", default_features = false } reqwest = { version = "0.11.22", default-features = false } tokio = { version = "1.32.0" } -pyo3 = { version = "0.19" } +pyo3 = { version = "0.20.0" } +pythonize = { version = "0.20.0" } prost = { version = "0.12.1" } prost-types = { version = "0.12.1" } -object_store = { version="0.7" } +object_store = { version= "0.7.1" } -datafusion = { version = "32.0.0" } -datafusion-common = { version = "32.0.0", default_features = false} -datafusion-expr = { version = "32.0.0" } -datafusion-proto = { version = "32.0.0" } -datafusion-physical-expr = { version = "32.0.0" } -datafusion-optimizer = { version = "32.0.0" } +datafusion = { version = "33.0.0" } +datafusion-common = { version = "33.0.0", default_features = false} +datafusion-expr = { version = "33.0.0" } +datafusion-proto = { version = "33.0.0" } +datafusion-physical-expr = { version = "33.0.0" } +datafusion-optimizer = { version = "33.0.0" } [profile.release] ## Tell `rustc` to use highest performance optimization and perform Link Time Optimization diff --git a/vegafusion-common/src/data/scalar.rs b/vegafusion-common/src/data/scalar.rs index 4518f58d2..df2d3ce5e 100644 --- a/vegafusion-common/src/data/scalar.rs +++ b/vegafusion-common/src/data/scalar.rs @@ -1,10 +1,13 @@ use crate::error::{Result, VegaFusionError}; +use arrow::array::{new_empty_array, Array, ArrayRef, ListArray}; +use datafusion_common::DataFusionError; +use datafusion_common::utils::array_into_list_array; pub use datafusion_common::ScalarValue; #[cfg(feature = "json")] use { - arrow::datatypes::{DataType, Field}, + arrow::datatypes::DataType, serde_json::{Map, Value}, std::ops::Deref, std::sync::Arc, @@ -62,18 +65,18 @@ impl ScalarValueHelpers for ScalarValue { } } Value::Array(elements) => { - let (elements, dtype) = if elements.is_empty() { - (Vec::new(), DataType::Float64) + let array: ListArray = if elements.is_empty() { + array_into_list_array(Arc::new(new_empty_array(&DataType::Float64))) } else { let elements: Vec<_> = elements .iter() .map(ScalarValue::from_json) .collect::>>()?; - let dtype = elements[0].data_type(); - (elements, dtype) + + array_into_list_array(ScalarValue::iter_to_array(elements)?) }; - ScalarValue::List(Some(elements), Arc::new(Field::new("item", dtype, true))) + ScalarValue::List(Arc::new(array)) } }; Ok(scalar_value) @@ -132,13 +135,12 @@ impl ScalarValueHelpers for ScalarValue { ScalarValue::IntervalDayTime(Some(_v)) => { unimplemented!() } - ScalarValue::List(Some(v), _) => Value::Array( - v.clone() - .into_iter() - .map(|v| v.to_json()) + ScalarValue::List(a) => Value::Array( + a.list_el_to_scalar_vec()? + .iter() + .map(|s| s.to_json()) .collect::>>()?, ), - ScalarValue::List(None, _) => Value::Array(Vec::new()), ScalarValue::Struct(Some(v), fields) => { let mut pairs: Map = Default::default(); for (val, field) in v.iter().zip(fields.deref()) { @@ -173,7 +175,8 @@ impl ScalarValueHelpers for ScalarValue { } fn to_f64x2(&self) -> Result<[f64; 2]> { - if let ScalarValue::List(Some(elements), _) = self { + if let ScalarValue::List(array) = self { + let elements = array.list_el_to_scalar_vec()?; if let [v0, v1] = elements.as_slice() { return Ok([v0.to_f64()?, v1.to_f64()?]); } @@ -195,3 +198,55 @@ impl ScalarValueHelpers for ScalarValue { }) } } + +pub trait ArrayRefHelpers { + fn to_scalar_vec(&self) -> std::result::Result, DataFusionError>; + + fn list_el_to_scalar_vec(&self) -> std::result::Result, DataFusionError>; + + fn list_el_len(&self) -> std::result::Result; + + fn list_el_dtype(&self) -> std::result::Result; +} + +impl ArrayRefHelpers for ArrayRef { + /// Convert ArrayRef into vector of ScalarValues + fn to_scalar_vec(&self) -> std::result::Result, DataFusionError> { + (0..self.len()) + .map(|i| Ok(ScalarValue::try_from_array(self, i)?)) + .collect::, DataFusionError>>() + } + + /// Extract Vec for single element ListArray (as is stored inside ScalarValue::List(arr)) + fn list_el_to_scalar_vec(&self) -> std::result::Result, DataFusionError> { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_to_scalar_vec called on non-List type".to_string(), + ))?; + a.value(0).to_scalar_vec() + } + + /// Extract length of single element ListArray + fn list_el_len(&self) -> std::result::Result { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_len called on non-List type".to_string(), + ))?; + Ok(a.value(0).len()) + } + + /// Extract data type of single element ListArray + fn list_el_dtype(&self) -> std::result::Result { + let a = self + .as_any() + .downcast_ref::() + .ok_or(DataFusionError::Internal( + "list_el_len called on non-List type".to_string(), + ))?; + Ok(a.value(0).data_type().clone()) + } +} diff --git a/vegafusion-common/src/data/table.rs b/vegafusion-common/src/data/table.rs index 37b896b02..a1cbe43fa 100644 --- a/vegafusion-common/src/data/table.rs +++ b/vegafusion-common/src/data/table.rs @@ -13,6 +13,7 @@ use crate::{ error::{Result, ResultWithContext, VegaFusionError}, }; +use arrow::array::new_empty_array; #[cfg(feature = "prettyprint")] use arrow::util::pretty::pretty_format_batches; use std::{ @@ -41,6 +42,7 @@ use { #[cfg(feature = "base64")] use base64::{engine::general_purpose, Engine as _}; +use datafusion_common::utils::array_into_list_array; #[derive(Clone, Debug)] pub struct VegaFusionTable { @@ -182,30 +184,11 @@ impl VegaFusionTable { pub fn to_scalar_value(&self) -> Result { if self.num_rows() == 0 { // Return empty list with (arbitrary) Float64 type - let dtype = DataType::Float64; - return Ok(ScalarValue::List( - Some(Vec::new()), - Arc::new(Field::new("item", dtype, true)), - )); + let array = Arc::new(new_empty_array(&DataType::Float64)); + return Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))); } - - let mut elements: Vec = Vec::new(); - for batch in &self.batches { - let array = Arc::new(StructArray::from(batch.clone())) as ArrayRef; - - for i in 0..array.len() { - let scalar = ScalarValue::try_from_array(&array, i).with_context(|| { - "Failed to convert record batch row to ScalarValue".to_string() - })?; - elements.push(scalar) - } - } - - let dtype = elements[0].data_type(); - Ok(ScalarValue::List( - Some(elements), - Arc::new(Field::new("item", dtype, true)), - )) + let array = Arc::new(StructArray::from(self.to_record_batch()?)) as ArrayRef; + Ok(ScalarValue::List(Arc::new(array_into_list_array(array)))) } #[cfg(feature = "json")] diff --git a/vegafusion-core/src/task_graph/memory.rs b/vegafusion-core/src/task_graph/memory.rs index 68676b51b..b7d753bb0 100644 --- a/vegafusion-core/src/task_graph/memory.rs +++ b/vegafusion-core/src/task_graph/memory.rs @@ -47,17 +47,7 @@ pub fn inner_size_of_scalar(value: &ScalarValue) -> usize { ScalarValue::LargeUtf8(Some(s)) => size_of_val(s.as_bytes()) + size_of::(), ScalarValue::Binary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), ScalarValue::LargeBinary(Some(b)) => size_of_val(b.as_slice()) + size_of::>(), - ScalarValue::List(Some(values), field) => { - let values_bytes: usize = size_of::>() - + values - .iter() - .map(|v| size_of::() + inner_size_of_scalar(v)) - .sum::(); - - let dtype_bytes = size_of::() + inner_size_of_dtype(field.data_type()); - - values_bytes + dtype_bytes - } + ScalarValue::List(array) => size_of::>() + size_of_array_ref(array), ScalarValue::Struct(Some(values), fields) => { let values_bytes: usize = size_of::>() + values diff --git a/vegafusion-core/src/task_graph/task_value.rs b/vegafusion-core/src/task_graph/task_value.rs index 734ca1eb4..f84d32328 100644 --- a/vegafusion-core/src/task_graph/task_value.rs +++ b/vegafusion-core/src/task_graph/task_value.rs @@ -70,7 +70,7 @@ impl TryFrom<&TaskValue> for ProtoTaskValue { fn try_from(value: &TaskValue) -> std::result::Result { match value { TaskValue::Scalar(scalar) => { - let scalar_array = scalar.to_array(); + let scalar_array = scalar.to_array()?; let scalar_rb = RecordBatch::try_from_iter(vec![("value", scalar_array)])?; let ipc_bytes = VegaFusionTable::from(scalar_rb).to_ipc_bytes()?; Ok(Self { diff --git a/vegafusion-dataframe/src/dataframe.rs b/vegafusion-dataframe/src/dataframe.rs index 7e2f39b45..93cfecfba 100644 --- a/vegafusion-dataframe/src/dataframe.rs +++ b/vegafusion-dataframe/src/dataframe.rs @@ -112,7 +112,7 @@ pub trait DataFrame: Send + Sync + 'static { async fn with_index(&self, index_name: &str) -> Result> { if self.schema().column_with_name(index_name).is_some() { // Column is already present, don't overwrite - self.select(vec![Expr::Wildcard]).await + self.select(vec![Expr::Wildcard { qualifier: None }]).await } else { let selections = vec![ Expr::WindowFunction(expr::WindowFunction { @@ -129,7 +129,7 @@ pub trait DataFrame: Send + Sync + 'static { }, }) .alias(index_name), - Expr::Wildcard, + Expr::Wildcard { qualifier: None }, ]; self.select(selections).await } diff --git a/vegafusion-datafusion-udfs/src/udafs/mod.rs b/vegafusion-datafusion-udfs/src/udafs/mod.rs index 7e6fbc27a..4f3a7d41b 100644 --- a/vegafusion-datafusion-udfs/src/udafs/mod.rs +++ b/vegafusion-datafusion-udfs/src/udafs/mod.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use vegafusion_common::arrow::array::{Array, ArrayRef, UInt32Array}; use vegafusion_common::arrow::compute::sort_to_indices; use vegafusion_common::arrow::datatypes::{DataType, Field, FieldRef}; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{create_udaf, Accumulator, AggregateUDF, Volatility}; @@ -19,8 +19,8 @@ pub(crate) struct PercentileContAccumulator { impl Accumulator for PercentileContAccumulator { fn state(&self) -> Result, DataFusionError> { - let state = ScalarValue::new_list(Some(self.all_values.clone()), self.data_type.clone()); - Ok(vec![state]) + let state = ScalarValue::new_list(self.all_values.as_slice(), &self.data_type); + Ok(vec![ScalarValue::List(Arc::new(state))]) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<(), DataFusionError> { @@ -43,14 +43,13 @@ impl Accumulator for PercentileContAccumulator { assert!(matches!(array.data_type(), DataType::List(_))); for index in 0..array.len() { match ScalarValue::try_from_array(array, index)? { - ScalarValue::List(Some(values), _) => { - for scalar in values { + ScalarValue::List(array) => { + for scalar in array.list_el_to_scalar_vec()? { if !scalar_is_non_finite(&scalar) { self.all_values.push(scalar); } } } - ScalarValue::List(None, _) => {} // skip empty state v => { return Err(DataFusionError::Internal(format!( "unexpected state in percentile_cont. Expected DataType::List, got {v:?}" diff --git a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs index 9145ea710..e223518ec 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/indexof.rs @@ -6,7 +6,7 @@ use vegafusion_common::arrow::array::{ }; use vegafusion_common::arrow::compute::cast; use vegafusion_common::arrow::datatypes::DataType; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; use vegafusion_common::datafusion_expr::{ ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF, Signature, @@ -24,8 +24,9 @@ fn make_indexof_udf() -> ScalarUDF { let indexof_fn: ScalarFunctionImplementation = Arc::new(|args: &[ColumnarValue]| { // Signature ensures there is a single argument let (array, array_dtype) = match &args[0] { - ColumnarValue::Scalar(ScalarValue::List(Some(scalar_array), field)) => { - (scalar_array.clone(), field.data_type().clone()) + ColumnarValue::Scalar(ScalarValue::List(array)) => { + let scalar_array = array.list_el_to_scalar_vec()?; + (scalar_array, array.list_el_dtype()?) } _ => { return Err(DataFusionError::Internal( diff --git a/vegafusion-datafusion-udfs/src/udfs/array/length.rs b/vegafusion-datafusion-udfs/src/udfs/array/length.rs index 3f166cab2..eb0da7603 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/length.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/length.rs @@ -21,8 +21,9 @@ fn make_length_udf() -> ScalarUDF { Ok(match arg { ColumnarValue::Scalar(value) => { match value { - ScalarValue::List(Some(arr), _) => { - ColumnarValue::Scalar(ScalarValue::from(arr.len() as f64)) + ScalarValue::List(arr) => { + let arr = arr.as_any().downcast_ref::().unwrap(); + ColumnarValue::Scalar(ScalarValue::from(arr.value(0).len() as f64)) } ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => { ColumnarValue::Scalar(ScalarValue::from(s.len() as f64)) diff --git a/vegafusion-datafusion-udfs/src/udfs/array/span.rs b/vegafusion-datafusion-udfs/src/udfs/array/span.rs index 925818e47..90319efdb 100644 --- a/vegafusion-datafusion-udfs/src/udfs/array/span.rs +++ b/vegafusion-datafusion-udfs/src/udfs/array/span.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use vegafusion_common::arrow::array::{Array, ListArray}; use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datafusion_common::{DataFusionError, ScalarValue}; @@ -25,21 +26,31 @@ fn make_span_udf() -> ScalarUDF { // Span of scalar (including null) is 0 ColumnarValue::Scalar(ScalarValue::from(0.0)) } - ScalarValue::List(Some(arr), element_type) => { - match element_type.data_type() { + ScalarValue::List(arr) => { + // Unwrap single element ListArray + let arr = arr.as_any().downcast_ref::().unwrap(); + let arr = arr.value(0); + match arr.data_type() { DataType::Float64 => { if arr.is_empty() { // Span of empty array is 0 ColumnarValue::Scalar(ScalarValue::from(0.0)) } else { - let first = arr.first().unwrap().to_f64().unwrap(); - let last = arr.last().unwrap().to_f64().unwrap(); + let first = ScalarValue::try_from_array(&arr, 0) + .unwrap() + .to_f64() + .unwrap(); + let last = ScalarValue::try_from_array(&arr, arr.len() - 1) + .unwrap() + .to_f64() + .unwrap(); ColumnarValue::Scalar(ScalarValue::from(last - first)) } } _ => { return Err(DataFusionError::Internal(format!( - "Unexpected element type for span function: {element_type}" + "Unexpected element type for span function: {}", + arr.data_type() ))) } } diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs index d358ae35a..77eaffd4a 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_part_tz.rs @@ -18,7 +18,7 @@ fn make_date_part_tz_udf() -> ScalarUDF { // [1] data array let timestamp_array = match &args[1] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = to_timestamp_ms(×tamp_array)?; diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs index f375023d8..921a7ddec 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/date_to_utc_timestamp.rs @@ -19,7 +19,7 @@ fn make_date_to_utc_timestamp() -> ScalarUDF { // [0] data array let date_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs index 7a8be8d31..7a3f041e4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/epoch_to_utc_timestamp.rs @@ -16,7 +16,7 @@ fn make_epoch_to_utc_timestamp() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = cast( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs index 35228436b..4c2048b88 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/format_timestamp.rs @@ -19,7 +19,7 @@ fn make_format_timestamp_udf() -> ScalarUDF { // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] time format string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs index 190abe336..e26c7b937 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/from_utc_timestamp.rs @@ -23,7 +23,7 @@ fn make_from_utc_timestamp() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs index c53fbcf06..4b460bf1e 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/make_utc_timestamp.rs @@ -42,11 +42,11 @@ fn make_utc_timestamp_udf() -> ScalarUDF { let args = if let Some(len) = len { args.iter() .map(|arg| arg.clone().into_array(len)) - .collect::>() + .collect::, DataFusionError>>()? } else { args.iter() .map(|arg| arg.clone().into_array(1)) - .collect::>() + .collect::, DataFusionError>>()? }; // To int64 arrays diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs index 151cbd103..4bba48e40 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/str_to_utc_timestamp.rs @@ -334,7 +334,7 @@ fn make_str_to_utc_timestamp_udf() -> ScalarUDF { // [0] data array let str_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs index b2eccd41d..71066586c 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/timeunit.rs @@ -39,7 +39,7 @@ fn unpack_timeunit_udf_args( DataFusionError::Internal(format!("Failed to parse {tz_str} as a timezone")) })?; - let timestamp = columns[0].clone().into_array(1); + let timestamp = columns[0].clone().into_array(1)?; let timestamp = process_input_datetime(×tamp, &tz)?; Ok(( diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs index b35f93024..036cca048 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/to_utc_timestamp.rs @@ -22,7 +22,7 @@ fn make_to_utc_timestamp_udf() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; // [1] timezone string diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs index 01ef27608..1fd4780e4 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_epoch.rs @@ -18,7 +18,7 @@ fn make_utc_timestamp_to_epoch_ms_udf() -> ScalarUDF { // [0] data array let data_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let data_array = to_timestamp_ms(&data_array)?; diff --git a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs index 12c4f6ec3..88ae6749b 100644 --- a/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs +++ b/vegafusion-datafusion-udfs/src/udfs/datetime/utc_timestamp_to_str.rs @@ -20,7 +20,7 @@ fn make_utc_timestamp_to_str_udf() -> ScalarUDF { // [0] data array let timestamp_array = match &args[0] { ColumnarValue::Array(array) => array.clone(), - ColumnarValue::Scalar(scalar) => scalar.to_array(), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, }; let timestamp_array = to_timestamp_ms(×tamp_array)?; diff --git a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs index 2e7921a94..b7e912878 100644 --- a/vegafusion-datafusion-udfs/src/udfs/member/mod.rs +++ b/vegafusion-datafusion-udfs/src/udfs/member/mod.rs @@ -62,17 +62,16 @@ pub fn make_get_element_udf(index: i32) -> ScalarUDF { Ok(match arg { ColumnarValue::Scalar(value) => { match value { - ScalarValue::List(Some(arr), element_dtype) => { - match arr.get(index as usize) { - Some(element) => { + ScalarValue::List(arr) => { + let arr = arr.as_any().downcast_ref::().unwrap(); + match ScalarValue::try_from_array(&arr.value(0), index as usize) { + Ok(element) => { // Scalar element of list ColumnarValue::Scalar(element.clone()) } - None => { + _ => { // out of bounds, null - ColumnarValue::Scalar( - ScalarValue::try_from(element_dtype.data_type()).unwrap(), - ) + ColumnarValue::Scalar(ScalarValue::try_from(arr.data_type())?) } } } diff --git a/vegafusion-python-embed/Cargo.toml b/vegafusion-python-embed/Cargo.toml index c482d91af..6179b7042 100644 --- a/vegafusion-python-embed/Cargo.toml +++ b/vegafusion-python-embed/Cargo.toml @@ -15,7 +15,7 @@ protobuf-src = [ "vegafusion-core/protobuf-src",] [dependencies] log = "0.4.17" env_logger = "0.10.0" -pythonize = "0.19.0" +pythonize = { workspace=true } async-trait = "0.1.73" uuid = "1.3.0" diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs index 9a63d94b3..99d21abb6 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_resolve.rs @@ -1,12 +1,11 @@ +use datafusion_common::utils::array_into_list_array; use datafusion_expr::{lit, Expr}; use itertools::Itertools; use std::collections::HashMap; use std::convert::TryFrom; use std::str::FromStr; use std::sync::Arc; - -use vegafusion_common::arrow::datatypes::{DataType, Field}; -use vegafusion_common::data::scalar::ScalarValueHelpers; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValueHelpers}; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datafusion_common::{DFSchema, ScalarValue}; use vegafusion_common::error::{Result, VegaFusionError}; @@ -63,8 +62,8 @@ pub fn vl_selection_resolve_fn( let _op = parse_args(args)?; // Extract vector of rows for selection dataset - let rows = if let ScalarValue::List(Some(elements), _) = table.to_scalar_value()? { - elements + let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { + array.list_el_to_scalar_vec()? } else { unreachable!() }; @@ -79,16 +78,17 @@ pub fn vl_selection_resolve_fn( for (field, value) in row_spec.fields.iter().zip(&row_spec.values) { let value = match field.typ { SelectionType::Enum => { - if let ScalarValue::List(Some(elements), _) = value { - elements.clone() + if let ScalarValue::List(array) = value { + array.list_el_to_scalar_vec()? } else { vec![value.clone()] } } _ => { match &value { - ScalarValue::List(Some(elements), _) if elements.len() == 2 => { + ScalarValue::List(array) if array.list_el_len()? == 2 => { // Don't assume elements are in ascending order + let elements = array.list_el_to_scalar_vec()?; let first = elements[0].to_f64()?; let second = elements[1].to_f64()?; @@ -98,9 +98,6 @@ pub fn vl_selection_resolve_fn( (second, first) }; vec![ScalarValue::from(low), ScalarValue::from(high)] - // ScalarValue::List(Some(Box::new(vec![ - // ScalarValue::from(low), ScalarValue::from(high) - // ])), Box::new(DataType::Float64)) } v => { return Err(VegaFusionError::internal(format!( @@ -116,20 +113,20 @@ pub fn vl_selection_resolve_fn( } } - let props: Vec<_> = props + let props = props .into_iter() .map(|(name, values)| { // Turn values into a scalar list - let dtype = values - .get(0) - .map(|s| s.data_type()) - .unwrap_or(DataType::Float64); - let values = ScalarValue::List(Some(values), Arc::new(Field::new("item", dtype, true))); - (name, values) + let values = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(values)?, + ))); + Ok((name, values)) }) + .collect::>>()?; + let props: Vec<_> = props + .iter() .sorted_by_key(|(n, _)| n.clone()) - .collect(); - + .collect::>(); let props: Vec<_> = props .iter() .map(|(name, value)| (name.as_str(), value.clone())) diff --git a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index 6e6956555..0bbf318f3 100644 --- a/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-runtime/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -10,7 +10,7 @@ use std::str::FromStr; use std::sync::Arc; use vegafusion_common::arrow::datatypes::{DataType, TimeUnit}; use vegafusion_common::column::flat_col; -use vegafusion_common::data::scalar::ScalarValue; +use vegafusion_common::data::scalar::{ArrayRefHelpers, ScalarValue}; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datafusion_common::DFSchema; use vegafusion_common::datatypes::{ @@ -142,8 +142,8 @@ impl FieldSpec { let expr = match self.typ { SelectionType::Enum => { let field_type = field_col.get_type(schema)?; - let list_scalars = if let ScalarValue::List(Some(elements), _) = &values { - elements.clone() + let list_scalars = if let ScalarValue::List(array) = &values { + array.list_el_to_scalar_vec()? } else { // convert values to single element list vec![values.clone()] @@ -178,7 +178,8 @@ impl FieldSpec { }; let (low, high) = match &values { - ScalarValue::List(Some(elements), _) if elements.len() == 2 => { + ScalarValue::List(array) if array.list_el_len()? == 2 => { + let elements = array.list_el_to_scalar_vec()?; let first = Self::cast_test_scalar( elements[0].clone(), &field_dtype, @@ -405,7 +406,7 @@ impl TryFrom for SelectionRow { .get("values") .with_context(|| "Missing required property 'values'".to_string())?; let values = match struct_values.get(*values_index) { - Some(ScalarValue::List(Some(elements), _)) => elements.clone(), + Some(ScalarValue::List(array)) => array.list_el_to_scalar_vec()?, _ => { return Err(VegaFusionError::internal( "Expected 'values' to be an array".to_string(), @@ -420,8 +421,8 @@ impl TryFrom for SelectionRow { let mut fields: Vec = Vec::new(); match struct_values.get(*fields_index) { - Some(ScalarValue::List(Some(elements), _)) => { - for el in elements.iter() { + Some(ScalarValue::List(array)) => { + for el in array.list_el_to_scalar_vec()?.iter() { fields.push(FieldSpec::try_from(el.clone())?) } } @@ -508,8 +509,8 @@ pub fn vl_selection_test_fn( let op = parse_args(args)?; // Extract vector of rows for selection dataset - let rows = if let ScalarValue::List(Some(elements), _) = table.to_scalar_value()? { - elements + let rows = if let ScalarValue::List(array) = table.to_scalar_value()? { + array.list_el_to_scalar_vec()? } else { unreachable!() }; diff --git a/vegafusion-runtime/src/expression/compiler/member.rs b/vegafusion-runtime/src/expression/compiler/member.rs index b097de44a..d1804237b 100644 --- a/vegafusion-runtime/src/expression/compiler/member.rs +++ b/vegafusion-runtime/src/expression/compiler/member.rs @@ -33,7 +33,7 @@ pub fn compile_member( )?; let prop_str = evaluated_property.to_string(); if is_numeric_datatype(&evaluated_property.data_type()) { - let int_array = cast(&evaluated_property.to_array(), &DataType::Int64).unwrap(); + let int_array = cast(&evaluated_property.to_array()?, &DataType::Int64).unwrap(); let int_array = int_array.as_any().downcast_ref::().unwrap(); index = Some(int_array.value(0) as usize); } else { diff --git a/vegafusion-runtime/src/expression/compiler/mod.rs b/vegafusion-runtime/src/expression/compiler/mod.rs index 951763421..ebbf40cbd 100644 --- a/vegafusion-runtime/src/expression/compiler/mod.rs +++ b/vegafusion-runtime/src/expression/compiler/mod.rs @@ -70,8 +70,10 @@ mod test_compile { use std::convert::TryFrom; use std::ops::Deref; use std::sync::Arc; + use datafusion_common::utils::array_into_list_array; use vegafusion_common::arrow::datatypes::{DataType, Field, Schema}; use vegafusion_common::column::flat_col; + use vegafusion_core::arrow::array::{Float64Array, new_empty_array}; use vegafusion_core::arrow::datatypes::Fields; use vegafusion_datafusion_udfs::udfs::array::constructor::ARRAY_CONSTRUCTOR_UDF; use vegafusion_datafusion_udfs::udfs::object::make_object_constructor_udf; @@ -399,14 +401,8 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![ - ScalarValue::from(1.0), - ScalarValue::from(2.0), - ScalarValue::from(3.0), - ]), - Arc::new(Field::new("item", DataType::Float64, true)), - ); + + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -427,10 +423,8 @@ mod test_compile { // Check evaluated value. Empty array is given Float64 data type let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![]), - Arc::new(Field::new("item", DataType::Float64, true)), - ); + + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(new_empty_array(&DataType::Float64)))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); @@ -463,27 +457,13 @@ mod test_compile { // Check evaluated value let result_value = result_expr.eval_to_scalar().unwrap(); - let expected_value = ScalarValue::List( - Some(vec![ - ScalarValue::List( - Some(vec![ScalarValue::from(1.0), ScalarValue::from(2.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ScalarValue::List( - Some(vec![ScalarValue::from(3.0), ScalarValue::from(4.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ScalarValue::List( - Some(vec![ScalarValue::from(5.0), ScalarValue::from(6.0)]), - Arc::new(Field::new("item", DataType::Float64, true)), - ), - ]), - Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Float64, true))), - true, - )), - ); + let expected_value = ScalarValue::List(Arc::new(array_into_list_array(ScalarValue::iter_to_array( + vec![ + ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![1.0, 2.0]))))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![3.0, 4.0]))))), + ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![5.0, 6.0]))))), + ] + ).unwrap()))); println!("value: {result_value:?}"); assert_eq!(result_value, expected_value); diff --git a/vegafusion-runtime/src/transform/bin.rs b/vegafusion-runtime/src/transform/bin.rs index 553309216..66b6d4fed 100644 --- a/vegafusion-runtime/src/transform/bin.rs +++ b/vegafusion-runtime/src/transform/bin.rs @@ -7,6 +7,7 @@ use async_trait::async_trait; use datafusion_expr::lit; use datafusion_common::scalar::ScalarValue; +use datafusion_common::utils::array_into_list_array; use datafusion_common::DFSchema; use datafusion_expr::{abs, floor, when, Expr}; use float_cmp::approx_eq; @@ -15,7 +16,6 @@ use std::sync::Arc; use vegafusion_common::column::{flat_col, unescaped_col}; use vegafusion_common::data::scalar::ScalarValueHelpers; use vegafusion_common::datatypes::to_numeric; -use vegafusion_core::arrow::datatypes::{DataType, Field}; use vegafusion_core::error::{Result, VegaFusionError}; use vegafusion_core::proto::gen::transforms::Bin; use vegafusion_core::task_graph::task_value::TaskValue; @@ -52,7 +52,9 @@ impl TransformTrait for Bin { let bin_index = floor((numeric_field.clone().sub(lit(start)).div(lit(step))).add(lit(1.0e-14))) .alias(bin_index_name); - let sql_df = sql_df.select(vec![Expr::Wildcard, bin_index]).await?; + let sql_df = sql_df + .select(vec![Expr::Wildcard { qualifier: None }, bin_index]) + .await?; // Add column with bin start let bin_start = (flat_col(bin_index_name).mul(lit(step))).add(lit(start)); @@ -121,11 +123,9 @@ impl TransformTrait for Bin { fn compute_output_value(bin_tx: &Bin, start: f64, stop: f64, step: f64) -> Option { let mut fname = bin_tx.field.clone(); fname.insert_str(0, "bin_"); - - let fields = ScalarValue::List( - Some(vec![ScalarValue::from(bin_tx.field.as_str())]), - Arc::new(Field::new("item", DataType::Utf8, true)), - ); + let fields = ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![ScalarValue::from(bin_tx.field.as_str())]).ok()?, + ))); if bin_tx.signal.is_some() { Some(TaskValue::Scalar(ScalarValue::from(vec![ diff --git a/vegafusion-runtime/src/transform/extent.rs b/vegafusion-runtime/src/transform/extent.rs index 338f46926..2458afdf6 100644 --- a/vegafusion-runtime/src/transform/extent.rs +++ b/vegafusion-runtime/src/transform/extent.rs @@ -2,10 +2,10 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; +use datafusion_common::utils::array_into_list_array; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::{max, min, Expr}; use std::sync::Arc; -use vegafusion_common::arrow::datatypes::Field; use vegafusion_common::column::unescaped_col; use vegafusion_common::data::table::VegaFusionTable; use vegafusion_common::datatypes::to_numeric; @@ -61,10 +61,8 @@ fn extract_extent_list(table: &VegaFusionTable) -> Result { let max_val_scalar = ScalarValue::try_from_array(max_val_array, 0).unwrap(); // Build two-element list of the extents - let element_datatype = min_val_scalar.data_type(); - let extent_list = TaskValue::Scalar(ScalarValue::List( - Some(vec![min_val_scalar, max_val_scalar]), - Arc::new(Field::new("item", element_datatype, true)), - )); + let extent_list = TaskValue::Scalar(ScalarValue::List(Arc::new(array_into_list_array( + ScalarValue::iter_to_array(vec![min_val_scalar, max_val_scalar])?, + )))); Ok(extent_list) } diff --git a/vegafusion-runtime/src/transform/identifier.rs b/vegafusion-runtime/src/transform/identifier.rs index 57640d67a..5facb150d 100644 --- a/vegafusion-runtime/src/transform/identifier.rs +++ b/vegafusion-runtime/src/transform/identifier.rs @@ -41,7 +41,7 @@ impl TransformTrait for Identifier { .alias(&self.r#as); let result = dataframe - .select(vec![Expr::Wildcard, row_number_expr]) + .select(vec![Expr::Wildcard { qualifier: None }, row_number_expr]) .await?; Ok((result, Default::default())) diff --git a/vegafusion-runtime/tests/test_image_comparison.rs b/vegafusion-runtime/tests/test_image_comparison.rs index 2964dec79..e9d9aaece 100644 --- a/vegafusion-runtime/tests/test_image_comparison.rs +++ b/vegafusion-runtime/tests/test_image_comparison.rs @@ -60,91 +60,91 @@ mod test_custom_specs { spec_name, tolerance, extract_inline_values, - case("custom/stacked_bar", 0.001, false), - case("custom/bar_colors", 0.001, false), + // case("custom/stacked_bar", 0.001, false), + // case("custom/bar_colors", 0.001, false), case("custom/imdb_histogram", 0.001, false), - case("custom/flights_crossfilter_a", 0.001, false), - case("custom/flights_crossfilter_csv", 0.001, false), - case("custom/log_scaled_histogram", 0.001, false), - case("custom/non_linear_histogram", 0.001, false), - case("custom/relative_frequency_histogram", 0.001, false), - case("custom/kde_iris", 0.001, false), - case("custom/2d_circles_histogram_imdb", 0.001, false), - case("custom/2d_histogram_imdb", 0.001, false), - case("custom/cumulative_window_imdb", 0.001, false), - case("custom/density_and_cumulative_histograms", 0.001, false), - case("custom/mean_strip_plot_movies", 0.001, false), - case("custom/table_heatmap_cars", 0.001, false), - case("custom/difference_from_mean", 0.001, false), - case("custom/nested_concat_align", 0.001, false), - case("custom/imdb_dashboard_cross_height", 0.001, false), - case("custom/stacked_bar_weather_year", 0.001, false), - case("custom/stacked_bar_weather_month", 0.001, false), - case("custom/stacked_bar_normalize", 0.001, false), - case("custom/layer_bar_labels_grey", 0.001, false), - case("custom/bar_month_temporal_initial", 0.001, false), - case("custom/selection_layer_bar_month", 0.001, false), - case("custom/interactive_layered_crossfilter", 0.001, false), - case("custom/interactive_seattle_weather", 0.001, false), - case("custom/concat_marginal_histograms", 0.001, false), - case("custom/joinaggregate_movie_rating", 0.001, false), - case("custom/joinaggregate_text_color_contrast", 0.001, false), - case("custom/cumulative_running_window", 0.001, false), - case("custom/point_bubble", 0.001, false), - case("custom/circle_bubble_health_income", 0.001, false), - case("custom/line_color_stocks", 0.001, false), - case("custom/line_slope_barley", 0.001, false), - case("custom/connected_scatterplot", 0.001, false), - case("custom/layer_line_co2_concentration", 0.001, false), - case("custom/window_rank_matches", 0.001, false), - case("custom/circle_github_punchcard", 0.001, false), - case("custom/rect_lasagna", 0.001, false), - case("custom/rect_heatmap_weather", 0.001, false), - case("custom/layer_line_rolling_mean_point_raw", 0.001, false), - case("custom/layer_histogram_global_mean", 0.001, false), - case("custom/layer_precipitation_mean", 0.001, false), - case("custom/wheat_wages", 0.001, false), - case("custom/trellis_stacked_bar", 0.001, false), - case("custom/trellis_bar_histogram", 0.001, false), - case("custom/interactive_average", 0.001, false), - case("custom/histogram_responsive", 0.001, false), - case("custom/grouped_bar_chart_with_error_bars", 0.001, false), - case("custom/one_dot_per_zipcode", 0.001, false), - case("custom/ridgeline", 0.001, false), - case("custom/binned_scatter", 0.001, false), - case("custom/seattle_temps_heatmap", 0.001, false), - case("custom/movies_agg_parameterize", 0.001, false), - case("custom/escaped_column_name1", 0.001, false), - case("custom/layered_movies", 0.001, false), - case("custom/shipping_mixed_scales", 0.001, false), - case("custom/datum_color", 0.001, false), - case("custom/bug_153", 0.001, false), - case("custom/period_in_field_name", 0.001, false), - case("custom/period_space_in_field_name", 0.001, false), - case("custom/pivot_tooltip1", 0.001, true), - case("custom/sorted_pivot_lines", 0.001, false), - case("custom/stacked_bar_initial_selection", 0.001, false), - case("custom/time_boolean_bug", 0.001, false), - case("custom/full_bin_histogram_selection", 0.001, false), - case("custom/datetime_scatter", 0.001, false), - case("custom/stack_divide_by_zero_error", 0.001, false), - case("custom/casestudy-us_population_pyramid_over_time", 0.001, true), - case("custom/sin_cos", 0.001, true), - case("custom/area_streamgraph", 0.001, true), - case("custom/pivot_join_on_bug", 0.001, true), - case("custom/special_chars_bar", 0.001, true), - case("custom/bar_sort_x_axis_categorical", 0.001, true), - case("custom/taxi_dashboard", 0.001, true), - case("custom/periods_in_formula_output", 0.001, true), - case("custom/bin_transform_rounding", 0.001, true), - case("custom/geojson_inline", 0.001, true), - case("custom/gh_361", 0.001, true), - case("custom/gh_379", 0.001, true), - case("custom/gh_383", 0.001, true), - case("custom/gh_391", 0.001, true), - case("custom/facet_grouped_bar_with_error_bars", 0.001, true), - case("custom/facet_grouped_bar_with_error_bars_with_sort", 0.001, true), - case("custom/binned_ordinal", 0.001, true) + // case("custom/flights_crossfilter_a", 0.001, false), + // case("custom/flights_crossfilter_csv", 0.001, false), + // case("custom/log_scaled_histogram", 0.001, false), + // case("custom/non_linear_histogram", 0.001, false), + // case("custom/relative_frequency_histogram", 0.001, false), + // case("custom/kde_iris", 0.001, false), + // case("custom/2d_circles_histogram_imdb", 0.001, false), + // case("custom/2d_histogram_imdb", 0.001, false), + // case("custom/cumulative_window_imdb", 0.001, false), + // case("custom/density_and_cumulative_histograms", 0.001, false), + // case("custom/mean_strip_plot_movies", 0.001, false), + // case("custom/table_heatmap_cars", 0.001, false), + // case("custom/difference_from_mean", 0.001, false), + // case("custom/nested_concat_align", 0.001, false), + // case("custom/imdb_dashboard_cross_height", 0.001, false), + // case("custom/stacked_bar_weather_year", 0.001, false), + // case("custom/stacked_bar_weather_month", 0.001, false), + // case("custom/stacked_bar_normalize", 0.001, false), + // case("custom/layer_bar_labels_grey", 0.001, false), + // case("custom/bar_month_temporal_initial", 0.001, false), + // case("custom/selection_layer_bar_month", 0.001, false), + // case("custom/interactive_layered_crossfilter", 0.001, false), + // case("custom/interactive_seattle_weather", 0.001, false), + // case("custom/concat_marginal_histograms", 0.001, false), + // case("custom/joinaggregate_movie_rating", 0.001, false), + // case("custom/joinaggregate_text_color_contrast", 0.001, false), + // case("custom/cumulative_running_window", 0.001, false), + // case("custom/point_bubble", 0.001, false), + // case("custom/circle_bubble_health_income", 0.001, false), + // case("custom/line_color_stocks", 0.001, false), + // case("custom/line_slope_barley", 0.001, false), + // case("custom/connected_scatterplot", 0.001, false), + // case("custom/layer_line_co2_concentration", 0.001, false), + // case("custom/window_rank_matches", 0.001, false), + // case("custom/circle_github_punchcard", 0.001, false), + // case("custom/rect_lasagna", 0.001, false), + // case("custom/rect_heatmap_weather", 0.001, false), + // case("custom/layer_line_rolling_mean_point_raw", 0.001, false), + // case("custom/layer_histogram_global_mean", 0.001, false), + // case("custom/layer_precipitation_mean", 0.001, false), + // case("custom/wheat_wages", 0.001, false), + // case("custom/trellis_stacked_bar", 0.001, false), + // case("custom/trellis_bar_histogram", 0.001, false), + // case("custom/interactive_average", 0.001, false), + // case("custom/histogram_responsive", 0.001, false), + // case("custom/grouped_bar_chart_with_error_bars", 0.001, false), + // case("custom/one_dot_per_zipcode", 0.001, false), + // case("custom/ridgeline", 0.001, false), + // case("custom/binned_scatter", 0.001, false), + // case("custom/seattle_temps_heatmap", 0.001, false), + // case("custom/movies_agg_parameterize", 0.001, false), + // case("custom/escaped_column_name1", 0.001, false), + // case("custom/layered_movies", 0.001, false), + // case("custom/shipping_mixed_scales", 0.001, false), + // case("custom/datum_color", 0.001, false), + // case("custom/bug_153", 0.001, false), + // case("custom/period_in_field_name", 0.001, false), + // case("custom/period_space_in_field_name", 0.001, false), + // case("custom/pivot_tooltip1", 0.001, true), + // case("custom/sorted_pivot_lines", 0.001, false), + // case("custom/stacked_bar_initial_selection", 0.001, false), + // case("custom/time_boolean_bug", 0.001, false), + // case("custom/full_bin_histogram_selection", 0.001, false), + // case("custom/datetime_scatter", 0.001, false), + // case("custom/stack_divide_by_zero_error", 0.001, false), + // case("custom/casestudy-us_population_pyramid_over_time", 0.001, true), + // case("custom/sin_cos", 0.001, true), + // case("custom/area_streamgraph", 0.001, true), + // case("custom/pivot_join_on_bug", 0.001, true), + // case("custom/special_chars_bar", 0.001, true), + // case("custom/bar_sort_x_axis_categorical", 0.001, true), + // case("custom/taxi_dashboard", 0.001, true), + // case("custom/periods_in_formula_output", 0.001, true), + // case("custom/bin_transform_rounding", 0.001, true), + // case("custom/geojson_inline", 0.001, true), + // case("custom/gh_361", 0.001, true), + // case("custom/gh_379", 0.001, true), + // case("custom/gh_383", 0.001, true), + // case("custom/gh_391", 0.001, true), + // case("custom/facet_grouped_bar_with_error_bars", 0.001, true), + // case("custom/facet_grouped_bar_with_error_bars_with_sort", 0.001, true), + // case("custom/binned_ordinal", 0.001, true) )] fn test_image_comparison(spec_name: &str, tolerance: f64, extract_inline_values: bool) { println!("spec_name: {spec_name}"); diff --git a/vegafusion-runtime/tests/test_vegajs_runtime.rs b/vegafusion-runtime/tests/test_vegajs_runtime.rs index 65fb14354..bdbcc85d7 100644 --- a/vegafusion-runtime/tests/test_vegajs_runtime.rs +++ b/vegafusion-runtime/tests/test_vegajs_runtime.rs @@ -9,9 +9,10 @@ use serde_json::json; use std::collections::HashMap; use std::fs; use std::sync::Arc; +use datafusion_common::utils::array_into_list_array; -use vegafusion_common::arrow::datatypes::{DataType, Field}; use vegafusion_common::data::table::VegaFusionTable; +use vegafusion_core::arrow::array::Float64Array; use vegafusion_core::planning::watch::{ ExportUpdateBatch, ExportUpdateJSON, ExportUpdateNamespace, Watch, WatchNamespace, }; @@ -122,10 +123,7 @@ fn test_evaluate_filter_transform() { // Check extent signal assert_eq!( result_signals, - vec![ScalarValue::List( - Some(vec![ScalarValue::from(6.0), ScalarValue::from(10.0)]), - Arc::new(Field::new("item", DataType::Float64, true)) - )] + vec![ScalarValue::List(Arc::new(array_into_list_array(Arc::new(Float64Array::from(vec![6.0, 10.0])))))] ); let expected_dataset = VegaFusionTable::from_json(&json!([ diff --git a/vegafusion-sql/src/compile/expr.rs b/vegafusion-sql/src/compile/expr.rs index 4a24eef46..083441517 100644 --- a/vegafusion-sql/src/compile/expr.rs +++ b/vegafusion-sql/src/compile/expr.rs @@ -187,6 +187,7 @@ impl ToSqlExpr for Expr { SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } }; @@ -220,14 +221,17 @@ impl ToSqlExpr for Expr { TryCastMode::Supported => SqlExpr::TryCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::JustUseCast => SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::SafeCast => SqlExpr::SafeCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, }, TryCastMode::SupportedOnStringsOtherwiseJustCast => { if let DataType::Utf8 | DataType::LargeUtf8 = from_dtype { @@ -235,12 +239,14 @@ impl ToSqlExpr for Expr { SqlExpr::TryCast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } } else { // Fall back to regular CAST SqlExpr::Cast { expr: Box::new(expr.to_sql(dialect, schema)?), data_type: sql_data_type, + format: None, } } } @@ -379,6 +385,7 @@ impl ToSqlExpr for Expr { BuiltinScalarFunction::ArrayReplaceN => "array_replace_n", BuiltinScalarFunction::ArrayReplaceAll => "array_replace_all", BuiltinScalarFunction::ArraySlice => "array_slice", + BuiltinScalarFunction::ArrayIntersect => "array_intersect", BuiltinScalarFunction::Decode => "decode", BuiltinScalarFunction::Encode => "encode", BuiltinScalarFunction::Cot => "cot", @@ -387,6 +394,7 @@ impl ToSqlExpr for Expr { BuiltinScalarFunction::Nanvl => "nanvl", BuiltinScalarFunction::Flatten => "flatten", BuiltinScalarFunction::StringToArray => "string_to_array", + BuiltinScalarFunction::ToTimestampNanos => "to_timestamp_nanos", }; translate_scalar_function(fun_name, args, dialect, schema) } @@ -447,6 +455,8 @@ impl ToSqlExpr for Expr { return Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![Ident::new(alt_fun)]), args: vec![], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -533,6 +543,8 @@ impl ToSqlExpr for Expr { quote_style: None, }]), args, + filter: None, + null_treatment: None, over: Some(over), distinct: false, special: false, @@ -582,7 +594,7 @@ impl ToSqlExpr for Expr { negated: *negated, }) } - Expr::Wildcard => Err(VegaFusionError::internal( + Expr::Wildcard { .. } => Err(VegaFusionError::internal( "Wildcard cannot be converted to SQL", )), Expr::Exists { .. } => Err(VegaFusionError::internal( @@ -594,9 +606,6 @@ impl ToSqlExpr for Expr { Expr::ScalarSubquery(_) => Err(VegaFusionError::internal( "ScalarSubquery cannot be converted to SQL", )), - Expr::QualifiedWildcard { .. } => Err(VegaFusionError::internal( - "QualifiedWildcard cannot be converted to SQL", - )), Expr::GroupingSet(_) => Err(VegaFusionError::internal( "GroupingSet cannot be converted to SQL", )), @@ -631,6 +640,8 @@ fn translate_scalar_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -664,6 +675,8 @@ fn translate_aggregate_function( Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![ident]), args, + filter: None, + null_treatment: None, over: None, distinct, special: false, diff --git a/vegafusion-sql/src/compile/function_arg.rs b/vegafusion-sql/src/compile/function_arg.rs index 99b3a2f77..e257f8666 100644 --- a/vegafusion-sql/src/compile/function_arg.rs +++ b/vegafusion-sql/src/compile/function_arg.rs @@ -20,13 +20,13 @@ impl ToSqlFunctionArg for Expr { schema: &DFSchema, ) -> Result { Ok(match self { - Expr::Wildcard => SqlFunctionArgExpr::Wildcard, - Expr::QualifiedWildcard { qualifier } => { - SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { - value: qualifier.clone(), - quote_style: None, - }])) - } + Expr::Wildcard { qualifier: None } => SqlFunctionArgExpr::Wildcard, + Expr::Wildcard { + qualifier: Some(qualifier), + } => SqlFunctionArgExpr::QualifiedWildcard(ObjectName(vec![Ident { + value: qualifier.clone(), + quote_style: None, + }])), expr => SqlFunctionArgExpr::Expr(expr.to_sql(dialect, schema)?), }) } diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs index eefe47d1d..4e459a949 100644 --- a/vegafusion-sql/src/compile/scalar.rs +++ b/vegafusion-sql/src/compile/scalar.rs @@ -1,6 +1,7 @@ use crate::compile::data_type::ToSqlDataType; use crate::compile::expr::ToSqlExpr; use crate::dialect::Dialect; +use arrow::array::Array; use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::scalar::ScalarValue; use datafusion_common::DFSchema; @@ -47,6 +48,7 @@ impl ToSqlScalar for ScalarValue { false, ))), data_type: cast_dtype, + format: None, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -79,6 +81,7 @@ impl ToSqlScalar for ScalarValue { false, ))), data_type: cast_dtype, + format: None, }) } else { Ok(SqlExpr::Value(SqlValue::Null)) @@ -142,26 +145,24 @@ impl ToSqlScalar for ScalarValue { ScalarValue::FixedSizeBinary(_, _) => Err(VegaFusionError::internal( "FixedSizeBinary cannot be converted to SQL", )), - ScalarValue::List(args, _) => { + ScalarValue::List(array) => { let function_ident = Ident { value: "make_list".to_string(), quote_style: None, }; - let args = args - .clone() - .unwrap_or_default() - .iter() - .map(|expr| { - Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr( - expr.to_sql(dialect)?, - ))) + let args = (0..array.len()) + .map(|i| { + let sql_expr = ScalarValue::try_from_array(array, i)?.to_sql(dialect)?; + Ok(SqlFunctionArg::Unnamed(FunctionArgExpr::Expr(sql_expr))) }) .collect::>>()?; Ok(SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![function_ident]), args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -280,6 +281,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { None => Ok(SqlExpr::Cast { expr: Box::new(ScalarValue::Utf8(None).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, + format: None, }), Some(days) => { let date = epoch.add(chrono::Duration::days(*days as i64)); @@ -287,6 +289,7 @@ fn date32_to_date(days: &Option, dialect: &Dialect) -> Result { Ok(SqlExpr::Cast { expr: Box::new(ScalarValue::from(date_str.as_str()).to_sql(dialect)?), data_type: DataType::Date32.to_sql(dialect)?, + format: None, }) } } diff --git a/vegafusion-sql/src/compile/select.rs b/vegafusion-sql/src/compile/select.rs index b55ff858e..099e42858 100644 --- a/vegafusion-sql/src/compile/select.rs +++ b/vegafusion-sql/src/compile/select.rs @@ -2,7 +2,7 @@ use crate::compile::expr::ToSqlExpr; use crate::dialect::Dialect; use datafusion_common::DFSchema; use datafusion_expr::{expr, Expr}; -use sqlparser::ast::{Ident, SelectItem as SqlSelectItem}; +use sqlparser::ast::{Ident, ObjectName, SelectItem as SqlSelectItem}; use vegafusion_common::error::Result; pub trait ToSqlSelectItem { @@ -19,7 +19,16 @@ impl ToSqlSelectItem for Expr { quote_style: Some(dialect.quote_style), }, }, - Expr::Wildcard => SqlSelectItem::Wildcard(Default::default()), + Expr::Wildcard { qualifier: None } => SqlSelectItem::Wildcard(Default::default()), + Expr::Wildcard { + qualifier: Some(qualifier), + } => SqlSelectItem::QualifiedWildcard( + ObjectName(vec![Ident { + value: qualifier.to_string(), + quote_style: Some(dialect.quote_style), + }]), + Default::default(), + ), expr => SqlSelectItem::UnnamedExpr(expr.to_sql(dialect, schema)?), }) } diff --git a/vegafusion-sql/src/connection/datafusion_conn.rs b/vegafusion-sql/src/connection/datafusion_conn.rs index b1cd94b73..12e66ed8c 100644 --- a/vegafusion-sql/src/connection/datafusion_conn.rs +++ b/vegafusion-sql/src/connection/datafusion_conn.rs @@ -197,14 +197,12 @@ impl Connection for DataFusionConnection { writeln!(file, "{body}").unwrap(); } - let path = tempdir.path().to_str().unwrap(); - // Build final csv schema that combines the requested and inferred schemas - let final_schema = build_csv_schema(&df_csv_opts, path, &self.ctx).await?; + let final_schema = build_csv_schema(&df_csv_opts, &filepath, &self.ctx).await?; df_csv_opts = df_csv_opts.schema(&final_schema); // Load through VegaFusionTable so that temp file can be deleted - let df = self.ctx.read_csv(path, df_csv_opts).await?; + let df = self.ctx.read_csv(&filepath, df_csv_opts).await?; let schema: SchemaRef = Arc::new(df.schema().into()) as SchemaRef; let batches = df.collect().await?; diff --git a/vegafusion-sql/src/dataframe/mod.rs b/vegafusion-sql/src/dataframe/mod.rs index 5ee92268d..fa7386241 100644 --- a/vegafusion-sql/src/dataframe/mod.rs +++ b/vegafusion-sql/src/dataframe/mod.rs @@ -13,8 +13,8 @@ use datafusion_expr::{ WindowFrameBound, WindowFrameUnits, WindowFunction, }; use sqlparser::ast::{ - Cte, Expr as SqlExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, TableAlias, - TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, + Cte, Expr as SqlExpr, GroupByExpr, Ident, Query, Select, SelectItem, SetExpr, Statement, + TableAlias, TableFactor, TableWithJoins, Values, WildcardAdditionalOptions, With, }; use sqlparser::parser::Parser; use std::any::Any; @@ -278,7 +278,7 @@ impl SqlDataFrame { selection: None, from: Default::default(), lateral_views: Default::default(), - group_by: Default::default(), + group_by: GroupByExpr::Expressions(Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), @@ -320,6 +320,7 @@ impl SqlDataFrame { body: Box::new(values_body), order_by: Default::default(), limit: None, + limit_by: vec![], offset: None, fetch: None, locks: Default::default(), @@ -385,7 +386,7 @@ impl SqlDataFrame { }], lateral_views: Default::default(), selection: None, - group_by: Default::default(), + group_by: GroupByExpr::Expressions(Vec::new()), cluster_by: Default::default(), distribute_by: Default::default(), sort_by: Default::default(), @@ -398,6 +399,7 @@ impl SqlDataFrame { body: Box::new(select_body), order_by: Default::default(), limit: None, + limit_by: vec![], offset: None, fetch: None, locks: Default::default(), @@ -923,7 +925,7 @@ impl SqlDataFrame { // then union the results. This is required to make sure stacks do not overlap. Negative // values stack in the negative direction and positive values stack in the positive // direction. - let schema_exprs = vec![Expr::Wildcard, window_expr]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, window_expr]; let new_schema = make_new_schema_from_exprs( self.schema.as_ref(), schema_exprs.as_slice(), @@ -967,7 +969,10 @@ impl SqlDataFrame { // Create __stack column with numeric field let stack_col_name = "__stack"; let dataframe = self - .select(vec![Expr::Wildcard, numeric_field.alias(stack_col_name)]) + .select(vec![ + Expr::Wildcard { qualifier: None }, + numeric_field.alias(stack_col_name), + ]) .await?; let dataframe = dataframe @@ -990,7 +995,7 @@ impl SqlDataFrame { .to_string(); // Add __total column with total or total per partition - let schema_exprs = vec![Expr::Wildcard, total_agg]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, total_agg]; let new_schema = make_new_schema_from_exprs( &dataframe.schema(), schema_exprs.as_slice(), @@ -1043,7 +1048,9 @@ impl SqlDataFrame { .alias(cumulative_field); // Perform selection to add new field value - let dataframe = dataframe.select(vec![Expr::Wildcard, window_expr]).await?; + let dataframe = dataframe + .select(vec![Expr::Wildcard { qualifier: None }, window_expr]) + .await?; // Build final_selection let mut final_selection: Vec<_> = input_fields @@ -1066,7 +1073,7 @@ impl SqlDataFrame { .to_string(); // Compute new schema - let schema_exprs = vec![Expr::Wildcard, max_total]; + let schema_exprs = vec![Expr::Wildcard { qualifier: None }, max_total]; let new_schema = make_new_schema_from_exprs( &dataframe.schema(), schema_exprs.as_slice(), @@ -1395,7 +1402,7 @@ fn make_new_schema_from_exprs( ) -> Result { let mut fields: Vec = Vec::new(); for expr in exprs { - if let Expr::Wildcard = expr { + if let Expr::Wildcard { .. } = expr { // Add field for each input schema field fields.extend(schema.fields().iter().map(|f| f.as_ref().clone())); } else { diff --git a/vegafusion-sql/src/dialect/mod.rs b/vegafusion-sql/src/dialect/mod.rs index c42ee0514..06026fd4d 100644 --- a/vegafusion-sql/src/dialect/mod.rs +++ b/vegafusion-sql/src/dialect/mod.rs @@ -516,7 +516,7 @@ impl Dialect { (DataType::Float16, float64dtype.clone()), (DataType::Float32, float64dtype.clone()), (DataType::Float64, float64dtype), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -796,7 +796,7 @@ impl Dialect { (DataType::Float16, SqlDataType::Float(None)), (DataType::Float32, SqlDataType::Float(None)), (DataType::Float64, SqlDataType::Double), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -962,7 +962,7 @@ impl Dialect { (DataType::Float16, SqlDataType::Float(None)), (DataType::Float32, SqlDataType::Float(None)), (DataType::Float64, SqlDataType::Double), - (DataType::Utf8, SqlDataType::String), + (DataType::Utf8, SqlDataType::String(None)), ( DataType::Timestamp(TimeUnit::Millisecond, None), SqlDataType::Timestamp(None, TimezoneInfo::None), @@ -1750,6 +1750,8 @@ impl BinaryOperatorTransformer for ModulusOpToFunction { quote_style: None, }]), args: vec![arg0, arg1], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1793,6 +1795,8 @@ impl FunctionTransformer for RenameFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1825,6 +1829,8 @@ impl FunctionTransformer for ExpWithPowFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1854,6 +1860,7 @@ impl FunctionTransformer for CastArgsFunctionTransformer { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Cast { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: self.cast_dtype.clone(), + format: None, }))) }) .collect::>>()?; @@ -1864,6 +1871,8 @@ impl FunctionTransformer for CastArgsFunctionTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1902,6 +1911,8 @@ impl FunctionTransformer for LogBaseTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1929,6 +1940,7 @@ impl FunctionTransformer for LogBaseWithLnTransformer { Ok(FunctionArg::Unnamed(FunctionArgExpr::Expr(SqlExpr::Cast { expr: Box::new(arg.to_sql(dialect, schema)?), data_type: cast_dtype.clone(), + format: None, }))) }) .collect::>>()? @@ -1942,6 +1954,8 @@ impl FunctionTransformer for LogBaseWithLnTransformer { quote_style: None, }]), args: sql_args, + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -1957,6 +1971,8 @@ impl FunctionTransformer for LogBaseWithLnTransformer { quote_style: None, }]), args: vec![base_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs index 15f085719..245a29eb5 100644 --- a/vegafusion-sql/src/dialect/transforms/date_add_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_add_tz.rs @@ -67,6 +67,8 @@ fn maybe_from_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { SqlValue::SingleQuotedString(time_zone.to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -90,6 +92,8 @@ fn maybe_to_utc(ts_expr: SqlExpr, time_zone: &str) -> SqlExpr { SqlValue::SingleQuotedString(time_zone.to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -126,6 +130,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -150,6 +156,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { }, ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -167,6 +175,8 @@ impl FunctionTransformer for DateAddTzBigQueryTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -249,6 +259,8 @@ impl FunctionTransformer for DateAddTzDatabricksTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_in_tz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -339,6 +351,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -361,6 +375,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -384,6 +400,8 @@ impl FunctionTransformer for DateAddTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_add_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs index 810a57729..f6444205c 100644 --- a/vegafusion-sql/src/dialect/transforms/date_part_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_part_tz.rs @@ -111,6 +111,8 @@ impl FunctionTransformer for DatePartTzWithDatePartAndAtTimezoneTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -172,6 +174,8 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -202,6 +206,8 @@ impl FunctionTransformer for DatePartTzClickhouseTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( to_timezone_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -241,6 +247,8 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -259,6 +267,8 @@ impl FunctionTransformer for DatePartTzWithFromUtcAndDatePartTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -297,6 +307,8 @@ impl FunctionTransformer for DatePartTzMySqlTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -342,6 +354,8 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -360,6 +374,8 @@ impl FunctionTransformer for DatePartTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs index 098d4f330..c9df3acb1 100644 --- a/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/date_to_utc_timestamp.rs @@ -52,6 +52,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastAndAtTimeZoneTransformer let timestamp_arg = SqlExpr::Cast { expr: Box::new(date_arg), data_type: SqlDataType::Timestamp(None, SqlTimezoneInfo::None), + format: None, }; let utc_timestamp = if time_zone == "UTC" { @@ -97,6 +98,7 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { }]), Vec::new(), ), + format: None, }; if time_zone == "UTC" { @@ -117,6 +119,8 @@ impl FunctionTransformer for DateToUtcTimestampSnowflakeTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -158,6 +162,8 @@ impl FunctionTransformer for DateToUtcTimestampWithFunctionTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -198,6 +204,7 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(date_arg), data_type: self.timestamp_type.clone(), + format: None, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -210,6 +217,8 @@ impl FunctionTransformer for DateToUtcTimestampWithCastFunctionAtTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -252,6 +261,8 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -269,6 +280,8 @@ impl FunctionTransformer for DateToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -302,6 +315,8 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_arg))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -326,6 +341,8 @@ impl FunctionTransformer for DateToUtcTimestampMySqlTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs index bfeae3625..d03f0c948 100644 --- a/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs +++ b/vegafusion-sql/src/dialect/transforms/date_trunc_tz.rs @@ -74,6 +74,8 @@ impl FunctionTransformer for DateTruncTzWithDateTruncAndAtTimezoneTransformer { quote_style: None, }]), args: vec![part_func_arg, ts_func_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -130,6 +132,8 @@ impl FunctionTransformer for DateTruncTzWithTimestampTruncTransformer { quote_style: None, }]), args: vec![ts_func_arg, part_func_arg, tz_func_arg], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -179,6 +183,8 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -199,6 +205,8 @@ impl FunctionTransformer for DateTruncTzClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -241,6 +249,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { SqlValue::SingleQuotedString(time_zone.clone()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -259,6 +269,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -279,6 +291,8 @@ impl FunctionTransformer for DateTruncTzWithFromUtcAndDateTruncTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -320,6 +334,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg1)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -338,6 +354,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(timestamp_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -361,6 +379,8 @@ impl FunctionTransformer for DateTruncTzSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_trunc_in_tz)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs index a32b04905..322b8843b 100644 --- a/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/epoch_ms_to_utc_timestamp.rs @@ -44,6 +44,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampBigQueryTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -88,6 +90,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( div_1000_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -101,6 +105,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( floor_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -119,6 +125,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(mod_1000_expr)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(from_unix_time_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -150,6 +158,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampDuckDbTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ms_expr))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -194,6 +204,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( div_1000_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -207,6 +219,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampPostgresTransformer { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( floor_expr, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -270,6 +284,8 @@ impl FunctionTransformer for EpochMsToUtcTimestampSnowflakeTransformer { SqlValue::Number("3".to_string(), false), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs index 2b7203e9f..042463fc4 100644 --- a/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/make_utc_timestamp.rs @@ -104,6 +104,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.month)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.day)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -120,6 +122,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -143,6 +147,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { }, ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -158,6 +164,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(date_expr)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(time_with_ms_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -175,6 +183,8 @@ impl FunctionTransformer for MakeUtcTimestampBigQueryTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -215,6 +225,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -234,6 +246,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.millisecond)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -254,6 +268,8 @@ impl FunctionTransformer for MakeUtcTimestampDatabricksTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -295,6 +311,8 @@ impl FunctionTransformer for MakeUtcTimestampDuckDbTransformer { SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.minute)), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_args.second)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -348,6 +366,8 @@ impl FunctionTransformer for MakeUtcTimestampPostgresTransformer { SqlValue::SingleQuotedString(sql_args.time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -402,6 +422,8 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { ))), })), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -425,6 +447,8 @@ impl FunctionTransformer for MakeUtcTimestampSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(make_timestamp_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs index 6f888a0fe..a14e2e087 100644 --- a/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/str_to_utc_timestamp.rs @@ -54,6 +54,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastAndAtTimeZoneTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), + format: None, }; let utc_expr = if time_zone == "UTC" { cast_expr @@ -103,6 +104,7 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { let cast_expr = SqlExpr::Cast { expr: Box::new(sql_arg0), data_type: self.timestamp_type.clone(), + format: None, }; let fn_expr = SqlExpr::Function(SqlFunction { name: SqlObjectName(vec![SqlIdent { @@ -115,6 +117,8 @@ impl FunctionTransformer for StrToUtcTimestampWithCastFunctionAtTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -161,6 +165,8 @@ impl FunctionTransformer for StrToUtcTimestampWithFunctionTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -195,6 +201,8 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -212,6 +220,8 @@ impl FunctionTransformer for StrToUtcTimestampClickhouseTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -245,6 +255,8 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(sql_arg0))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -269,6 +281,8 @@ impl FunctionTransformer for StrToUtcTimestampMySqlTransformer { SqlValue::SingleQuotedString("UTC".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -306,6 +320,7 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { }]), Vec::new(), ), + format: None, }; if time_zone == "UTC" { @@ -326,6 +341,8 @@ impl FunctionTransformer for StrToUtcTimestampSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(cast_timestamp_ntz_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs index dc573974f..05b5e23ab 100644 --- a/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs +++ b/vegafusion-sql/src/dialect/transforms/to_utc_timestamp.rs @@ -117,6 +117,8 @@ impl FunctionTransformer for ToUtcTimestampSnowflakeTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -155,6 +157,8 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { quote_style: None, }]), args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_arg))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -172,6 +176,8 @@ impl FunctionTransformer for ToUtcTimestampBigQueryTransform { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs index 59c5ad598..f125d11ee 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_epoch_ms.rs @@ -46,6 +46,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( ts_expr.clone(), ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -68,6 +70,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDatabricksTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -116,6 +120,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { args: vec![SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr( ts_expr.clone(), ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -138,6 +144,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsDuckdbTransform { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -189,6 +197,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsPostgresTransform { right: Box::new(SqlExpr::Value(SqlValue::Number("1000".to_string(), false))), }, ))], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -226,6 +236,8 @@ impl FunctionTransformer for UtcTimestampToEpochMsSnowflakeTransform { }))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, diff --git a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs index 3705124fe..79b2c29cb 100644 --- a/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs +++ b/vegafusion-sql/src/dialect/transforms/utc_timestamp_to_str.rs @@ -58,6 +58,8 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -75,6 +77,8 @@ impl FunctionTransformer for UtcTimestampToStrBigQueryTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(datetime_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -114,6 +118,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString(time_zone), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -132,6 +138,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString("y-MM-dd HH:mm:ss.SSS".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -154,6 +162,8 @@ impl FunctionTransformer for UtcTimestampToStrDatabricksTransformer { SqlValue::SingleQuotedString("T".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -204,6 +214,8 @@ impl FunctionTransformer for UtcTimestampToStrDuckDBTransformer { SqlValue::SingleQuotedString("%Y-%m-%dT%H:%M:%S.%g".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -254,6 +266,8 @@ impl FunctionTransformer for UtcTimestampToStrPostgresTransformer { SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.MS".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -298,6 +312,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { ))), SqlFunctionArg::Unnamed(SqlFunctionArgExpr::Expr(ts_expr)), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false, @@ -316,6 +332,8 @@ impl FunctionTransformer for UtcTimestampToStrSnowflakeTransformer { SqlValue::SingleQuotedString("YYYY-MM-DD\"T\"HH24:MI:SS.FF3".to_string()), ))), ], + filter: None, + null_treatment: None, over: None, distinct: false, special: false,