diff --git a/Cargo.lock b/Cargo.lock index bbd9f7474..6ca4adbee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -358,12 +358,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" - [[package]] name = "base64" version = "0.21.0" @@ -502,9 +496,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", + "js-sys", "num-integer", "num-traits", "serde", + "time", + "wasm-bindgen", "winapi", ] @@ -679,7 +676,7 @@ dependencies = [ [[package]] name = "datafusion" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "ahash", "apache-avro", @@ -727,7 +724,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "apache-avro", "arrow", @@ -742,7 +739,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "ahash", "arrow", @@ -754,7 +751,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "arrow", "async-trait", @@ -770,7 +767,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "ahash", "arrow", @@ -821,7 +818,7 @@ dependencies = [ [[package]] name = "datafusion-row" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "arrow", "datafusion-common", @@ -832,7 +829,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "arrow-schema", "datafusion-common", @@ -844,11 +841,13 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "17.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20" dependencies = [ "async-recursion", + "chrono", "datafusion", "itertools", + "object_store", "prost 0.11.6", "prost-build 0.9.0", "prost-types 0.11.6", @@ -1051,7 +1050,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1545,7 +1544,7 @@ checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.42.0", ] @@ -1644,12 +1643,12 @@ dependencies = [ [[package]] name = "object_store" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4201837dc4c27a8670f0363b1255cd3845a4f0c521211cced1ed14c1d0cc6d2" +checksum = "1f344e51ec9584d2f51199c0c29c6f73dddd04ade986497875bf8fa2f178caf0" dependencies = [ "async-trait", - "base64 0.20.0", + "base64", "bytes", "chrono", "futures", @@ -1722,7 +1721,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.0", + "base64", "brotli", "bytes", "chrono", @@ -1755,9 +1754,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.5.4" +version = "2.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab62d2fa33726dbe6321cc97ef96d8cde531e3eeaf858a058de53a8a6d40d8f" +checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" dependencies = [ "thiserror", "ucd-trie", @@ -1920,9 +1919,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd4149c8c3975099622b4e1962dac27565cf5663b76452c3e2b66e0b6824277" +checksum = "06a3d8e8a46ab2738109347433cb7b96dffda2e4a218b03ef27090238886b147" dependencies = [ "cfg-if", "indoc", @@ -1937,9 +1936,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd09fe469834db21ee60e0051030339e5d361293d8cb5ec02facf7fdcf52dbf" +checksum = "75439f995d07ddfad42b192dfcf3bc66a7ecfd8b4a1f5f6f046aa5c2c5d7677d" dependencies = [ "once_cell", "target-lexicon", @@ -1947,9 +1946,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c427c9a96b9c5b12156dbc11f76b14f49e9aae8905ca783ea87c249044ef137" +checksum = "839526a5c07a17ff44823679b68add4a58004de00512a95b6c1c98a6dcac0ee5" dependencies = [ "libc", "pyo3-build-config", @@ -1957,9 +1956,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b822bbba9d60630a44d2109bc410489bb2f439b33e3a14ddeb8a40b378a7c4" +checksum = "bd44cf207476c6a9760c4653559be4f206efafb924d3e4cbf2721475fc0d6cc5" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1969,9 +1968,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ae898104f7c99db06231160770f3e40dad6eb9021daddc0fedfa3e41dff10a" +checksum = "dc1f43d8e30460f36350d18631ccf85ded64c059829208fe680904c65bcd0a4c" dependencies = [ "proc-macro2", "quote", @@ -2089,7 +2088,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ - "base64 0.21.0", + "base64", "bytes", "encoding_rs", "futures-core", @@ -2185,7 +2184,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64", ] [[package]] @@ -2318,9 +2317,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ "itoa 1.0.5", "ryu", @@ -2574,6 +2573,17 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2652,9 +2662,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "bc6a3b08b64e6dfad376fa2432c7b1f01522e37a623c3050bc95db2d3ff21583" dependencies = [ "bytes", "futures-core", @@ -2907,6 +2917,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3194,9 +3210,9 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.3+zstd.1.5.2" +version = "6.0.4+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e4a3f57d13d0ab7e478665c60f35e2a613dcd527851c2c7287ce5c787e134a" +checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" dependencies = [ "libc", "zstd-sys", @@ -3204,9 +3220,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.6+zstd.1.5.2" +version = "2.0.7+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a3f9792c0c3dc6c165840a75f47ae1f4da402c2d006881129579f6597e801b" +checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index d1657f9cd..30e04545e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,11 +34,11 @@ default = ["mimalloc"] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.0", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { git="https://github.com/apache/arrow-datafusion.git", rev="a265509b272084e758b9ec21f2133856f56e617d", features = ["pyarrow", "avro"] } -datafusion-expr = { git="https://github.com/apache/arrow-datafusion.git", rev="a265509b272084e758b9ec21f2133856f56e617d" } -datafusion-optimizer = { git="https://github.com/apache/arrow-datafusion.git", rev="a265509b272084e758b9ec21f2133856f56e617d" } -datafusion-common = { git="https://github.com/apache/arrow-datafusion.git", rev="a265509b272084e758b9ec21f2133856f56e617d", features = ["pyarrow"] } -datafusion-substrait = { git="https://github.com/apache/arrow-datafusion.git", rev="a265509b272084e758b9ec21f2133856f56e617d" } +datafusion = { git="https://github.com/apache/arrow-datafusion.git", rev="8a262c3ec0a100607ae97641d7a374c847a71f20", features = ["pyarrow", "avro"] } +datafusion-expr = { git="https://github.com/apache/arrow-datafusion.git", rev="8a262c3ec0a100607ae97641d7a374c847a71f20" } +datafusion-optimizer = { git="https://github.com/apache/arrow-datafusion.git", rev="8a262c3ec0a100607ae97641d7a374c847a71f20" } +datafusion-common = { git="https://github.com/apache/arrow-datafusion.git", rev="8a262c3ec0a100607ae97641d7a374c847a71f20", features = ["pyarrow"] } +datafusion-substrait = { git="https://github.com/apache/arrow-datafusion.git", rev="8a262c3ec0a100607ae97641d7a374c847a71f20" } uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "*", optional = true, default-features = false } async-trait = "0.1" diff --git a/src/substrait.rs b/src/substrait.rs index 0c42e2104..b9e5c9257 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -19,10 +19,12 @@ use pyo3::prelude::*; use crate::context::PySessionContext; use crate::errors::py_datafusion_err; +use crate::errors::DataFusionError; use crate::{logical::PyLogicalPlan, utils::wait_for_future}; -use crate::errors::DataFusionError; -use datafusion_substrait::{consumer, producer, serializer, substrait::proto::Plan}; +use datafusion_substrait::logical_plan::{consumer, producer}; +use datafusion_substrait::serializer; +use datafusion_substrait::substrait::proto::Plan; #[pyclass(name = "plan", module = "datafusion.substrait", subclass, unsendable)] #[derive(Debug, Clone)]