From 1cec6c29dc538724e05691a31e092b344d8e1b44 Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Tue, 16 Apr 2024 11:25:28 -0500 Subject: [PATCH] deps: upgrade pyo3 and arrow deps Arrow needs to be upgraded alongside pyo3 because they both link to python. Arrow's upgrade to pyo3 is in master but not yet released as of v51. https://github.com/apache/arrow-rs/pull/5566 NOTE: Arrow marked `ffi::from_ffi` unsafe. https://github.com/apache/arrow-rs/pull/5080 --- Cargo.lock | 151 +++++++++++--------- Cargo.toml | 12 +- apis/rust/operator/src/raw.rs | 2 +- apis/rust/operator/types/src/lib.rs | 2 +- binaries/runtime/src/operator/shared_lib.rs | 2 +- libraries/extensions/dora-record/Cargo.toml | 2 +- 6 files changed, 92 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59f4aa42e..d0a475fb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -161,11 +161,9 @@ checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" [[package]] name = "arrow" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb738d83750ec705808f6d44046d165e6bb8623f64e29a4d53fcb136ab22dfb" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ - "ahash", "arrow-arith", "arrow-array", "arrow-buffer", @@ -184,9 +182,8 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c3d17fc5b006e7beeaebfb1d2edfc92398b981f82d9744130437909b72a468" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", @@ -199,9 +196,8 @@ dependencies = [ [[package]] name = "arrow-array" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55705ada5cdde4cb0f202ffa6aa756637e33fea30e13d8d0d0fd6a24ffcee1e3" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "ahash", "arrow-buffer", @@ -215,9 +211,8 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a722f90a09b94f295ab7102542e97199d3500128843446ef63e410ad546c5333" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "bytes", "half", @@ -226,26 +221,27 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af01fc1a06f6f2baf31a04776156d47f9f31ca5939fe6d00cd7a059f95a46ff1" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "atoi", + "base64 0.22.0", "chrono", "half", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83cbbfde86f9ecd3f875c42a73d8aeab3d95149cd80129b18d09e039ecf5391b" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +258,8 @@ dependencies = [ [[package]] name = "arrow-data" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a547195e607e625e7fafa1a7269b8df1a4a612c919efd9b26bd86e74538f3a" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-buffer", "arrow-schema", @@ -274,9 +269,8 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36bf091502ab7e37775ff448413ef1ffff28ff93789acb669fffdd51b394d51" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", @@ -288,9 +282,8 @@ dependencies = [ [[package]] name = "arrow-json" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ac346bc84846ab425ab3c8c7b6721db90643bc218939677ed7e071ccbfb919d" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", @@ -308,9 +301,8 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4502123d2397319f3a13688432bc678c61cb1582f2daa01253186da650bf5841" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", @@ -323,9 +315,8 @@ dependencies = [ [[package]] name = "arrow-row" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249fc5a07906ab3f3536a6e9f118ec2883fbcde398a97a5ba70053f0276abda4" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "ahash", "arrow-array", @@ -338,9 +329,8 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7a8c3f97f5ef6abd862155a6f39aaba36b029322462d72bbcfa69782a50614" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "bitflags 2.4.0", "serde", @@ -348,9 +338,8 @@ dependencies = [ [[package]] name = "arrow-select" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f868f4a5001429e20f7c1994b5cd1aa68b82e3db8cf96c559cdb56dc8be21410" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "ahash", "arrow-array", @@ -362,15 +351,15 @@ dependencies = [ [[package]] name = "arrow-string" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27fdf8fc70040a2dee78af2e217479cb5b263bd7ab8711c7999e74056eb688a" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", "regex-syntax 0.8.2", @@ -565,6 +554,15 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.5.3" @@ -666,6 +664,12 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64ct" version = "1.6.0" @@ -798,9 +802,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -809,9 +813,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -2030,9 +2034,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -3167,9 +3171,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memoffset" @@ -3915,9 +3919,8 @@ dependencies = [ [[package]] name = "parquet" -version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239229e6a668ab50c61de3dce61cf0fa1069345f7aa0f4c934491f92205a4945" +version = "51.0.0" +source = "git+https://github.com/apache/arrow-rs#f2765283b67e64a6923352ed23398ca9ca194ef2" dependencies = [ "ahash", "arrow-array", @@ -3927,12 +3930,13 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.4", + "base64 0.22.0", "brotli", "bytes", "chrono", "flate2", "futures", + "half", "hashbrown 0.14.3", "lz4_flex", "num", @@ -3944,6 +3948,7 @@ dependencies = [ "tokio", "twox-hash", "zstd", + "zstd-sys", ] [[package]] @@ -4221,6 +4226,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4301,9 +4312,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ "cfg-if 1.0.0", "eyre", @@ -4311,6 +4322,7 @@ dependencies = [ "libc", "memoffset 0.9.0", "parking_lot", + "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -4320,9 +4332,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -4330,9 +4342,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -4340,9 +4352,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4352,21 +4364,22 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck 0.4.1", "proc-macro2", + "pyo3-build-config", "quote", "syn 2.0.48", ] [[package]] name = "pythonize" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850" +checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c" dependencies = [ "pyo3", "serde", @@ -4938,9 +4951,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "safemem" diff --git a/Cargo.toml b/Cargo.toml index 0ad7abd0b..1a5db333d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,12 +65,12 @@ dora-coordinator = { version = "0.3.3", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-msg-gen = { path = "libraries/extensions/ros2-bridge/msg-gen" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } -arrow = "48.0.0" -arrow-schema = "48.0.0" -arrow-data = "48.0.0" -arrow-array = "48.0.0" -pyo3 = "0.20.0" -pythonize = "0.20.0" +arrow = { git = "https://github.com/apache/arrow-rs" } +arrow-schema = { git = "https://github.com/apache/arrow-rs" } +arrow-data = { git = "https://github.com/apache/arrow-rs" } +arrow-array = { git = "https://github.com/apache/arrow-rs" } +pyo3 = "0.21" +pythonize = "0.21" [package] name = "dora-examples" diff --git a/apis/rust/operator/src/raw.rs b/apis/rust/operator/src/raw.rs index 2ff89c64d..6634e68fb 100644 --- a/apis/rust/operator/src/raw.rs +++ b/apis/rust/operator/src/raw.rs @@ -44,7 +44,7 @@ pub unsafe fn dora_on_event( status: DoraStatus::Continue, }; }; - let data = arrow::ffi::from_ffi(data_array, &input.schema); + let data = unsafe { arrow::ffi::from_ffi(data_array, &input.schema) }; match data { Ok(data) => Event::Input { diff --git a/apis/rust/operator/types/src/lib.rs b/apis/rust/operator/types/src/lib.rs index 7bff62348..7f299b008 100644 --- a/apis/rust/operator/types/src/lib.rs +++ b/apis/rust/operator/types/src/lib.rs @@ -165,7 +165,7 @@ pub fn dora_free_input_id(_input_id: char_p_boxed) {} #[ffi_export] pub fn dora_read_data(input: &mut Input) -> Option> { let data_array = input.data_array.take()?; - let data = arrow::ffi::from_ffi(data_array, &input.schema).ok()?; + let data = unsafe {arrow::ffi::from_ffi(data_array, &input.schema).ok()? }; let array = ArrowData(arrow::array::make_array(data)); let bytes: &[u8] = TryFrom::try_from(&array).ok()?; Some(bytes.to_owned().into()) diff --git a/binaries/runtime/src/operator/shared_lib.rs b/binaries/runtime/src/operator/shared_lib.rs index 811c3cd07..984a760b9 100644 --- a/binaries/runtime/src/operator/shared_lib.rs +++ b/binaries/runtime/src/operator/shared_lib.rs @@ -124,7 +124,7 @@ impl<'lib> SharedLibraryOperator<'lib> { ..Default::default() }; - let arrow_array = match arrow::ffi::from_ffi(data_array, &schema) { + let arrow_array = match unsafe { arrow::ffi::from_ffi(data_array, &schema) } { Ok(a) => a, Err(err) => return DoraResult::from_error(err.to_string()), }; diff --git a/libraries/extensions/dora-record/Cargo.toml b/libraries/extensions/dora-record/Cargo.toml index 9d7077169..c1079da19 100644 --- a/libraries/extensions/dora-record/Cargo.toml +++ b/libraries/extensions/dora-record/Cargo.toml @@ -14,4 +14,4 @@ dora-node-api = { workspace = true, features = ["tracing"] } eyre = "0.6.8" chrono = "0.4.31" dora-tracing = { workspace = true } -parquet = { version = "48.0.0", features = ["async"] } +parquet = { git = "https://github.com/apache/arrow-rs", features = ["async"] }