diff --git a/Cargo.lock b/Cargo.lock index aa1044954..816e180f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,11 +317,10 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ - "ahash", "arrow-arith", "arrow-array", "arrow-buffer", @@ -340,9 +339,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef983914f477d4278b068f13b3224b7d19eb2b807ac9048544d3bfebdf2554c4" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ "arrow-array", "arrow-buffer", @@ -355,9 +354,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash", "arrow-buffer", @@ -371,9 +370,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55512d988c6fbd76e514fd3ff537ac50b0a675da5a245e4fdad77ecfd654205f" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" dependencies = [ "bytes", "half", @@ -382,26 +381,29 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655ee51a2156ba5375931ce21c1b2494b1d9260e6dcdc6d4db9060c37dc3325b" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "atoi", + "base64 0.22.1", "chrono", "half", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bb689997ad5b6660b3ce3638bd6b383d668ec555ed41ad7c6559cbb2e4f91" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ "arrow-array", "arrow-buffer", @@ -418,9 +420,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc2b9fec74763427e2e5575b8cc31ce96ba4c9b4eb05ce40e0616d9fad12461" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ "arrow-buffer", "arrow-schema", @@ -440,23 +442,23 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eaa6ab203cc6d89b7eaa1ac781c1dfeef325454c5d5a0419017f95e6bafc03c" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", "arrow-data", "arrow-schema", - "flatbuffers", + "flatbuffers 24.3.25", ] [[package]] name = "arrow-json" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb64e30d9b73f66fdc5c52d5f4cf69bbf03d62f64ffeafa0715590a5320baed7" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -474,9 +476,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a818951c0d11c428dda03e908175969c262629dd20bd0850bd6c7a8c3bfe48" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ "arrow-array", "arrow-buffer", @@ -489,9 +491,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash", "arrow-array", @@ -504,9 +506,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf4d737bba93da59f16129bec21e087aed0be84ff840e74146d4703879436cb" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" dependencies = [ "bitflags 2.5.0", "serde", @@ -514,9 +516,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash", "arrow-array", @@ -528,15 +530,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15aed5624bb23da09142f58502b59c23f5bea607393298bb81dab1ce60fc769" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", "regex-syntax 0.8.3", @@ -871,6 +874,15 @@ dependencies = [ "syn 2.0.65", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.5.3" @@ -1228,9 +1240,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.5.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1239,9 +1251,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -3197,6 +3209,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flatbuffers" +version = "24.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.0.30" @@ -3924,7 +3946,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.7", "tokio", "tower-service", "tracing", @@ -5855,9 +5877,9 @@ dependencies = [ [[package]] name = "parquet" -version = "48.0.1" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597" +checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash", "arrow-array", @@ -5867,12 +5889,13 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.7", + "base64 0.22.1", "brotli", "bytes", "chrono", "flate2", "futures", + "half", "hashbrown 0.14.5", "lz4_flex", "num", @@ -5884,6 +5907,7 @@ dependencies = [ "tokio", "twox-hash", "zstd", + "zstd-sys", ] [[package]] @@ -6453,9 +6477,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ "cfg-if 1.0.0", "eyre", @@ -6473,9 +6497,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -6483,9 +6507,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -6493,9 +6517,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -6505,9 +6529,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -6518,9 +6542,9 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850" +checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c" dependencies = [ "pyo3", "serde", @@ -7453,7 +7477,7 @@ dependencies = [ "camino", "clang-format", "convert_case", - "flatbuffers", + "flatbuffers 23.5.26", "indent", "itertools 0.12.1", "prettyplease 0.2.20", @@ -9559,7 +9583,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "static_assertions", ] @@ -11326,27 +11350,27 @@ checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index ddabbf53b..de1899966 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,12 +66,12 @@ dora-coordinator = { version = "0.3.4", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-msg-gen = { path = "libraries/extensions/ros2-bridge/msg-gen" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } -arrow = "48.0.0" -arrow-schema = "48.0.0" -arrow-data = "48.0.0" -arrow-array = "48.0.0" -pyo3 = "0.20.0" -pythonize = "0.20.0" +arrow = { version = "52" } +arrow-schema = { version = "52" } +arrow-data = { version = "52" } +arrow-array = { version = "52" } +pyo3 = "0.21" +pythonize = "0.21" [package] name = "dora-examples" diff --git a/apis/python/node/src/lib.rs b/apis/python/node/src/lib.rs index 95fd29c7a..ba9c6c46f 100644 --- a/apis/python/node/src/lib.rs +++ b/apis/python/node/src/lib.rs @@ -122,17 +122,17 @@ impl Node { &mut self, output_id: String, data: PyObject, - metadata: Option<&PyDict>, + metadata: Option>, py: Python, ) -> eyre::Result<()> { let parameters = pydict_to_metadata(metadata)?; - if let Ok(py_bytes) = data.downcast::(py) { + if let Ok(py_bytes) = data.downcast_bound::(py) { let data = py_bytes.as_bytes(); self.node .send_output_bytes(output_id.into(), parameters, data.len(), data) .wrap_err("failed to send output")?; - } else if let Ok(arrow_array) = arrow::array::ArrayData::from_pyarrow(data.as_ref(py)) { + } else if let Ok(arrow_array) = arrow::array::ArrayData::from_pyarrow_bound(data.bind(py)) { self.node.send_output( output_id.into(), parameters, @@ -251,9 +251,10 @@ pub fn start_runtime() -> eyre::Result<()> { } #[pymodule] -fn dora(_py: Python, m: &PyModule) -> PyResult<()> { - dora_ros2_bridge_python::create_dora_ros2_bridge_module(m)?; - m.add_function(wrap_pyfunction!(start_runtime, m)?)?; +fn dora(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { + dora_ros2_bridge_python::create_dora_ros2_bridge_module(&m)?; + + m.add_function(wrap_pyfunction!(start_runtime, &m)?)?; m.add_class::()?; m.add_class::()?; m.setattr("__version__", env!("CARGO_PKG_VERSION"))?; diff --git a/apis/python/operator/src/lib.rs b/apis/python/operator/src/lib.rs index c57960ea8..bc74279e9 100644 --- a/apis/python/operator/src/lib.rs +++ b/apis/python/operator/src/lib.rs @@ -1,7 +1,7 @@ use arrow::{array::ArrayRef, pyarrow::ToPyArrow}; use dora_node_api::{merged::MergedEvent, Event, Metadata, MetadataParameters}; use eyre::{Context, Result}; -use pyo3::{exceptions::PyLookupError, prelude::*, types::PyDict}; +use pyo3::{exceptions::PyLookupError, prelude::*, pybacked::PyBackedStr, types::PyDict}; /// Dora Event #[pyclass] @@ -126,11 +126,15 @@ impl From> for PyEvent { } } -pub fn pydict_to_metadata(dict: Option<&PyDict>) -> Result { +pub fn pydict_to_metadata(dict: Option>) -> Result { let mut default_metadata = MetadataParameters::default(); if let Some(metadata) = dict { for (key, value) in metadata.iter() { - match key.extract::<&str>().context("Parsing metadata keys")? { + match key + .extract::() + .context("Parsing metadata keys")? + .as_ref() + { "watermark" => { default_metadata.watermark = value.extract().context("parsing watermark failed")?; @@ -140,7 +144,7 @@ pub fn pydict_to_metadata(dict: Option<&PyDict>) -> Result { value.extract().context("parsing deadline failed")?; } "open_telemetry_context" => { - let otel_context: &str = value + let otel_context: PyBackedStr = value .extract() .context("parsing open telemetry context failed")?; default_metadata.open_telemetry_context = otel_context.to_string(); @@ -152,8 +156,8 @@ pub fn pydict_to_metadata(dict: Option<&PyDict>) -> Result { Ok(default_metadata) } -pub fn metadata_to_pydict<'a>(metadata: &'a Metadata, py: Python<'a>) -> &'a PyDict { - let dict = PyDict::new(py); +pub fn metadata_to_pydict<'a>(metadata: &'a Metadata, py: Python<'a>) -> pyo3::Bound<'a, PyDict> { + let dict = PyDict::new_bound(py); dict.set_item( "open_telemetry_context", &metadata.parameters.open_telemetry_context, diff --git a/apis/rust/operator/src/raw.rs b/apis/rust/operator/src/raw.rs index 2ff89c64d..6634e68fb 100644 --- a/apis/rust/operator/src/raw.rs +++ b/apis/rust/operator/src/raw.rs @@ -44,7 +44,7 @@ pub unsafe fn dora_on_event( status: DoraStatus::Continue, }; }; - let data = arrow::ffi::from_ffi(data_array, &input.schema); + let data = unsafe { arrow::ffi::from_ffi(data_array, &input.schema) }; match data { Ok(data) => Event::Input { diff --git a/apis/rust/operator/types/src/lib.rs b/apis/rust/operator/types/src/lib.rs index 4f562e9dc..d235114dd 100644 --- a/apis/rust/operator/types/src/lib.rs +++ b/apis/rust/operator/types/src/lib.rs @@ -166,7 +166,7 @@ pub fn dora_free_input_id(_input_id: char_p_boxed) {} #[ffi_export] pub fn dora_read_data(input: &mut Input) -> Option> { let data_array = input.data_array.take()?; - let data = arrow::ffi::from_ffi(data_array, &input.schema).ok()?; + let data = unsafe { arrow::ffi::from_ffi(data_array, &input.schema).ok()? }; let array = ArrowData(arrow::array::make_array(data)); let bytes: &[u8] = TryFrom::try_from(&array).ok()?; Some(bytes.to_owned().into()) diff --git a/binaries/runtime/src/operator/python.rs b/binaries/runtime/src/operator/python.rs index d4bebf665..82d01ebb9 100644 --- a/binaries/runtime/src/operator/python.rs +++ b/binaries/runtime/src/operator/python.rs @@ -12,7 +12,7 @@ use dora_operator_api_types::DoraStatus; use eyre::{bail, eyre, Context, Result}; use pyo3::{ pyclass, - types::{IntoPyDict, PyDict}, + types::{IntoPyDict, PyAnyMethods, PyDict, PyTracebackMethods}, Py, PyAny, Python, }; use std::{ @@ -23,7 +23,7 @@ use tokio::sync::{mpsc::Sender, oneshot}; use tracing::{error, field, span, warn}; fn traceback(err: pyo3::PyErr) -> eyre::Report { - let traceback = Python::with_gil(|py| err.traceback(py).and_then(|t| t.format().ok())); + let traceback = Python::with_gil(|py| err.traceback_bound(py).and_then(|t| t.format().ok())); if let Some(traceback) = traceback { eyre::eyre!("{traceback}\n{err}") } else { @@ -78,7 +78,9 @@ pub fn run( let parent_path = parent_path .to_str() .ok_or_else(|| eyre!("module path is not valid utf8"))?; - let sys = py.import("sys").wrap_err("failed to import `sys` module")?; + let sys = py + .import_bound("sys") + .wrap_err("failed to import `sys` module")?; let sys_path = sys .getattr("path") .wrap_err("failed to import `sys.path` module")?; @@ -90,14 +92,14 @@ pub fn run( .wrap_err("failed to append module path to python search path")?; } - let module = py.import(module_name).map_err(traceback)?; + let module = py.import_bound(module_name).map_err(traceback)?; let operator_class = module .getattr("Operator") .wrap_err("no `Operator` class found in module")?; - let locals = [("Operator", operator_class)].into_py_dict(py); + let locals = [("Operator", operator_class)].into_py_dict_bound(py); let operator = py - .eval("Operator()", None, Some(locals)) + .eval_bound("Operator()", None, Some(&locals)) .map_err(traceback)?; operator.setattr( "dataflow_descriptor", @@ -141,11 +143,11 @@ pub fn run( .wrap_err("could not extract operator state as a PyDict")?; // Reload module let module = py - .import(module_name) + .import_bound(module_name) .map_err(traceback) .wrap_err(format!("Could not retrieve {module_name} while reloading"))?; let importlib = py - .import("importlib") + .import_bound("importlib") .wrap_err("failed to import `importlib` module")?; let module = importlib .call_method("reload", (module,), None) @@ -155,9 +157,9 @@ pub fn run( .wrap_err("no `Operator` class found in module")?; // Create a new reloaded operator - let locals = [("Operator", reloaded_operator_class)].into_py_dict(py); + let locals = [("Operator", reloaded_operator_class)].into_py_dict_bound(py); let operator: Py = py - .eval("Operator()", None, Some(locals)) + .eval_bound("Operator()", None, Some(&locals)) .map_err(traceback) .wrap_err("Could not initialize reloaded operator")? .into(); @@ -185,17 +187,6 @@ pub fn run( let status = Python::with_gil(|py| -> Result { let span = span!(tracing::Level::TRACE, "on_event", input_id = field::Empty); let _ = span.enter(); - // We need to create a new scoped `GILPool` because the dora-runtime - // is currently started through a `start_runtime` wrapper function, - // which is annotated with `#[pyfunction]`. This attribute creates an - // initial `GILPool` that lasts for the entire lifetime of the `dora-runtime`. - // However, we want the `PyBytes` created below to be freed earlier. - // creating a new scoped `GILPool` tied to this closure, will free `PyBytes` - // at the end of the closure. - // See https://github.com/PyO3/pyo3/pull/2864 and - // https://github.com/PyO3/pyo3/issues/2853 for more details. - let pool = unsafe { py.new_pool() }; - let py = pool.python(); // Add metadata context if we have a tracer and // incoming input has some metadata. @@ -300,8 +291,8 @@ mod callback_impl { use eyre::{eyre, Context, Result}; use pyo3::{ pymethods, - types::{PyBytes, PyDict}, - PyObject, Python, + types::{PyBytes, PyBytesMethods, PyDict}, + Bound, PyObject, Python, }; use tokio::sync::oneshot; use tracing::{field, span}; @@ -318,7 +309,7 @@ mod callback_impl { &mut self, output: &str, data: PyObject, - metadata: Option<&PyDict>, + metadata: Option>, py: Python, ) -> Result<()> { let parameters = pydict_to_metadata(metadata) @@ -354,12 +345,12 @@ mod callback_impl { } }; - let (sample, type_info) = if let Ok(py_bytes) = data.downcast::(py) { + let (sample, type_info) = if let Ok(py_bytes) = data.downcast_bound::(py) { let data = py_bytes.as_bytes(); let mut sample = allocate_sample(data.len())?; sample.copy_from_slice(data); (sample, ArrowTypeInfo::byte_array(data.len())) - } else if let Ok(arrow_array) = ArrayData::from_pyarrow(data.as_ref(py)) { + } else if let Ok(arrow_array) = ArrayData::from_pyarrow_bound(data.bind(py)) { let total_len = required_data_size(&arrow_array); let mut sample = allocate_sample(total_len)?; diff --git a/binaries/runtime/src/operator/shared_lib.rs b/binaries/runtime/src/operator/shared_lib.rs index 811c3cd07..984a760b9 100644 --- a/binaries/runtime/src/operator/shared_lib.rs +++ b/binaries/runtime/src/operator/shared_lib.rs @@ -124,7 +124,7 @@ impl<'lib> SharedLibraryOperator<'lib> { ..Default::default() }; - let arrow_array = match arrow::ffi::from_ffi(data_array, &schema) { + let arrow_array = match unsafe { arrow::ffi::from_ffi(data_array, &schema) } { Ok(a) => a, Err(err) => return DoraResult::from_error(err.to_string()), }; diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index a36375919..ee7f68309 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -15,8 +15,8 @@ use eyre::{eyre, Context, ContextCompat, Result}; use futures::{Stream, StreamExt}; use pyo3::{ prelude::{pyclass, pymethods}, - types::{PyDict, PyList, PyModule}, - PyAny, PyObject, PyResult, Python, + types::{PyAnyMethods, PyDict, PyList, PyModule, PyModuleMethods}, + Bound, PyAny, PyObject, PyResult, Python, }; use typed::{deserialize::StructDeserializer, TypeInfo, TypedValue}; @@ -57,7 +57,7 @@ impl Ros2Context { pub fn new(ros_paths: Option>) -> eyre::Result { Python::with_gil(|py| -> Result<()> { let warnings = py - .import("warnings") + .import_bound("warnings") .wrap_err("failed to import `warnings` module")?; warnings .call_method1("warn", ("dora-rs ROS2 Bridge is unstable and may change at any point without it being considered a breaking change",)) @@ -322,8 +322,8 @@ impl Ros2Publisher { /// :type data: pyarrow.Array /// :rtype: None /// - pub fn publish(&self, data: &PyAny) -> eyre::Result<()> { - let pyarrow = PyModule::import(data.py(), "pyarrow")?; + pub fn publish(&self, data: Bound<'_, PyAny>) -> eyre::Result<()> { + let pyarrow = PyModule::import_bound(data.py(), "pyarrow")?; let data = if data.is_instance_of::() { // convert to arrow struct scalar @@ -332,15 +332,15 @@ impl Ros2Publisher { data }; - let data = if data.is_instance(pyarrow.getattr("StructScalar")?)? { + let data = if data.is_instance(&pyarrow.getattr("StructScalar")?)? { // convert to arrow array - let list = PyList::new(data.py(), [data]); + let list = PyList::new_bound(data.py(), [data]); pyarrow.getattr("array")?.call1((list,))? } else { data }; - let value = arrow::array::ArrayData::from_pyarrow(data)?; + let value = arrow::array::ArrayData::from_pyarrow_bound(&data)?; //// add type info to ensure correct serialization (e.g. struct types //// and map types need to be serialized differently) let typed_value = TypedValue { @@ -431,7 +431,7 @@ impl Stream for Ros2SubscriptionStream { } } -pub fn create_dora_ros2_bridge_module(m: &PyModule) -> PyResult<()> { +pub fn create_dora_ros2_bridge_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs index c6875b7f1..2b841589e 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs @@ -37,10 +37,12 @@ mod tests { use arrow::pyarrow::ToPyArrow; use pyo3::types::IntoPyDict; + use pyo3::types::PyAnyMethods; use pyo3::types::PyDict; use pyo3::types::PyList; use pyo3::types::PyModule; use pyo3::types::PyTuple; + use pyo3::PyNativeType; use pyo3::Python; use serde::de::DeserializeSeed; use serde::Serialize; @@ -61,13 +63,13 @@ mod tests { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); //.join("test_utils.py"); // Adjust this path as needed // Add the Python module's directory to sys.path - py.run( + py.run_bound( "import sys; sys.path.append(str(path))", - Some([("path", path)].into_py_dict(py)), + Some(&[("path", path)].into_py_dict_bound(py)), None, )?; - let my_module = PyModule::import(py, "test_utils")?; + let my_module = PyModule::import_bound(py, "test_utils")?; let arrays: &PyList = my_module.getattr("TEST_ARRAYS")?.extract()?; for array_wrapper in arrays.iter() { @@ -77,7 +79,7 @@ mod tests { println!("Checking {}::{}", package_name, message_name); let in_pyarrow = arrays.get_item(2)?; - let array = arrow::array::ArrayData::from_pyarrow(in_pyarrow)?; + let array = arrow::array::ArrayData::from_pyarrow_bound(&in_pyarrow.as_borrowed())?; let type_info = TypeInfo { package_name: package_name.into(), message_name: message_name.clone().into(), @@ -99,17 +101,17 @@ mod tests { let out_pyarrow = out_value.to_pyarrow(py)?; - let test_utils = PyModule::import(py, "test_utils")?; - let context = PyDict::new(py); + let test_utils = PyModule::import_bound(py, "test_utils")?; + let context = PyDict::new_bound(py); context.set_item("test_utils", test_utils)?; context.set_item("in_pyarrow", in_pyarrow)?; context.set_item("out_pyarrow", out_pyarrow)?; let _ = py - .eval( + .eval_bound( "test_utils.is_subset(in_pyarrow, out_pyarrow)", - Some(context), + Some(&context), None, ) .context("could not check if it is a subset")?; diff --git a/tool_nodes/dora-record/Cargo.toml b/tool_nodes/dora-record/Cargo.toml index 9d7077169..246afbcf7 100644 --- a/tool_nodes/dora-record/Cargo.toml +++ b/tool_nodes/dora-record/Cargo.toml @@ -14,4 +14,4 @@ dora-node-api = { workspace = true, features = ["tracing"] } eyre = "0.6.8" chrono = "0.4.31" dora-tracing = { workspace = true } -parquet = { version = "48.0.0", features = ["async"] } +parquet = { version = "52", features = ["async"] } diff --git a/tool_nodes/dora-record/src/main.rs b/tool_nodes/dora-record/src/main.rs index fb10bf18e..3d9268fe7 100644 --- a/tool_nodes/dora-record/src/main.rs +++ b/tool_nodes/dora-record/src/main.rs @@ -62,7 +62,6 @@ async fn main() -> eyre::Result<()> { let mut writer = AsyncArrowWriter::try_new( file, schema.clone(), - 0, Some( WriterProperties::builder() .set_compression(parquet::basic::Compression::BROTLI(