From b82622d765c5d84119fb6540cede8ed493cb9612 Mon Sep 17 00:00:00 2001 From: Tesla <1394466835@qq.com> Date: Fri, 29 Nov 2024 15:38:17 +0800 Subject: [PATCH 1/4] feat: upgrade pyo03 to 0.23 --- bindings/python/CONTRIBUTING.md | 2 +- bindings/python/Cargo.toml | 6 ++---- bindings/python/src/file.rs | 12 +++++++---- bindings/python/src/lib.rs | 36 ++++++++++++++------------------- bindings/python/src/lister.rs | 7 ++++++- bindings/python/src/operator.rs | 32 ++++++++++++++++++++--------- bindings/python/src/utils.rs | 4 ++-- 7 files changed, 56 insertions(+), 43 deletions(-) diff --git a/bindings/python/CONTRIBUTING.md b/bindings/python/CONTRIBUTING.md index 0cce0a8f0f63..e8c8dec0f5f6 100644 --- a/bindings/python/CONTRIBUTING.md +++ b/bindings/python/CONTRIBUTING.md @@ -46,7 +46,7 @@ After `venv` has been prepared, you can activate it by `source venv/bin/activate To simplify our work, we will utilize the tool [`maturin`](https://github.com/PyO3/maturin). Kindly install it beforehand. ```shell -pip install maturin[patchelf] +pip install 'maturin[patchelf]' ``` ## Build diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index bee110942267..2625c1559089 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -163,10 +163,8 @@ futures = "0.3.28" opendal = { version = ">=0", path = "../../core", features = [ "layers-blocking", ] } -pyo3 = "0.22.5" -pyo3-async-runtimes = { version = "0.22.0", features = [ - "tokio-runtime", -] } +pyo3 = "0.23.2" +pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] } tokio = "1" [target.'cfg(unix)'.dependencies.opendal] diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs index 374db32ded04..4ed2a369c07e 100644 --- a/bindings/python/src/file.rs +++ b/bindings/python/src/file.rs @@ -463,12 +463,14 @@ impl AsyncFile { } }; - let ret = reader + let pos = reader .seek(whence) .await .map_err(|err| PyIOError::new_err(err.to_string()))?; - Ok(Python::with_gil(|py| ret.into_py(py))) + Ok(pos) }) + .and_then(|pos| pos.into_pyobject(py).map_err(Into::into)) + .map(|pyobj| pyobj.into_any()) } /// Return the current stream position. @@ -495,8 +497,10 @@ impl AsyncFile { .stream_position() .await .map_err(|err| PyIOError::new_err(err.to_string()))?; - Ok(Python::with_gil(|py| pos.into_py(py))) + Ok(pos) }) + .and_then(|pos| pos.into_pyobject(py).map_err(Into::into)) + .map(|pyobj| pyobj.into_any()) } fn close<'p>(&'p mut self, py: Python<'p>) -> PyResult> { @@ -514,7 +518,7 @@ impl AsyncFile { } fn __aenter__<'a>(slf: PyRef<'a, Self>, py: Python<'a>) -> PyResult> { - let slf = slf.into_py(py); + let slf = slf.into_pyobject(py)?.into_any().unbind(); future_into_py(py, async move { Ok(slf) }) } diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index b499dc42aef2..f05b4b4de85b 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -87,35 +87,29 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; // Layer module - let layers_module = PyModule::new_bound(py, "layers")?; + let layers_module = PyModule::new(py, "layers")?; layers_module.add_class::()?; layers_module.add_class::()?; layers_module.add_class::()?; m.add_submodule(&layers_module)?; - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("opendal.layers", layers_module)?; - let exception_module = PyModule::new_bound(py, "exceptions")?; - exception_module.add("Error", py.get_type_bound::())?; - exception_module.add("Unexpected", py.get_type_bound::())?; - exception_module.add("Unsupported", py.get_type_bound::())?; - exception_module.add("ConfigInvalid", py.get_type_bound::())?; - exception_module.add("NotFound", py.get_type_bound::())?; - exception_module.add( - "PermissionDenied", - py.get_type_bound::(), - )?; - exception_module.add("IsADirectory", py.get_type_bound::())?; - exception_module.add("NotADirectory", py.get_type_bound::())?; - exception_module.add("AlreadyExists", py.get_type_bound::())?; - exception_module.add("IsSameFile", py.get_type_bound::())?; - exception_module.add( - "ConditionNotMatch", - py.get_type_bound::(), - )?; + let exception_module = PyModule::new(py, "exceptions")?; + exception_module.add("Error", py.get_type::())?; + exception_module.add("Unexpected", py.get_type::())?; + exception_module.add("Unsupported", py.get_type::())?; + exception_module.add("ConfigInvalid", py.get_type::())?; + exception_module.add("NotFound", py.get_type::())?; + exception_module.add("PermissionDenied", py.get_type::())?; + exception_module.add("IsADirectory", py.get_type::())?; + exception_module.add("NotADirectory", py.get_type::())?; + exception_module.add("AlreadyExists", py.get_type::())?; + exception_module.add("IsSameFile", py.get_type::())?; + exception_module.add("ConditionNotMatch", py.get_type::())?; m.add_submodule(&exception_module)?; - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("opendal.exceptions", exception_module)?; Ok(()) diff --git a/bindings/python/src/lister.rs b/bindings/python/src/lister.rs index e2074016fae3..49f6da33b275 100644 --- a/bindings/python/src/lister.rs +++ b/bindings/python/src/lister.rs @@ -42,7 +42,12 @@ impl BlockingLister { } fn __next__(mut slf: PyRefMut<'_, Self>) -> PyResult> { match slf.0.next() { - Some(Ok(entry)) => Ok(Some(Entry::new(entry).into_py(slf.py()))), + Some(Ok(entry)) => Ok(Some( + Entry::new(entry) + .into_pyobject(slf.py())? + .into_any() + .unbind(), + )), Some(Err(err)) => { let pyerr = format_pyerr(err); Err(pyerr) diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs index c412cdc919c5..0185bc494ce6 100644 --- a/bindings/python/src/operator.rs +++ b/bindings/python/src/operator.rs @@ -228,10 +228,10 @@ impl Operator { } fn __getnewargs_ex__(&self, py: Python) -> PyResult { - let args = vec![self.__scheme.to_string().to_object(py)]; - let args = PyTuple::new_bound(py, args); - let kwargs = self.__map.clone().into_py(py); - Ok(PyTuple::new_bound(py, [args.to_object(py), kwargs.to_object(py)]).to_object(py)) + let args = vec![self.__scheme.to_string()]; + let args = PyTuple::new(py, args)?.into_any().unbind(); + let kwargs = self.__map.clone().into_pyobject(py)?.into_any().unbind(); + Ok(PyTuple::new(py, [args, kwargs])?.into_any().unbind()) } } @@ -434,7 +434,14 @@ impl AsyncOperator { let this = self.core.clone(); future_into_py(py, async move { let lister = this.lister(&path).await.map_err(format_pyerr)?; - let pylister: PyObject = Python::with_gil(|py| AsyncLister::new(lister).into_py(py)); + + let pylister = Python::with_gil(|py| { + AsyncLister::new(lister) + .into_pyobject(py) + .map_err(|err| PyErr::new::(err.to_string())) + .map(|py_obj| py_obj.into_any().unbind()) + })?; + Ok(pylister) }) } @@ -448,7 +455,12 @@ impl AsyncOperator { .recursive(true) .await .map_err(format_pyerr)?; - let pylister: PyObject = Python::with_gil(|py| AsyncLister::new(lister).into_py(py)); + let pylister: PyObject = Python::with_gil(|py| { + AsyncLister::new(lister) + .into_pyobject(py) + .map_err(|err| PyErr::new::(err.to_string())) + .map(|py_obj| py_obj.into_any().unbind()) + })?; Ok(pylister) }) } @@ -543,10 +555,10 @@ impl AsyncOperator { } fn __getnewargs_ex__(&self, py: Python) -> PyResult { - let args = vec![self.__scheme.to_string().to_object(py)]; - let args = PyTuple::new_bound(py, args); - let kwargs = self.__map.clone().into_py(py); - Ok(PyTuple::new_bound(py, [args.to_object(py), kwargs.to_object(py)]).to_object(py)) + let args = vec![self.__scheme.to_string()]; + let args = PyTuple::new(py, args)?.into_any().unbind(); + let kwargs = self.__map.clone().into_pyobject(py)?.into_any().unbind(); + Ok(PyTuple::new(py, [args, kwargs])?.into_any().unbind()) } } diff --git a/bindings/python/src/utils.rs b/bindings/python/src/utils.rs index eb58e85ef8e2..b204662239c8 100644 --- a/bindings/python/src/utils.rs +++ b/bindings/python/src/utils.rs @@ -33,14 +33,14 @@ impl Buffer { /// Consume self to build a bytes pub fn into_bytes(self, py: Python) -> PyResult> { - let buffer = self.into_py(py); + let buffer = self.into_pyobject(py)?.into_any().unbind(); unsafe { PyObject::from_owned_ptr_or_err(py, ffi::PyBytes_FromObject(buffer.as_ptr())) } } /// Consume self to build a bytes pub fn into_bytes_ref(self, py: Python) -> PyResult> { - let buffer = self.into_py(py); + let buffer = self.into_pyobject(py)?.into_any().unbind(); let view = unsafe { Bound::from_owned_ptr_or_err(py, ffi::PyBytes_FromObject(buffer.as_ptr()))? }; From 8fa405c70be867857e416225792a2c7474239394 Mon Sep 17 00:00:00 2001 From: ultraman <1394466835@qq.com> Date: Sat, 30 Nov 2024 07:57:42 +0800 Subject: [PATCH 2/4] update --- bindings/python/src/lister.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bindings/python/src/lister.rs b/bindings/python/src/lister.rs index 49f6da33b275..3103533f0427 100644 --- a/bindings/python/src/lister.rs +++ b/bindings/python/src/lister.rs @@ -77,10 +77,17 @@ impl AsyncLister { let mut lister = lister.lock().await; let entry = lister.try_next().await.map_err(format_pyerr)?; match entry { - Some(entry) => Ok(Python::with_gil(|py| Entry::new(entry).into_py(py))), + Some(entry) => Python::with_gil(|py| { + let py_obj = Entry::new(entry).into_pyobject(py)?.into_any().unbind(); + Ok(Some(py_obj)) + }), None => Err(PyStopAsyncIteration::new_err("stream exhausted")), } - })?; - Ok(Some(fut.into())) + }); + + match fut { + Ok(fut) => Ok(Some(fut.into())), + Err(e) => Err(e), + } } } From 4f368a0df517c8b04437359965d55b1f59d90212 Mon Sep 17 00:00:00 2001 From: Tesla <1394466835@qq.com> Date: Mon, 2 Dec 2024 09:32:20 +0800 Subject: [PATCH 3/4] chore: usage update --- bindings/python/src/file.rs | 3 ++- bindings/python/src/lister.rs | 4 ++-- bindings/python/src/operator.rs | 7 ++++--- bindings/python/src/utils.rs | 5 +++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs index 4ed2a369c07e..8cfcbf51ea5b 100644 --- a/bindings/python/src/file.rs +++ b/bindings/python/src/file.rs @@ -32,6 +32,7 @@ use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyIOError; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use pyo3::IntoPyObjectExt; use pyo3_async_runtimes::tokio::future_into_py; use tokio::sync::Mutex; @@ -518,7 +519,7 @@ impl AsyncFile { } fn __aenter__<'a>(slf: PyRef<'a, Self>, py: Python<'a>) -> PyResult> { - let slf = slf.into_pyobject(py)?.into_any().unbind(); + let slf = slf.into_py_any(py)?; future_into_py(py, async move { Ok(slf) }) } diff --git a/bindings/python/src/lister.rs b/bindings/python/src/lister.rs index 3103533f0427..55e08bef6687 100644 --- a/bindings/python/src/lister.rs +++ b/bindings/python/src/lister.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use futures::TryStreamExt; use pyo3::exceptions::PyStopAsyncIteration; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use pyo3_async_runtimes::tokio::future_into_py; use tokio::sync::Mutex; @@ -78,7 +78,7 @@ impl AsyncLister { let entry = lister.try_next().await.map_err(format_pyerr)?; match entry { Some(entry) => Python::with_gil(|py| { - let py_obj = Entry::new(entry).into_pyobject(py)?.into_any().unbind(); + let py_obj = Entry::new(entry).into_py_any(py)?; Ok(Some(py_obj)) }), None => Err(PyStopAsyncIteration::new_err("stream exhausted")), diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs index 0185bc494ce6..995f3b7441f2 100644 --- a/bindings/python/src/operator.rs +++ b/bindings/python/src/operator.rs @@ -23,6 +23,7 @@ use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::types::PyDict; use pyo3::types::PyTuple; +use pyo3::IntoPyObjectExt; use pyo3_async_runtimes::tokio::future_into_py; use crate::*; @@ -230,7 +231,7 @@ impl Operator { fn __getnewargs_ex__(&self, py: Python) -> PyResult { let args = vec![self.__scheme.to_string()]; let args = PyTuple::new(py, args)?.into_any().unbind(); - let kwargs = self.__map.clone().into_pyobject(py)?.into_any().unbind(); + let kwargs = self.__map.clone().into_py_any(py)?; Ok(PyTuple::new(py, [args, kwargs])?.into_any().unbind()) } } @@ -556,8 +557,8 @@ impl AsyncOperator { fn __getnewargs_ex__(&self, py: Python) -> PyResult { let args = vec![self.__scheme.to_string()]; - let args = PyTuple::new(py, args)?.into_any().unbind(); - let kwargs = self.__map.clone().into_pyobject(py)?.into_any().unbind(); + let args = PyTuple::new(py, args)?.into_py_any(py)?; + let kwargs = self.__map.clone().into_py_any(py)?; Ok(PyTuple::new(py, [args, kwargs])?.into_any().unbind()) } } diff --git a/bindings/python/src/utils.rs b/bindings/python/src/utils.rs index b204662239c8..9970b3304e4f 100644 --- a/bindings/python/src/utils.rs +++ b/bindings/python/src/utils.rs @@ -19,6 +19,7 @@ use std::os::raw::c_int; use pyo3::ffi; use pyo3::prelude::*; +use pyo3::IntoPyObjectExt; /// A bytes-like object that implements buffer protocol. #[pyclass(module = "opendal")] @@ -33,14 +34,14 @@ impl Buffer { /// Consume self to build a bytes pub fn into_bytes(self, py: Python) -> PyResult> { - let buffer = self.into_pyobject(py)?.into_any().unbind(); + let buffer = self.into_py_any(py)?; unsafe { PyObject::from_owned_ptr_or_err(py, ffi::PyBytes_FromObject(buffer.as_ptr())) } } /// Consume self to build a bytes pub fn into_bytes_ref(self, py: Python) -> PyResult> { - let buffer = self.into_pyobject(py)?.into_any().unbind(); + let buffer = self.into_py_any(py)?; let view = unsafe { Bound::from_owned_ptr_or_err(py, ffi::PyBytes_FromObject(buffer.as_ptr()))? }; From 07578049269e6f7b71c66a01783377a9e4f32761 Mon Sep 17 00:00:00 2001 From: Tesla <1394466835@qq.com> Date: Tue, 3 Dec 2024 09:11:50 +0800 Subject: [PATCH 4/4] apply patch --- bindings/python/src/file.rs | 6 ++---- bindings/python/src/lister.rs | 7 +------ bindings/python/src/operator.rs | 22 ++++++---------------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs index 8cfcbf51ea5b..2c2172324bb7 100644 --- a/bindings/python/src/file.rs +++ b/bindings/python/src/file.rs @@ -470,8 +470,7 @@ impl AsyncFile { .map_err(|err| PyIOError::new_err(err.to_string()))?; Ok(pos) }) - .and_then(|pos| pos.into_pyobject(py).map_err(Into::into)) - .map(|pyobj| pyobj.into_any()) + .and_then(|pos| pos.into_bound_py_any(py)) } /// Return the current stream position. @@ -500,8 +499,7 @@ impl AsyncFile { .map_err(|err| PyIOError::new_err(err.to_string()))?; Ok(pos) }) - .and_then(|pos| pos.into_pyobject(py).map_err(Into::into)) - .map(|pyobj| pyobj.into_any()) + .and_then(|pos| pos.into_bound_py_any(py)) } fn close<'p>(&'p mut self, py: Python<'p>) -> PyResult> { diff --git a/bindings/python/src/lister.rs b/bindings/python/src/lister.rs index 55e08bef6687..6019689de282 100644 --- a/bindings/python/src/lister.rs +++ b/bindings/python/src/lister.rs @@ -42,12 +42,7 @@ impl BlockingLister { } fn __next__(mut slf: PyRefMut<'_, Self>) -> PyResult> { match slf.0.next() { - Some(Ok(entry)) => Ok(Some( - Entry::new(entry) - .into_pyobject(slf.py())? - .into_any() - .unbind(), - )), + Some(Ok(entry)) => Ok(Some(Entry::new(entry).into_py_any(slf.py())?)), Some(Err(err)) => { let pyerr = format_pyerr(err); Err(pyerr) diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs index 995f3b7441f2..76cdb09f8653 100644 --- a/bindings/python/src/operator.rs +++ b/bindings/python/src/operator.rs @@ -230,9 +230,9 @@ impl Operator { fn __getnewargs_ex__(&self, py: Python) -> PyResult { let args = vec![self.__scheme.to_string()]; - let args = PyTuple::new(py, args)?.into_any().unbind(); + let args = PyTuple::new(py, args)?.into_py_any(py)?; let kwargs = self.__map.clone().into_py_any(py)?; - Ok(PyTuple::new(py, [args, kwargs])?.into_any().unbind()) + Ok(PyTuple::new(py, [args, kwargs])?.into_py_any(py)?) } } @@ -435,13 +435,7 @@ impl AsyncOperator { let this = self.core.clone(); future_into_py(py, async move { let lister = this.lister(&path).await.map_err(format_pyerr)?; - - let pylister = Python::with_gil(|py| { - AsyncLister::new(lister) - .into_pyobject(py) - .map_err(|err| PyErr::new::(err.to_string())) - .map(|py_obj| py_obj.into_any().unbind()) - })?; + let pylister = Python::with_gil(|py| AsyncLister::new(lister).into_py_any(py))?; Ok(pylister) }) @@ -456,12 +450,8 @@ impl AsyncOperator { .recursive(true) .await .map_err(format_pyerr)?; - let pylister: PyObject = Python::with_gil(|py| { - AsyncLister::new(lister) - .into_pyobject(py) - .map_err(|err| PyErr::new::(err.to_string())) - .map(|py_obj| py_obj.into_any().unbind()) - })?; + let pylister: PyObject = + Python::with_gil(|py| AsyncLister::new(lister).into_py_any(py))?; Ok(pylister) }) } @@ -559,7 +549,7 @@ impl AsyncOperator { let args = vec![self.__scheme.to_string()]; let args = PyTuple::new(py, args)?.into_py_any(py)?; let kwargs = self.__map.clone().into_py_any(py)?; - Ok(PyTuple::new(py, [args, kwargs])?.into_any().unbind()) + Ok(PyTuple::new(py, [args, kwargs])?.into_py_any(py)?) } }