chore(pyo3-arrow): Bump to pyo3 0.27 (#421)

* chore(pyo3-arrow): Bump to pyo3 0.27

* use git rev

* Improved dev docs for updating arro3

* Use released numpy

Author: kylebarron

Cargo.lock

@@ -36,7 +36,7 @@ numpy = "0.26"
 object_store = "0.12.4"
 parquet = "56.2"
 pyo3 = { version = "0.26", features = ["macros", "indexmap"] }
-pyo3-arrow = { path = "./pyo3-arrow" }
+pyo3-arrow = { git = "https://github.com/kylebarron/arro3", rev = "e2de4da2f732667dd796335ea7def8b111f79838" }
 pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
 pyo3-file = { git = "https://github.com/alex/pyo3-file", rev = "82b96dc188a48b88421f41b24cb13da114fd2fe8" }
 pyo3-object_store = "0.6"

Cargo.toml

@@ -54,3 +54,23 @@ RUSTUP_TOOLCHAIN=nightly \
     --target wasm32-unknown-emscripten \
     -i python3.11
 ```
+
+## Updating pyo3 version
+
+It takes a few steps to update for a new pyo3 version. We have to do these steps in a specific order because of intertangled dependencies.
+
+arro3 depends on pyo3-arrow and pyo3-object_store. pyo3-object_store itself depends on pyo3-arrow, so we have to wait for a pyo3-arrow release before we can update pyo3-object_store, and then we need a `pyo3-object_store` release before we can update arro3 itself.
+
+1. For simplicity, arro3 often uses the workspace dependency version of `pyo3-arrow`, with `pyo3-arrow = { path = "./pyo3-arrow" }` in `Cargo.toml`. Instead, we need to unlink this so that we can update arro3.
+
+    Change this to use the git dependency instead, pointing to the latest commit hash of `pyo3-arrow` that uses the existing pyo3 version. I.e. something like this
+
+    ```toml
+    pyo3-arrow = { git = "https://github.com/kylebarron/arro3", rev = "e2de4da2f732667dd796335ea7def8b111f79838" }
+    ```
+    Where this `rev` is the current `main`.
+
+2. Update pyo3 version in `pyo3-arrow/Cargo.toml`. Open a PR and release a new version of `pyo3-arrow`. Note that `pyo3-arrow` also depends on `numpy`, so we need to wait for a new release there, but that should be pretty fast.
+3. Ensure `pyo3-async-runtimes` and `pyo3-file` have releases for latest `pyo3`.
+4. Update `pyo3-object-store` to depend on the new `pyo3-arrow` version. Open a PR and release a new version of `pyo3-object-store`.
+5. Finally, update `arro3/Cargo.toml` to depend on the new `pyo3-arrow` and `pyo3-object-store` versions, and update pyo3 version. Open a PR and release a new version of `arro3`.

DEVELOP.md

@@ -27,10 +27,10 @@ arrow-schema = "56"
 arrow-select = "56"
 chrono = "0.4"
 chrono-tz = "0.10"
-pyo3 = { version = "0.26", features = ["chrono", "chrono-tz", "indexmap"] }
+pyo3 = { version = "0.27", features = ["chrono", "chrono-tz", "indexmap"] }
 half = "2"
 indexmap = "2"
-numpy = { version = "0.26", features = ["half"] }
+numpy = { version = "0.27", features = ["half"] }
 thiserror = "1"
 
 [lib]

pyo3-arrow/Cargo.lock

@@ -410,7 +410,7 @@ impl PyArray {
             return buf.try_into();
         }
 
-        let numpy_array: Bound<PyUntypedArray> = FromPyObject::extract_bound(&numpy_array)?;
+        let numpy_array: Bound<PyUntypedArray> = numpy_array.extract()?;
         let arrow_array = from_numpy(py, &numpy_array)?;
         Ok(Self::from_array_ref(arrow_array))
     }

pyo3-arrow/Cargo.toml

@@ -109,9 +109,11 @@ impl PyArrowBuffer {
     unsafe fn __releasebuffer__(&self, _view: *mut ffi::Py_buffer) {}
 }
 
-impl<'py> FromPyObject<'py> for PyArrowBuffer {
-    fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult<Self> {
-        let buffer = ob.extract::<AnyBufferProtocol>()?;
+impl<'py> FromPyObject<'_, 'py> for PyArrowBuffer {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result<Self, Self::Error> {
+        let buffer = obj.extract::<AnyBufferProtocol>()?;
         if !matches!(buffer, AnyBufferProtocol::UInt8(_)) {
             return Err(PyValueError::new_err("Expected u8 buffer protocol object"));
         }
@@ -136,27 +138,29 @@ pub enum AnyBufferProtocol {
     Float64(PyBuffer<f64>),
 }
 
-impl<'py> FromPyObject<'py> for AnyBufferProtocol {
-    fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult<Self> {
-        if let Ok(buf) = ob.extract() {
+impl<'py> FromPyObject<'_, 'py> for AnyBufferProtocol {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result<Self, Self::Error> {
+        if let Ok(buf) = obj.extract() {
             Ok(Self::UInt8(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::UInt16(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::UInt32(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::UInt64(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::Int8(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::Int16(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::Int32(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::Int64(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::Float32(buf))
-        } else if let Ok(buf) = ob.extract() {
+        } else if let Ok(buf) = obj.extract() {
             Ok(Self::Float64(buf))
         } else {
             Err(PyValueError::new_err("Not a buffer protocol object"))

pyo3-arrow/src/array.rs

@@ -17,9 +17,11 @@ use crate::PyField;
 
 struct PyTimeUnit(arrow_schema::TimeUnit);
 
-impl<'a> FromPyObject<'a> for PyTimeUnit {
-    fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult<Self> {
-        let s: String = ob.extract()?;
+impl<'a> FromPyObject<'_, 'a> for PyTimeUnit {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, 'a, PyAny>) -> Result<Self, Self::Error> {
+        let s: String = obj.extract()?;
         match s.to_lowercase().as_str() {
             "s" => Ok(Self(TimeUnit::Second)),
             "ms" => Ok(Self(TimeUnit::Millisecond)),

pyo3-arrow/src/buffer.rs

@@ -3,7 +3,7 @@
 use numpy::BorrowError;
 use pyo3::exceptions::{PyException, PyValueError};
 use pyo3::prelude::*;
-use pyo3::DowncastError;
+use pyo3::CastError;
 use thiserror::Error;
 
 /// The Error variants returned by this crate.
@@ -33,8 +33,8 @@ impl From<PyArrowError> for PyErr {
     }
 }
 
-impl<'a, 'py> From<DowncastError<'a, 'py>> for PyArrowError {
-    fn from(other: DowncastError<'a, 'py>) -> Self {
+impl<'a, 'py> From<CastError<'a, 'py>> for PyArrowError {
+    fn from(other: CastError<'a, 'py>) -> Self {
         Self::PyErr(PyValueError::new_err(format!(
             "Could not downcast: {}",
             other

pyo3-arrow/src/datatypes.rs

@@ -4,16 +4,18 @@ use crate::buffer::AnyBufferProtocol;
 use crate::ffi::from_python::utils::call_arrow_c_array;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
-use pyo3::{intern, PyAny, PyResult};
+use pyo3::{intern, PyAny};
 
-impl<'a> FromPyObject<'a> for PyArray {
-    fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult<Self> {
-        if ob.hasattr(intern!(ob.py(), "__arrow_c_array__"))? {
-            let (schema_capsule, array_capsule) = call_arrow_c_array(ob)?;
+impl<'py> FromPyObject<'_, 'py> for PyArray {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result<Self, Self::Error> {
+        if obj.hasattr(intern!(obj.py(), "__arrow_c_array__"))? {
+            let (schema_capsule, array_capsule) = call_arrow_c_array(&obj)?;
             Self::from_arrow_pycapsule(&schema_capsule, &array_capsule)
         } else {
             #[cfg(feature = "buffer_protocol")]
-            if let Ok(buf) = ob.extract::<AnyBufferProtocol>() {
+            if let Ok(buf) = obj.extract::<AnyBufferProtocol>() {
                 return Ok(buf.try_into()?);
             }