Add support for free-threaded CPython 3.13t

.github/workflows/build_wheels.yml

@@ -17,7 +17,7 @@ jobs:
         # cibuildwheel builds linux wheels inside a manylinux container
         # it also takes care of procuring the correct python version for us
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [39, 310, 311, 312, 313]
+        python-version: ["39", "310", "311", "312", "313", "313t"]
 
     steps:
       - uses: actions/checkout@v4
@@ -39,7 +39,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04-arm]
-        python-version: [39, 310, 311, 312, 313]
+        python-version: ["39", "310", "311", "312", "313", "313t"]
 
     steps:
       - uses: actions/checkout@v4

Cargo.toml

@@ -15,7 +15,7 @@ python = [
 ]
 
 [dependencies]
-pyo3 = { version = "0.22.2", default-features = false, features = [
+pyo3 = { version = "0.24.2", default-features = false, features = [
     "extension-module",
     "macros",
 ], optional = true }

pyproject.toml

@@ -20,6 +20,7 @@ requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"]
 
 [tool.cibuildwheel]
 build-frontend = "build"
+enable = ["cpython-freethreading"]
 build-verbosity = 1
 
 linux.before-all = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal"

src/py.rs

@@ -55,14 +55,13 @@ impl CoreBPE {
         py: Python,
         text: &str,
         allowed_special: HashSet<PyBackedStr>,
-    ) -> Py<PyAny> {
+    ) -> TiktokenBuffer {
         let tokens = py.allow_threads(|| {
             let allowed_special: HashSet<&str> =
                 allowed_special.iter().map(|s| s.as_ref()).collect();
             self.encode(text, &allowed_special).0
         });
-        let buffer = TiktokenBuffer { tokens };
-        buffer.into_py(py)
+        TiktokenBuffer { tokens }
     }
 
     fn _encode_bytes(&self, py: Python, bytes: &[u8]) -> Vec<Rank> {
@@ -115,19 +114,20 @@ impl CoreBPE {
         py: Python,
         text: &str,
         allowed_special: HashSet<PyBackedStr>,
-    ) -> Py<PyTuple> {
+    ) -> PyResult<Py<PyTuple>> {
         let (tokens, completions) = py.allow_threads(|| {
             let allowed_special: HashSet<&str> =
                 allowed_special.iter().map(|s| s.as_ref()).collect();
             self._encode_unstable_native(text, &allowed_special)
         });
-        let py_completions = PyList::new_bound(
+        let py_completions: Bound<'_, PyList> = PyList::new(
             py,
             completions
                 .iter()
-                .map(|seq| PyList::new_bound(py, &seq[..])),
-        );
-        (tokens, py_completions).into_py(py)
+                .map(|seq| PyList::new(py, &seq[..]))
+                .collect::<PyResult<Vec<_>>>(),
+        )?;
+        Ok((tokens, py_completions).into_pyobject(py)?.unbind().into())
     }
 
     fn encode_single_token(&self, piece: &[u8]) -> PyResult<Rank> {
@@ -156,17 +156,17 @@ impl CoreBPE {
     #[pyo3(name = "decode_bytes")]
     fn py_decode_bytes(&self, py: Python, tokens: Vec<Rank>) -> Result<Py<PyBytes>, PyErr> {
         match py.allow_threads(|| self.decode_bytes(&tokens)) {
-            Ok(bytes) => Ok(PyBytes::new_bound(py, &bytes).into()),
+            Ok(bytes) => Ok(PyBytes::new(py, &bytes).into()),
             Err(e) => Err(pyo3::exceptions::PyKeyError::new_err(format!("{}", e))),
         }
     }
 
     fn decode_single_token_bytes(&self, py: Python, token: Rank) -> PyResult<Py<PyBytes>> {
         if let Some(bytes) = self.decoder.get(&token) {
-            return Ok(PyBytes::new_bound(py, bytes).into());
+            return Ok(PyBytes::new(py, bytes).into());
         }
         if let Some(bytes) = self.special_tokens_decoder.get(&token) {
-            return Ok(PyBytes::new_bound(py, bytes).into());
+            return Ok(PyBytes::new(py, bytes).into());
         }
         Err(PyErr::new::<exceptions::PyKeyError, _>(token.to_string()))
     }
@@ -178,7 +178,7 @@ impl CoreBPE {
     fn token_byte_values(&self, py: Python) -> Vec<Py<PyBytes>> {
         self.sorted_token_bytes
             .iter()
-            .map(|x| PyBytes::new_bound(py, x).into())
+            .map(|x| PyBytes::new(py, x).into())
             .collect()
     }
 }
@@ -240,7 +240,7 @@ impl TiktokenBuffer {
     }
 }
 
-#[pymodule]
+#[pymodule(gil_used = false)]
 fn _tiktoken(_py: Python, m: &Bound<PyModule>) -> PyResult<()> {
     m.add_class::<CoreBPE>()?;
     Ok(())