Python3.11 support added. Dependencies changes. Github Actions updates

Gal Ben David · Gal Ben David · commit 1dcd8c43c049 · 2022-11-24T12:45:26.000+02:00
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -32,6 +32,7 @@ jobs:
           - '3.8'
           - '3.9'
           - '3.10'
+          - '3.11'
         os:
           - ubuntu-latest
           - macos-latest
@@ -40,11 +41,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Poetry
-        uses: abatilo/actions-poetry@v2.1.3
+        uses: abatilo/actions-poetry@v2
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -14,6 +14,7 @@ jobs:
           - '3.8'
           - '3.9'
           - '3.10'
+          - '3.11'
         os:
           - ubuntu-latest
           - macos-latest
@@ -22,7 +23,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Rust
@@ -32,7 +33,7 @@ jobs:
           toolchain: stable
           override: true
       - name: Publish Package
-        uses: messense/maturin-action@v1
+        uses: PyO3/maturin-action@v1
         with:
           command: publish
           args: --username=__token__ ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.7' && '' || '--no-sdist' }} --interpreter=python${{ !startsWith(matrix.os, 'windows') && matrix.python-version || '' }}
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pydomainextractor"
-version = "0.11.1"
+version = "0.12.0"
 authors = ["Gal Ben David <gal@intsights.com>"]
 edition = "2021"
 description = "A blazingly fast domain extraction library written in Rust"
@@ -28,6 +28,7 @@ classifier = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Rust",
 ]
 
@@ -36,12 +37,14 @@ name = "pydomainextractor"
 crate-type = ["cdylib"]
 
 [dependencies]
-ahash = "0.7"
-idna = "0.2"
+ahash = "0.8"
+idna = "0.3"
 memchr = "2"
+arraystring = "0.3.0"
+typenum = "1"
 
 [dependencies.pyo3]
-version = "0.16.5"
+version = "0.17.3"
 features = ["extension-module"]
 
 [profile.release]
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,5 +1,4 @@
 include README.md
 include images/logo.png
 graft tests
-recursive-include src *.h *.cpp
 recursive-include pydomainextractor *.py *.pyi
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["maturin>=0.12,<0.13"]
+requires = ["maturin>=0.14,<0.15"]
 build-backend = "maturin"
 
 [tool.maturin]
@@ -13,7 +13,7 @@ sdist-include = [
 
 [tool.poetry]
 name = "pydomainextractor"
-version = "0.11.1"
+version = "0.12.0"
 authors = ["Gal Ben David <gal@intsights.com>"]
 description = "A blazingly fast domain extraction library written in Rust"
 readme = "README.md"
@@ -38,6 +38,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Rust",
 ]
 
diff --git a/src/lib.rs b/src/lib.rs
@@ -1,9 +1,10 @@
 use ahash::{AHashMap, AHashSet};
 use pyo3::exceptions::PyValueError;
+use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::types::PyString;
-use std::ptr;
-use pyo3::once_cell::GILOnceCell;
+
+type DomainString = arraystring::ArrayString<typenum::U255>;
 
 #[derive(Default)]
 struct Suffix {
@@ -12,13 +13,7 @@ struct Suffix {
     sub_blacklist: AHashSet<String>,
 }
 
-static mut EMPTY_STRING: *mut pyo3::ffi::PyObject = ptr::null_mut();
-static mut SUFFIX_STRING: *mut pyo3::ffi::PyObject = ptr::null_mut();
-static mut DOMAIN_STRING: *mut pyo3::ffi::PyObject = ptr::null_mut();
-static mut SUBDOMAIN_STRING: *mut pyo3::ffi::PyObject = ptr::null_mut();
 static PUBLIC_SUFFIX_LIST_DATA: &str = include_str!("public_suffix_list.dat");
-static mut TEMP_DOMAIN_STRING: GILOnceCell<String> = GILOnceCell::new();
-
 
 #[pyclass]
 struct DomainExtractor {
@@ -31,21 +26,21 @@ impl DomainExtractor {
     #[new]
     fn new(
         suffix_list: Option<&str>,
-    ) -> PyResult<Self> {
+    ) -> Self {
         let (suffixes, tld_list) = if let Some(suffix_list) = suffix_list {
             parse_suffix_list(suffix_list)
         } else {
             parse_suffix_list(PUBLIC_SUFFIX_LIST_DATA)
         };
 
-        Ok(DomainExtractor { suffixes, tld_list })
+        DomainExtractor { suffixes, tld_list }
     }
 
     fn parse_domain_parts<'a>(
         &self,
         domain: &'a str,
-    ) -> Result<(&'a str, &'a str, &'a str), PyErr> {
-        let mut suffix_part: &str = "";
+    ) -> PyResult<(&'a str, &'a str, &'a str)> {
+        let mut suffix_part = "";
         let mut current_suffixes = &self.suffixes;
         let mut last_dot_index = domain.len();
         let mut in_wildcard_tld = false;
@@ -126,29 +121,26 @@ impl DomainExtractor {
 
     fn extract(
         &self,
+        py: Python,
         domain: &PyString,
-    ) -> PyResult<*mut pyo3::ffi::PyObject> {
+    ) -> PyResult<PyObject> {
         if domain.len().unwrap() > 255 {
             return Err(PyValueError::new_err("Invalid domain detected"));
         }
 
-        let domain_string = unsafe {
-            let temp_domain_string = TEMP_DOMAIN_STRING.get_mut().unwrap_unchecked();
-            temp_domain_string.clear();
-            temp_domain_string.push_str(domain.to_str().unwrap());
-            temp_domain_string.make_ascii_lowercase();
-
-            temp_domain_string
+        let mut domain_string = unsafe {
+            DomainString::from_str_unchecked(domain.to_string_lossy().as_ref())
         };
+        domain_string.make_ascii_lowercase();
 
-        let (suffix_part, domain_part, subdomain_part) = self.parse_domain_parts(domain_string)?;
+        let (suffix_part, domain_part, subdomain_part) = self.parse_domain_parts(domain_string.as_str())?;
 
         unsafe {
             let dict = pyo3::ffi::PyDict_New();
             for (fraction_key, fraction) in [
-                (SUFFIX_STRING, suffix_part),
-                (DOMAIN_STRING, domain_part),
-                (SUBDOMAIN_STRING, subdomain_part),
+                (intern!(py, "suffix").into_ptr(), suffix_part),
+                (intern!(py, "domain").into_ptr(), domain_part),
+                (intern!(py, "subdomain").into_ptr(), subdomain_part),
             ] {
                 if !fraction.is_empty() {
                     let substr = pyo3::ffi::PyUnicode_FromStringAndSize(
@@ -166,12 +158,12 @@ impl DomainExtractor {
                     pyo3::ffi::PyDict_SetItem(
                         dict,
                         fraction_key,
-                        EMPTY_STRING,
+                        intern!(py, "").into_ptr(),
                     );
                 }
             }
 
-            Ok(dict)
+            Ok(pyo3::PyObject::from_owned_ptr(py, dict))
         }
     }
 
@@ -184,12 +176,8 @@ impl DomainExtractor {
             return false;
         }
 
-        let domain_string = unsafe {
-            let temp_domain_string = TEMP_DOMAIN_STRING.get_mut().unwrap_unchecked();
-            temp_domain_string.clear();
-            temp_domain_string.push_str(domain.to_str().unwrap());
-
-            temp_domain_string
+        let mut domain_string = unsafe {
+            DomainString::from_str_unchecked(domain.to_string_lossy().as_ref())
         };
 
         for fraction in domain_string.split('.') {
@@ -208,15 +196,15 @@ impl DomainExtractor {
         }
 
         domain_string.make_ascii_lowercase();
-        if let Ok((suffix_part, domain_part, _subdomain_part)) = self.parse_domain_parts(domain_string) {
+        if let Ok((suffix_part, domain_part, _subdomain_part)) = self.parse_domain_parts(domain_string.as_str()) {
             if suffix_part.is_empty() || domain_part.is_empty() {
                 return false;
             }
 
-            if idna::domain_to_ascii(domain_string).is_err() {
+            if idna::domain_to_ascii(domain_string.as_str()).is_err() {
                 return false;
             }
-            if idna::domain_to_unicode(domain_string).1.is_err() {
+            if idna::domain_to_unicode(domain_string.as_str()).1.is_err() {
                 return false;
             }
 
@@ -234,8 +222,9 @@ impl DomainExtractor {
 
     fn extract_from_url(
         &self,
+        py: Python,
         url: &PyString,
-    ) -> PyResult<*mut pyo3::ffi::PyObject> {
+    ) -> PyResult<PyObject> {
         let mut url_str = url.to_str().unwrap();
 
         match memchr::memmem::find(url_str.as_bytes(), b"//") {
@@ -265,23 +254,19 @@ impl DomainExtractor {
             );
         }
 
-        let domain_string = unsafe {
-            let temp_domain_string = TEMP_DOMAIN_STRING.get_mut().unwrap_unchecked();
-            temp_domain_string.clear();
-            temp_domain_string.push_str(url_str);
-            temp_domain_string.make_ascii_lowercase();
-
-            temp_domain_string
+        let mut domain_string = unsafe {
+            DomainString::from_str_unchecked(url_str)
         };
+        domain_string.make_ascii_lowercase();
 
         let (suffix_part, domain_part, subdomain_part) = self.parse_domain_parts(domain_string.as_str())?;
 
         unsafe {
             let dict = pyo3::ffi::PyDict_New();
             for (fraction_key, fraction) in [
-                (SUFFIX_STRING, suffix_part),
-                (DOMAIN_STRING, domain_part),
-                (SUBDOMAIN_STRING, subdomain_part),
+                (intern!(py, "suffix").into_ptr(), suffix_part),
+                (intern!(py, "domain").into_ptr(), domain_part),
+                (intern!(py, "subdomain").into_ptr(), subdomain_part),
             ] {
                 if !fraction.is_empty() {
                     let substr = pyo3::ffi::PyUnicode_FromStringAndSize(
@@ -299,12 +284,12 @@ impl DomainExtractor {
                     pyo3::ffi::PyDict_SetItem(
                         dict,
                         fraction_key,
-                        EMPTY_STRING,
+                        intern!(py, "").into_ptr(),
                     );
                 }
             }
 
-            Ok(dict)
+            Ok(pyo3::PyObject::from_owned_ptr(py, dict))
         }
     }
 }
@@ -363,26 +348,9 @@ fn parse_suffix_list(
 
 #[pymodule]
 fn pydomainextractor(
-    py: Python,
+    _py: Python,
     m: &PyModule,
 ) -> PyResult<()> {
-    unsafe {
-        EMPTY_STRING = pyo3::ffi::PyUnicode_New(0, 127);
-        SUFFIX_STRING = pyo3::ffi::PyUnicode_FromStringAndSize(
-            "suffix".as_ptr() as *const i8,
-            "suffix".len() as isize,
-        );
-        DOMAIN_STRING = pyo3::ffi::PyUnicode_FromStringAndSize(
-            "domain".as_ptr() as *const i8,
-            "domain".len() as isize,
-        );
-        SUBDOMAIN_STRING = pyo3::ffi::PyUnicode_FromStringAndSize(
-            "subdomain".as_ptr() as *const i8,
-            "subdomain".len() as isize,
-        );
-        TEMP_DOMAIN_STRING.set(py, String::with_capacity(1024)).unwrap();
-    }
-
     m.add_class::<DomainExtractor>()?;
     Ok(())
 }
diff --git a/src/public_suffix_list.dat b/src/public_suffix_list.dat
