add rust toolchain (#662)

* add basic rust integration with setuptools-rust

* add rust compiler to build image

* add rust tests to pipeline

* add basic crosscompiling pipeline

* configure cibuildwheel

* use generic job

* run build in parallel

* use only archs

* remove i686 arch

* add upload artifacts

* fix containerbuilds

* build wheels

* build wheels

* build wheels

* build wheels

* build wheels

* add wheelbuild to publish ci

* add cibuildwheel to dev builds

.github/workflows/publish-latest-dev-release-to-pypi.yml

@@ -17,14 +17,24 @@ jobs:
       - name: Initialize Python
         uses: actions/setup-python@v1
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
-      - name: Install dependencies
-        run: |
-          pip install --upgrade pip wheel
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse --archs x86_64,aarch64
 
       - name: Build binary wheel and a source tarball
-        run: pip wheel --no-deps --wheel-dir ./dist .
+        run: pipx run build --sdist
+
+      - name: copy artifacts to dist folder
+        run: cp ./wheelhouse/* ./dist/
 
       - uses: marvinpinto/action-automatic-releases@latest
         with:

.github/workflows/publish-release-to-pypi.yml

@@ -14,14 +14,24 @@ jobs:
       - name: Initialize Python
         uses: actions/setup-python@v1
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip wheel
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse --archs x86_64,aarch64
 
       - name: Build binary wheel and a source tarball
-        run: pip wheel --no-deps --wheel-dir ./dist .
+        run: pipx run build --sdist
+
+      - name: copy artifacts to dist folder
+        run: cp ./wheelhouse/* ./dist/
 
       - name: Upload Artifact for next job
         uses: actions/upload-artifact@master

.github/workflows/testing.yml

@@ -4,6 +4,12 @@ on:
   workflow_call:
 
 jobs:
+  rust:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Perform tests for rust modules
+        run: cargo test
   python:
     runs-on: ubuntu-22.04
     strategy:

.gitignore

@@ -26,3 +26,6 @@ experiments
 logprep.log
 /charts/logprep/charts
 examples/k8s/charts
+*.so
+target
+wheelhouse

CHANGELOG.md

@@ -9,6 +9,8 @@
 ### Improvements
 
 * remove AutoRuleCorpusTester
+* adds support for rust extension development
+* adds prebuild wheels for architectures `x86_64` and `aarch64` on `manylinux` and `musllinux` based linux platforms to releases
 
 ### Bugfix
 

Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "rust"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "rust"
+path = "rust/lib.rs"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = "0.22.0"

Dockerfile

@@ -8,6 +8,8 @@ ARG no_proxy
 
 ADD . /logprep
 WORKDIR /logprep
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
 RUN python -m pip install --upgrade pip wheel setuptools>=72.2.0
 RUN python -m venv /opt/venv
 # Make sure we use the virtualenv:

MANIFEST.in

@@ -0,0 +1,2 @@
+include Cargo.toml
+recursive-include rust *

logprep/framework/pipeline_manager.py

@@ -2,7 +2,6 @@
 
 # pylint: disable=logging-fstring-interpolation
 
-import ctypes
 import logging
 import logging.handlers
 import multiprocessing
@@ -21,8 +20,6 @@
 from logprep.util.configuration import Configuration
 from logprep.util.logging import LogprepMPQueueListener, logqueue
 
-libc = ctypes.CDLL("libc.so.6")
-
 logger = logging.getLogger("Manager")
 
 
@@ -48,7 +45,7 @@ def throttle(self, batch_size=1):
                 self.wait_time, int(self.wait_time * self.consumed_percent / batch_size)
             )
             # sleep times in microseconds
-            libc.usleep(sleep_time)
+            time.sleep(sleep_time / 1000)
 
     def put(self, obj, block=True, timeout=None, batch_size=1):
         """Put an obj into the queue."""

pyproject.toml

@@ -1,18 +1,30 @@
 [build-system]
-requires = ["setuptools>=68.0.0", "setuptools-scm>=8.0", "wheel"]
+requires = [
+  "setuptools>=68.0.0",
+  "setuptools-scm>=8.0",
+  "wheel",
+  "setuptools-rust",
+]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]
+package-dir = { "logprep" = "logprep" }
 packages = ["logprep"]
 
-
 [tool.setuptools_scm]
 fallback_version = "unset"
 
+[[tool.setuptools-rust.ext-modules]]
+# Private Rust extension module to be nested into the Python package
+target = "rust" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml,
+# but you can add a prefix to nest it inside of a Python package.
+path = "Cargo.toml" # Default value, can be omitted
+binding = "PyO3"    # Default value, can be omitted
+
 [project]
 name = "logprep"
 description = "Logprep allows to collect, process and forward log messages from various data sources."
-requires-python = ">=3.10,<3.12.4"
+requires-python = ">=3.10"
 readme = "README.md"
 dynamic = ["version"]
 license = { file = "LICENSE" }
@@ -98,6 +110,8 @@ dev = [
   "pytest-cov",
   "responses",
   "jinja2",
+  "maturin",
+  "cibuildwheel",
 ]
 
 doc = [
@@ -133,3 +147,12 @@ extend-exclude = '''
 
 [tool.isort]
 profile = "black"
+
+[tool.cibuildwheel]
+build = "cp310-* cp311-* cp312-*"
+skip = "*pp* *i686-unknown-linux-musl*"
+
+[tool.cibuildwheel.linux]
+archs = "x86_64 i686 aarch64"
+before-build = "curl -sSf https://sh.rustup.rs | sh -s -- -y"
+environment = 'PATH=$HOME/.cargo/bin:$PATH'

rust/.gitignore

@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version

rust/lib.rs

@@ -0,0 +1,15 @@
+mod tests;
+use pyo3::prelude::*;
+
+/// Formats the sum of two numbers as string.
+#[pyfunction]
+fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
+    Ok((a + b).to_string())
+}
+
+/// A Python module implemented in Rust.
+#[pymodule]
+fn rust(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
+    Ok(())
+}

rust/tests/mod.rs

@@ -0,0 +1,5 @@
+#[cfg(test)]
+#[test]
+fn it_works() {
+    assert_eq!("4", "4");
+}

tests/unit/framework/test_pipeline.py

@@ -5,7 +5,6 @@
 import multiprocessing
 from copy import deepcopy
 from logging import DEBUG
-from multiprocessing import Lock
 from unittest import mock
 
 import pytest

tests/unit/framework/test_pipeline_manager.py

@@ -277,14 +277,14 @@ def test_throttling_put_throttles(self):
             mock_throttle.assert_called()
 
     def test_throttle_sleeps(self):
-        with mock.patch("logprep.framework.pipeline_manager.libc.usleep") as mock_sleep:
+        with mock.patch("time.sleep") as mock_sleep:
             queue = ThrottlingQueue(multiprocessing.get_context(), 100)
             with mock.patch.object(queue, "qsize", return_value=95):
                 queue.throttle()
             mock_sleep.assert_called()
 
     def test_throttle_sleep_time_increases_with_qsize(self):
-        with mock.patch("logprep.framework.pipeline_manager.libc.usleep") as mock_sleep:
+        with mock.patch("time.sleep") as mock_sleep:
             queue = ThrottlingQueue(multiprocessing.get_context(), 100)
             with mock.patch.object(queue, "qsize", return_value=91):
                 queue.throttle()

tests/unit/rust/test_rust.py

@@ -0,0 +1,5 @@
+import rust
+
+
+def test_example():
+    assert rust.sum_as_string(1, 2) == "3"