diff --git a/.github/workflows/publish-latest-dev-release-to-pypi.yml b/.github/workflows/publish-latest-dev-release-to-pypi.yml index d4fa4ed5d..69a99e72e 100644 --- a/.github/workflows/publish-latest-dev-release-to-pypi.yml +++ b/.github/workflows/publish-latest-dev-release-to-pypi.yml @@ -17,14 +17,24 @@ jobs: - name: Initialize Python uses: actions/setup-python@v1 with: - python-version: "3.10" + python-version: "3.11" - - name: Install dependencies - run: | - pip install --upgrade pip wheel + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse --archs x86_64,aarch64 - name: Build binary wheel and a source tarball - run: pip wheel --no-deps --wheel-dir ./dist . + run: pipx run build --sdist + + - name: copy artifacts to dist folder + run: cp ./wheelhouse/* ./dist/ - uses: marvinpinto/action-automatic-releases@latest with: diff --git a/.github/workflows/publish-release-to-pypi.yml b/.github/workflows/publish-release-to-pypi.yml index e9cfe1815..432563312 100644 --- a/.github/workflows/publish-release-to-pypi.yml +++ b/.github/workflows/publish-release-to-pypi.yml @@ -14,14 +14,24 @@ jobs: - name: Initialize Python uses: actions/setup-python@v1 with: - python-version: "3.10" + python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse --archs x86_64,aarch64 - name: Build binary wheel and a source tarball - run: pip wheel --no-deps --wheel-dir ./dist . + run: pipx run build --sdist + + - name: copy artifacts to dist folder + run: cp ./wheelhouse/* ./dist/ - name: Upload Artifact for next job uses: actions/upload-artifact@master diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c785383f8..fa54b279f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -4,6 +4,12 @@ on: workflow_call: jobs: + rust: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Perform tests for rust modules + run: cargo test python: runs-on: ubuntu-22.04 strategy: diff --git a/.gitignore b/.gitignore index bb0cf0161..873ae18a4 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ experiments logprep.log /charts/logprep/charts examples/k8s/charts +*.so +target +wheelhouse diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f43db15a..947e80198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ### Improvements * remove AutoRuleCorpusTester +* adds support for rust extension development +* adds prebuild wheels for architectures `x86_64` and `aarch64` on `manylinux` and `musllinux` based linux platforms to releases ### Bugfix diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..2d8b6bb8b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "rust" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "rust" +path = "rust/lib.rs" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = "0.22.0" diff --git a/Dockerfile b/Dockerfile index f5e2f8a12..28b35d4b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,8 @@ ARG no_proxy ADD . /logprep WORKDIR /logprep +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" RUN python -m pip install --upgrade pip wheel setuptools>=72.2.0 RUN python -m venv /opt/venv # Make sure we use the virtualenv: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..81ba8b93f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include Cargo.toml +recursive-include rust * diff --git a/logprep/framework/pipeline_manager.py b/logprep/framework/pipeline_manager.py index 7b374e78e..0c27d7348 100644 --- a/logprep/framework/pipeline_manager.py +++ b/logprep/framework/pipeline_manager.py @@ -2,7 +2,6 @@ # pylint: disable=logging-fstring-interpolation -import ctypes import logging import logging.handlers import multiprocessing @@ -21,8 +20,6 @@ from logprep.util.configuration import Configuration from logprep.util.logging import LogprepMPQueueListener, logqueue -libc = ctypes.CDLL("libc.so.6") - logger = logging.getLogger("Manager") @@ -48,7 +45,7 @@ def throttle(self, batch_size=1): self.wait_time, int(self.wait_time * self.consumed_percent / batch_size) ) # sleep times in microseconds - libc.usleep(sleep_time) + time.sleep(sleep_time / 1000) def put(self, obj, block=True, timeout=None, batch_size=1): """Put an obj into the queue.""" diff --git a/pyproject.toml b/pyproject.toml index bf280885e..c1f8799fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,30 @@ [build-system] -requires = ["setuptools>=68.0.0", "setuptools-scm>=8.0", "wheel"] +requires = [ + "setuptools>=68.0.0", + "setuptools-scm>=8.0", + "wheel", + "setuptools-rust", +] build-backend = "setuptools.build_meta" [tool.setuptools] +package-dir = { "logprep" = "logprep" } packages = ["logprep"] - [tool.setuptools_scm] fallback_version = "unset" +[[tool.setuptools-rust.ext-modules]] +# Private Rust extension module to be nested into the Python package +target = "rust" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, +# but you can add a prefix to nest it inside of a Python package. +path = "Cargo.toml" # Default value, can be omitted +binding = "PyO3" # Default value, can be omitted + [project] name = "logprep" description = "Logprep allows to collect, process and forward log messages from various data sources." -requires-python = ">=3.10,<3.12.4" +requires-python = ">=3.10" readme = "README.md" dynamic = ["version"] license = { file = "LICENSE" } @@ -98,6 +110,8 @@ dev = [ "pytest-cov", "responses", "jinja2", + "maturin", + "cibuildwheel", ] doc = [ @@ -133,3 +147,12 @@ extend-exclude = ''' [tool.isort] profile = "black" + +[tool.cibuildwheel] +build = "cp310-* cp311-* cp312-*" +skip = "*pp* *i686-unknown-linux-musl*" + +[tool.cibuildwheel.linux] +archs = "x86_64 i686 aarch64" +before-build = "curl -sSf https://sh.rustup.rs | sh -s -- -y" +environment = 'PATH=$HOME/.cargo/bin:$PATH' diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 000000000..c8f044299 --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version diff --git a/rust/lib.rs b/rust/lib.rs new file mode 100644 index 000000000..de1024aff --- /dev/null +++ b/rust/lib.rs @@ -0,0 +1,15 @@ +mod tests; +use pyo3::prelude::*; + +/// Formats the sum of two numbers as string. +#[pyfunction] +fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) +} + +/// A Python module implemented in Rust. +#[pymodule] +fn rust(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + Ok(()) +} diff --git a/rust/tests/mod.rs b/rust/tests/mod.rs new file mode 100644 index 000000000..d84a168fa --- /dev/null +++ b/rust/tests/mod.rs @@ -0,0 +1,5 @@ +#[cfg(test)] +#[test] +fn it_works() { + assert_eq!("4", "4"); +} diff --git a/tests/unit/framework/test_pipeline.py b/tests/unit/framework/test_pipeline.py index 4c4a2f36f..4ffb1db36 100644 --- a/tests/unit/framework/test_pipeline.py +++ b/tests/unit/framework/test_pipeline.py @@ -5,7 +5,6 @@ import multiprocessing from copy import deepcopy from logging import DEBUG -from multiprocessing import Lock from unittest import mock import pytest diff --git a/tests/unit/framework/test_pipeline_manager.py b/tests/unit/framework/test_pipeline_manager.py index f31cb3edf..36b474394 100644 --- a/tests/unit/framework/test_pipeline_manager.py +++ b/tests/unit/framework/test_pipeline_manager.py @@ -277,14 +277,14 @@ def test_throttling_put_throttles(self): mock_throttle.assert_called() def test_throttle_sleeps(self): - with mock.patch("logprep.framework.pipeline_manager.libc.usleep") as mock_sleep: + with mock.patch("time.sleep") as mock_sleep: queue = ThrottlingQueue(multiprocessing.get_context(), 100) with mock.patch.object(queue, "qsize", return_value=95): queue.throttle() mock_sleep.assert_called() def test_throttle_sleep_time_increases_with_qsize(self): - with mock.patch("logprep.framework.pipeline_manager.libc.usleep") as mock_sleep: + with mock.patch("time.sleep") as mock_sleep: queue = ThrottlingQueue(multiprocessing.get_context(), 100) with mock.patch.object(queue, "qsize", return_value=91): queue.throttle() diff --git a/tests/unit/rust/__init__.py b/tests/unit/rust/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/rust/test_rust.py b/tests/unit/rust/test_rust.py new file mode 100644 index 000000000..9bf95b675 --- /dev/null +++ b/tests/unit/rust/test_rust.py @@ -0,0 +1,5 @@ +import rust + + +def test_example(): + assert rust.sum_as_string(1, 2) == "3"