pyproject.toml dependency and build managment (#526)

* add complete pyproject.toml
* remove requirements documentation
* change the way to run logprep
* fix readthedocs.yaml
* fix pex build
* update changelog

---------

Co-authored-by: dtrai2 <[email protected]>

.devcontainer/Dockerfile

@@ -8,11 +8,6 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
 ARG NODE_VERSION="none"
 RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
 
-# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
-# COPY requirements.txt /tmp/pip-tmp/
-# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
-#    && rm -rf /tmp/pip-tmp
-
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     && apt-get -y install --no-install-recommends pandoc

.devcontainer/devcontainer.json

@@ -37,7 +37,7 @@
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 	// Use 'postCreateCommand' to run commands after the container is created.
-	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+	"postCreateCommand": "pip3 install --user .[dev]",
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode"
 }

.github/workflows/ci.yml

@@ -16,9 +16,9 @@ jobs:
         python-version: ["3.10"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -30,6 +30,8 @@ jobs:
           pip install virtualenv
           pip install wheel
           pip install pex
+          pip install pip-tools
+          pip-compile -o requirements.txt
 
       - name: Repack confluent-kafka wheel
         run: |
@@ -59,17 +61,17 @@ jobs:
         python-version: ["3.10", "3.11", "3.12"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Install dependencies
         run: |
           pip install --upgrade pip wheel
-          pip install -r requirements_dev.txt
+          pip install .[dev]
 
       - name: Perform unit tests
         env:
@@ -91,10 +93,10 @@ jobs:
         python-version: ["3.10"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -103,8 +105,7 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get -y install pandoc
           pip install --upgrade pip wheel
-          pip install -r requirements_dev.txt
-          pip install -r doc/requirements.txt
+          pip install .[doc]
 
       - name: build docs
         run: |
@@ -124,9 +125,9 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -141,7 +142,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install --upgrade pip wheel
-          pip install -r requirements_dev.txt
+          pip install .[dev]
 
       - name: check black formating
         run: |

.github/workflows/main.yml

@@ -14,9 +14,9 @@ jobs:
         python-version: ["3.10"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -28,6 +28,8 @@ jobs:
           pip install virtualenv
           pip install wheel
           pip install pex
+          pip install pip-tools
+          pip-compile -o requirements.txt
 
       - name: Repack confluent-kafka wheel
         run: |
@@ -57,17 +59,17 @@ jobs:
         python-version: ["3.10", "3.11", "3.12"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Install dependencies
         run: |
           pip install --upgrade pip wheel
-          pip install -r requirements_dev.txt
+          pip install .[dev]
 
       - name: Perform unit tests
         env:
@@ -93,9 +95,9 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -111,7 +113,7 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
           pip install --upgrade pip wheel
-          pip install -r requirements_dev.txt
+          pip install .[dev]
 
       - name: check black formating
         run: |

.github/workflows/publish-latest-dev-release-to-pypi.yml

@@ -24,7 +24,7 @@ jobs:
           pip install --upgrade pip wheel
 
       - name: Build binary wheel and a source tarball
-        run: python setup.py sdist bdist_wheel
+        run: pip wheel --no-deps --wheel-dir ./dist .
 
       - uses: marvinpinto/action-automatic-releases@latest
         with:

.github/workflows/publish-release-to-pypi.yml

@@ -21,7 +21,7 @@ jobs:
           python -m pip install --upgrade pip wheel
 
       - name: Build binary wheel and a source tarball
-        run: python setup.py sdist bdist_wheel
+        run: pip wheel --no-deps --wheel-dir ./dist .
 
       - name: Upload Artifact for next job
         uses: actions/upload-artifact@master

.readthedocs.yaml

@@ -29,5 +29,7 @@ sphinx:
 # Optionally declare the Python requirements required to build your docs
 python:
   install:
-    - requirements: requirements_dev.txt
-    - requirements: doc/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - doc

CHANGELOG.md

@@ -7,6 +7,8 @@
 
 ### Improvements
 
+* refactor logprep build process and requirements management
+
 ### Bugfix
 
 ## v10.0.3

Dockerfile

@@ -13,7 +13,7 @@ RUN python -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 RUN python -m pip install --upgrade pip wheel setuptools
 
-RUN if [ "$LOGPREP_VERSION" = "dev" ]; then python setup.py sdist bdist_wheel && pip install ./dist/logprep-*.whl;\
+RUN if [ "$LOGPREP_VERSION" = "dev" ]; then pip install .;\
     elif [ "$LOGPREP_VERSION" = "latest" ]; then pip install git+https://github.com/fkie-cad/Logprep.git@latest; \
     else pip install "logprep==$LOGPREP_VERSION"; fi
 

Makefile

@@ -1,17 +1,7 @@
-# Build requirements(_dev).txt with pinned package versions from requirements(_dev).in
-build-requirements:
-	pip-compile requirements.in
-	pip-compile requirements_dev.in
-
-# Upgrade pinning in requirements(_dev).txt to newest available versions
-upgrade-requirements:
-	pip-compile --upgrade requirements.in
-	pip-compile --upgrade requirements_dev.in
-
-# Uninstall all packaged that are not present in requirements.txt and install what is in requirements.txt
+# Install all packages
 install-packages:
-	pip-sync
+	pip install -e .[dev]
 
 # Test all pytests
 test:
-	pytest ./tests --cov=logprep --cov-report=xml -vvv
+	pytest ./tests --cov=logprep --cov-report=xml -vvv

README.md

@@ -255,7 +255,7 @@ contribute to them.
 ```
 git clone https://github.com/fkie-cad/Logprep.git
 cd Logprep
-pip install -r requirements.txt
+pip install .
 ```
 
 **3. Option:** Installation via Github Release
@@ -316,11 +316,7 @@ Where `$CONFIG` is the path or uri to a configuration file (see the documentatio
 
 ### Reload the Configuration
 
-To change the configuration of Logprep it is not needed to restart Logprep entirely.
-Instead, it can be issued to reload the configuration.
-For this, the signal `SIGUSR1` must be send to the Logprep process.
-
-Additionally, a `config_refresh_interval` can be set to periodically and automatically refresh the given configuration.
+A `config_refresh_interval` can be set to periodically and automatically refresh the given configuration.
 This can be useful in case of containerized environments (such as Kubernetes), when pod volumes often change
 on the fly.
 
@@ -540,8 +536,8 @@ be built locally via:
 
 ```
 sudo apt install pandoc
+pip install -e .[doc]
 cd ./doc/
-pip install -r ./requirements.txt
 make html
 ```
 

create_pex.yml

@@ -25,7 +25,7 @@
 
   - name: Upgrade pip and wheel
     shell: "{{ python_interpreter_location }} -m pip install pip --upgrade &&
-    {{ python_interpreter_location }} -m pip install wheel --upgrade"
+    {{ python_interpreter_location }} -m pip install wheel --upgrade && pip install pip-tools && pip-compile -o requirements.txt"
 
   - name: Repack confluent-kafka wheel
     shell: "cd {{ project_location }} &&

doc/README.md

@@ -1,4 +1,4 @@
 ## needed setup to generate documentation
 
 * install pandoc with `apt-get install pandoc`
-* install requirements with `pip install -r requirements.txt`
+* install requirements with `pip install .[doc]` in project root

doc/source/development/index.rst

@@ -9,6 +9,5 @@ Development
    connector_how_to
    processor_how_to
    register_a_new_component
-   requirements
    testing
    coding_examples

doc/source/getting_started.rst

@@ -27,10 +27,10 @@ contribute to them.
 
     git clone https://github.com/fkie-cad/Logprep.git
     cd Logprep
-    pip install -r requirements.txt
+    pip install .
 
 To see if the installation was successful run
-:code:`PYTHONPATH="." python3 logprep/run_logprep.py --version`.
+:code:`logprep --version`.
 
 **3. Option:** Installation via Github Release
 
@@ -49,7 +49,6 @@ This option can be used to build a container image from a specific commit
 ..  code-block:: bash
 
     git clone https://github.com/fkie-cad/Logprep.git
-    cd Logprep
     docker build -t logprep .
 
 To see if the installation was successful run :code:`docker run logprep --version`.
@@ -64,12 +63,6 @@ If you have installed it via PyPI or the Github Development release just run:
 
     logprep run $CONFIG
 
-If you have installed Logprep via cloning the repository then you should run it via:
-
-..  code-block:: bash
-
-    PYTHONPATH="." python3 logprep/run_logprep.py run $CONFIG
-
 Where :code:`$CONFIG` is the path to a configuration file.
 For more information see the :ref:`configuration` section.
 

logprep/abc/component.py

@@ -1,4 +1,5 @@
 """ abstract module for components"""
+
 from abc import ABC
 from functools import cached_property
 from logging import Logger

logprep/abc/connector.py

@@ -1,4 +1,5 @@
 """ abstract module for connectors"""
+
 from attrs import define, field
 
 from logprep.abc.component import Component

logprep/abc/getter.py

@@ -1,4 +1,5 @@
 """Module for getter interface"""
+
 import json
 import os
 import re