drop support for python 3.6, 3.7, 3.8 and add support for 3.10, 3.11 (#…

…228)

* drop support for python 3.6, 3.7, 3.8
* add support for python 3.10 and 3.11
* update Changelog

.dockerignore

@@ -2,3 +2,8 @@
 .github
 .venv
 .semgrep_rules
+.vscode
+htmlcov
+logs
+coverage.xml
+

.github/workflows/ci.yml

@@ -9,11 +9,11 @@ on:
 
 jobs:
   build-pex:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
-        python-version: ["3.6"]
+        python-version: ["3.9"]
 
     steps:
       - uses: actions/checkout@v3
@@ -50,14 +50,13 @@ jobs:
         with:
           name: Logprep
           path: logprep.pex
-        if: matrix.python-version == 3.6
 
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
@@ -69,7 +68,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
+          pip install --upgrade pip wheel
           pip install -r requirements_dev.txt
 
       - name: Perform unit tests
@@ -85,7 +85,7 @@ jobs:
           pytest  -vv tests/acceptance
 
   build-docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -102,7 +102,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
+          pip install --upgrade pip wheel
           pip install -r requirements_dev.txt
           pip install -r doc/requirements.txt
 
@@ -113,7 +114,7 @@ jobs:
           make clean html
 
   code-quality:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -140,17 +141,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
+          pip install --upgrade pip wheel
           pip install -r requirements_dev.txt
 
-      - name: Install semgrep if python version > 3.6
-        run: |
-          pip install semgrep
-        if: matrix.python-version == 3.7 || matrix.python-version == 3.8 || matrix.python-version == 3.9
-
       - name: check black formating
         run: |
-          pip install black
           black --check --diff --config ./pyproject.toml .
 
       - name: lint changed and added files

.github/workflows/main.yml

@@ -11,15 +11,15 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+        python-version: ["3.9"]
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: "pip"
 
       - name: Install dependencies
         run: |
@@ -48,26 +48,26 @@ jobs:
         with:
           name: Logprep
           path: logprep.pex
-        if: matrix.python-version == 3.6
 
   test:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: "pip"
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
+          pip install --upgrade pip wheel
           pip install -r requirements_dev.txt
 
       - name: Perform unit tests
@@ -87,7 +87,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+        python-version: ["3.9"]
 
     steps:
       - uses: actions/checkout@v3
@@ -99,7 +99,7 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: "pip"
 
       - name: Get changed python files
         id: changed-files
@@ -110,17 +110,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
+          pip install --upgrade pip wheel
           pip install -r requirements_dev.txt
 
-      - name: Install semgrep if python version > 3.6
-        run: |
-          pip install semgrep
-        if: matrix.python-version == 3.7 || matrix.python-version == 3.8 || matrix.python-version == 3.9
-
       - name: check black formating
         run: |
-          pip install black
           black --check --diff --config ./pyproject.toml .
 
       - name: lint changed and added files
@@ -135,5 +130,5 @@ jobs:
         uses: codecov/codecov-action@v2
 
       - name: Check semgrep rules
-        if: steps.changed-files.outputs.all_changed_files && (matrix.python-version == 3.7 || matrix.python-version == 3.8 || matrix.python-version == 3.9)
-        run: semgrep -c .semgrep_rules -c r/python --error -l python --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}
+        if: steps.changed-files.outputs.all_changed_files
+        run: semgrep -c .semgrep_rules -c r/python --error -l python --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}

.github/workflows/publish-latest-dev-release-to-pypi.yml

@@ -17,12 +17,11 @@ jobs:
       - name: Initialize Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.9
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install wheel
+          pip install --upgrade pip wheel
 
       - name: Build binary wheel and a source tarball
         run: python setup.py sdist bdist_wheel

.github/workflows/publish-release-to-pypi.yml

@@ -16,13 +16,12 @@ jobs:
       - name: Initialize Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.6
-          cache: 'pip'
+          python-version: 3.9
+          cache: "pip"
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install wheel
+          python -m pip install --upgrade pip wheel
 
       - name: Build binary wheel and a source tarball
         run: python setup.py sdist bdist_wheel
@@ -32,4 +31,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@master
         with:
           user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          password: ${{ secrets.PYPI_API_TOKEN }}

CHANGELOG.md

@@ -4,9 +4,19 @@
 
 ## next release
 
+### Breaking
+
+* drop support for python `3.6`, `3.7`, `3.8`
+
+### Features
+
 * Add an `http input connector` that spawns a uvicorn server which parses requests content to events.
 * provide the possibility to consume lists, rules and configuration from files and http endpoints
 
+### Improvements
+
+* add support for python `3.10` and `3.11`
+
 ## v4.0.0
 ### Breaking
 

Dockerfile

@@ -1,16 +1,18 @@
-FROM python:3.9-slim as build
+FROM python:3.11-slim as build
 
 ADD . /logprep
 WORKDIR /logprep
-RUN python -m pip install --upgrade pip && python -m pip install wheel
+RUN apt-get update && apt-get -y install libhyperscan-dev librdkafka-dev
+RUN python -m pip install --upgrade pip wheel
 RUN python setup.py sdist bdist_wheel
 
 
-FROM python:3.9-slim as prod
+FROM python:3.11-slim as prod
 RUN useradd -s /bin/sh -m -c "logprep user" logprep
 USER logprep
 ENV PATH=/home/logprep/.local/bin:$PATH
 COPY --from=build /logprep/dist/logprep-0+unknown-py3-none-any.whl /
+RUN apt-get update && apt-get -y install libhyperscan-dev librdkafka-dev
 RUN pip install logprep-0+unknown-py3-none-any.whl
 ENV PROMETHEUS_MULTIPROC_DIR=/tmp/logprep/prometheus_multiproc/
 ENV TLDEXTRACT_CACHE=/tmp/logprep/tld_extract_cache/

README.md

@@ -230,7 +230,10 @@ Details about the rule language and how to write rules for the processors can be
 
 ### Installation
 
-Python should be present on the system, currently supported are the versions 3.6 - 3.9.
+Python should be present on the system, currently supported are the versions 3.9 - 3.11.
+You have to install `hyperscan` and `rdkafka` before installing logprep. On debian based
+systems, this is performed by `sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev`.
+
 To install Logprep you have following options:
 
 **1. Option:** Installation via PyPI: 
@@ -276,16 +279,16 @@ Those can be executed via: `tox -e [name of the test environment]`.
 For Example:
 
 ```
-tox -e py36-all
+tox -e py39-all
 ```
 
 This runs all tests, calculates the test coverage and evaluates the code quality for the python 
-3.6 version.
+3.9 version.
 
 Multiple environments can be tested within one call: 
 
 ```
-tox -e py36-all -e py37-all -e py38-all -e py39-all
+tox -e py39-all -e py310-all -e py311-all
 ```
 
 If you want to run them in parallel attach the option `-p`.
@@ -301,15 +304,12 @@ tox -av
 In case the requirements change, the test environments must be rebuilt with the parameter `-r`:
 
 ```
-tox -e all -r
+tox -e py39 -e py310 -e py311 -r
 ```
 
 ### Semgrep
 
-To run the semgrep rules against the semgrep python registry at least python 3.7 is required. 
-Because of that and the default logprep support for python 3.6 semgrep is not part of the 
-requirements_dev.txt. 
-If you want to run semgrep rules use a python environment with version higher than 3.6 and run
+If you want to run semgrep rules run
 
 ```
 pip install semgrep
@@ -318,7 +318,7 @@ pip install semgrep
 Afterwards you can just call the tox environment with for example 
 
 ```
-tox -e py37-semgrep
+tox -e py39-semgrep
 ```
 
 ### Running Logprep

doc/source/getting_started.rst

@@ -5,7 +5,7 @@ Getting Started
 Installation
 ============
 
-Python should be present on the system, currently supported are the versions 3.6 - 3.9.
+Python should be present on the system. Currently, Python 3.9 - 3.11 are supported.
 To install Logprep you have following options:
 
 **1. Option:** Installation via PyPI:

logprep/abc/component.py

@@ -1,5 +1,4 @@
 """ abstract module for connectors"""
-import sys
 from abc import ABC
 from logging import Logger
 
@@ -17,10 +16,8 @@ class Config:
         type: str = field(validator=validators.instance_of(str))
         """Type of the component"""
 
-    __slots__ = ["name", "_logger", "_config"]
-
-    if not sys.version_info.minor < 7:
-        __slots__.append("__dict__")
+    # __dict__ is added to support functools.cached_property
+    __slots__ = ["name", "_logger", "_config", "__dict__"]
 
     name: str
     _logger: Logger

logprep/connector/confluent_kafka/input.py

@@ -27,8 +27,7 @@
         offset_reset_policy: smallest
 """
 import json
-import sys
-from functools import partial
+from functools import cached_property, partial
 from logging import Logger
 from socket import getfqdn
 from typing import Any, List, Tuple, Union
@@ -39,11 +38,6 @@
 from logprep.abc.input import CriticalInputError, Input
 from logprep.util.validators import dict_with_keys_validator
 
-if sys.version_info.minor < 8:  # pragma: no cover
-    from backports.cached_property import cached_property  # pylint: disable=import-error
-else:
-    from functools import cached_property
-
 
 class ConfluentKafkaInput(Input):
     """A kafka input connector."""