58 pipeline should fail on code quality (#64)

Add code quality checks to pipeline

* Add code-quality jobs
* Fix lintings to pass pylint and black tests
* Add coverage badge
* Add jobs for push to main

.github/workflows/ci.yml

@@ -1,8 +1,11 @@
 name: CI
 
+# only run tests for pull requests cause no file has to be changed without review
+# open -> open the pull request
+# synchronize -> push to branch of pull request
 on:
-  push:
-    branches: [ 'main', 'master', 'dev-*', 'fix-*' ]
+  pull_request:
+    types: [opened, synchronize]
 
 jobs:
   build-pex:
@@ -58,25 +61,65 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements_dev.txt
-
-    - name: Peroform unit tests
-      env:
-        PYTEST_ADDOPTS: "--color=yes"
-      run: |
-        pytest  -vv tests/unit
-
-    - name: Peroform acceptance tests
-      env:
-        PYTEST_ADDOPTS: "--color=yes"
-      run: |
-        pytest  -vv tests/acceptance
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+
+      - name: Perform unit tests
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
+        run: |
+          pytest  -vv tests/unit
+
+      - name: Perform acceptance tests
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
+        run: |
+          pytest  -vv tests/acceptance
+
+  code-quality:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Get changed python files
+        id: changed-files
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            **/*.py
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+
+      - name: check black formating
+        run: |
+          pip install black
+          black --check --diff --config ./pyproject.toml .
+
+      - name: lint changed and added files
+        run: |
+          pylint --rcfile=.pylintrc --fail-under 9.5 ${{ steps.changed-files.outputs.all_changed_files }}
+
+      - name: Run tests and collect coverage
+        run: pytest tests --cov=logprep --cov-report=xml
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v2

.github/workflows/main.yml

@@ -0,0 +1,123 @@
+name: CI
+
+# ensures that main has our basic test and code quality
+on:
+  push:
+    branches: [main]
+
+jobs:
+  build-pex:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            /opt/hostedtoolcache/Python/3.6.15/x64/lib/python3.6/site-packages/
+            /opt/hostedtoolcache/Python/3.6.15/x64/bin/ansible
+            /opt/hostedtoolcache/Python/3.6.15/x64/bin/virtualenv
+            /opt/hostedtoolcache/Python/3.6.15/x64/bin/pex
+          key: ${{ hashFiles('setup.py') }}-${{ hashFiles('requirements_dev.txt') }}
+
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install ansible
+          pip install virtualenv
+          pip install wheel
+          pip install pex
+
+      - name: Repack confluent-kafka wheel
+        run: |
+          rm -rf tmp_pip_cache &&
+          mkdir tmp_pip_cache &&
+          cd tmp_pip_cache &&
+          python -m pip download $(cat ../requirements.txt | grep confluent-kafka) &&
+          unzip * &&
+          rm *.whl &&
+          python -m wheel pack .
+
+      - name: Build Pex File
+        run: |
+          pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache
+
+      - name: Upload PEX
+        uses: actions/upload-artifact@v2
+        with:
+          name: Logprep
+          path: logprep.pex
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+
+      - name: Perform unit tests
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
+        run: |
+          pytest  -vv tests/unit
+
+      - name: Perform acceptance tests
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
+        run: |
+          pytest  -vv tests/acceptance
+
+  code-quality:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Get changed python files
+        id: changed-files
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            **/*.py
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+
+      - name: check black formating
+        run: |
+          pip install black
+          black --check --diff --config ./pyproject.toml .
+
+      - name: lint changed and added files
+        run: |
+          pylint --rcfile=.pylintrc --fail-under 9.5 ${{ steps.changed-files.outputs.all_changed_files }}
+
+      - name: Run tests and collect coverage
+        run: pytest tests --cov=logprep --cov-report=xml
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v2

README.md

@@ -5,7 +5,10 @@
 ![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/fkie-cad/logprep/CI/main)
 [![Documentation Status](https://readthedocs.org/projects/logprep/badge/?version=latest)](http://logprep.readthedocs.io/?badge=latest)
 ![GitHub contributors](https://img.shields.io/github/contributors/fkie-cad/Logprep)
-![GitHub Repo stars](https://img.shields.io/github/stars/fkie-cad/logprep?style=social)  
+<a href="https://codecov.io/github/fkie-cad/Logprep" target="_blank">
+    <img src="https://img.shields.io/codecov/c/github/fkie-cad/Logprep?color=%2334D058" alt="Coverage">
+</a>
+![GitHub Repo stars](https://img.shields.io/github/stars/fkie-cad/logprep?style=social)
 </h3>
 
 ## Introduction

logprep/processor/generic_resolver/factory.py

@@ -13,11 +13,9 @@ def create(name: str, configuration: dict, logger) -> GenericResolver:
         GenericResolverFactory._check_configuration(configuration)
 
         generic_resolver = GenericResolver(
-            name,
-            configuration['specific_rules'],
-            configuration['generic_rules'],
-            configuration.get("tree_config"),
-            logger
+            name=name,
+            configuration=configuration,
+            logger=logger,
         )
 
         return generic_resolver