Merge pull request #162 from guilieb/feature/qa-improvements

Update and fix issues raised by Python QA tools

.flake8

@@ -0,0 +1,28 @@
+[flake8]
+extend-ignore =
+    # Line too long
+    E501
+    # Whitespace issues
+    D2
+    # Quote issues
+    D3
+    # Docstring Content Issues
+    D4
+    # Missing docstring in public module
+    D100
+    # Missing docstring in public package
+    D104
+    # Missing docstring in magic method
+    D105
+    # Missing docstring in public nested class
+    D106
+    # Missing docstring in __init__
+    D107
+    # Line break occurred before a binary operator
+    W503
+    # Whitespace before ':'
+    E203
+extend-exclude =
+    tests/**,
+    docs/**
+max-line-length = 119

.github/ISSUE_TEMPLATE/bug_report.md

@@ -47,4 +47,3 @@ Please write the debug information inside <code>```</code> quotes in order to pr
 
 ```
 ```
-

.github/workflows/main.yml

@@ -5,11 +5,23 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: psf/black@stable
+      - name: Checkout the repository
+        uses: actions/checkout@v3
+      - name: Run black
+        uses: psf/black@stable
         with:
           src: "./melusine"
 
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the repository
+      uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+    - name: Run pre-commit
+      uses: pre-commit/[email protected]
+
   test:
     name: Test
     needs: lint
@@ -18,7 +30,7 @@ jobs:
       fail-fast: false
       max-parallel: 2
       matrix:
-        python-version: ["3.8", "3.10", "3.12"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
     - name: Checkout the repository
       uses: actions/checkout@v3

.gitignore

@@ -107,3 +107,6 @@ dmypy.json
 
 # Pytest-env
 pytest.ini
+
+# IDEA/Jetbrains
+.idea

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_language_version:
     python: python3
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v4.5.0
     hooks:
     -   id: check-ast
     -   id: check-byte-order-marker
@@ -11,7 +11,9 @@ repos:
     -   id: check-executables-have-shebangs
     -   id: check-json
     -   id: check-yaml
-        exclude: ^chart/
+        # Ignore mkdocs because the linter fails on the pymdownx specific
+        # syntax to inject Python code from configuration.
+        exclude: mkdocs.yml
     -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: ^(docs/|gdocs/)
@@ -24,42 +26,49 @@ repos:
         args: ['--maxkb=500']
     - id: no-commit-to-branch
       args: ['--branch', 'master', '--branch', 'develop']
+
 -   repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 23.12.1
     hooks:
     -   id: black
-        args: [--line-length=120]
         additional_dependencies: ['click==8.0.4']
+
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.931'
+    rev: 'v1.8.0'
     hooks:
     -   id: mypy
-        args: [--ignore-missing-imports, --disallow-untyped-defs, --show-error-codes, --no-site-packages]
         files: ^melusine
+
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 7.0.0
     hooks:
     -   id: flake8
-        exclude: '^tests/|^docs'
-        args: ['--ignore=E501,D2,D3,D4,D104,D100,D106,D107,W503,D105,E203']
         additional_dependencies: [ flake8-docstrings, "flake8-bugbear==22.8.23" ]
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.1.11
+  hooks:
+    - id: ruff
+
 -   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.4.2
+    rev: v5.10.1
     hooks:
     -   id: isort
-        args: ["--profile", "black", "-l", "120"]
+
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.2
+    rev: v3.15.0
     hooks:
     -   id: pyupgrade
         args: [--py37-plus]
+
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.8.0
+    rev: 1.16.0
     hooks:
     -   id: blacken-docs
-        additional_dependencies: [black==21.12b0]
+        additional_dependencies: [black>=22.1]
+
 -   repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v2.1.1
+    rev: v3.0.0
     hooks:
     -   id: conventional-pre-commit
         stages: [commit-msg]

melusine/__init__.py

@@ -2,11 +2,11 @@
 Top-level package.
 """
 import logging
-import pandas as pd
-
 from ctypes import CDLL, cdll
 from typing import Any, Optional
 
+import pandas as pd
+
 from melusine._config import config
 
 __all__ = ["config"]

melusine/backend/active_backend.py

@@ -102,7 +102,7 @@ def apply_transform(
             **kwargs,
         )
 
-    def copy(self, data: Any, fields: List[str] = None) -> Any:
+    def copy(self, data: Any, fields: Optional[List[str]] = None) -> Any:
         """
         Method to make a copy of the input dataset.
 
@@ -136,7 +136,7 @@ def get_fields(self, data: Any) -> List[str]:
         """
         return self.backend.get_fields(data=data)
 
-    def add_fields(self, left: Any, right: Any, fields: List[str] = None) -> Any:
+    def add_fields(self, left: Any, right: Any, fields: Optional[List[str]] = None) -> Any:
         """
         Method to add fields from the right object to the left object
 

melusine/backend/base_backend.py

@@ -51,7 +51,7 @@ def apply_transform(
         """
 
     @abstractmethod
-    def add_fields(self, left: Any, right: Any, fields: List[str] = None) -> Any:
+    def add_fields(self, left: Any, right: Any, fields: Optional[List[str]] = None) -> Any:
         """
         Method to add fields form the right object to the left object.
 
@@ -71,7 +71,7 @@ def add_fields(self, left: Any, right: Any, fields: List[str] = None) -> Any:
         """
 
     @abstractmethod
-    def copy(self, data: Any, fields: List[str] = None) -> Any:
+    def copy(self, data: Any, fields: Optional[List[str]] = None) -> Any:
         """
         Method to make a copy of the dataset.
 

melusine/backend/dict_backend.py

@@ -80,7 +80,9 @@ def apply_transform(
 
         return data
 
-    def add_fields(self, left: Dict[str, Any], right: Dict[str, Any], fields: List[str] = None) -> Dict[str, Any]:
+    def add_fields(
+        self, left: Dict[str, Any], right: Dict[str, Any], fields: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
         """
         Method to add fields form the right object to the left object.
 
@@ -106,7 +108,7 @@ def add_fields(self, left: Dict[str, Any], right: Dict[str, Any], fields: List[s
 
         return left
 
-    def copy(self, data: Dict[str, Any], fields: List[str] = None) -> Dict[str, Any]:
+    def copy(self, data: Dict[str, Any], fields: Optional[List[str]] = None) -> Dict[str, Any]:
         """
         Method to make a copy of the dataset.
 

melusine/backend/pandas_backend.py

@@ -238,7 +238,7 @@ def apply_transform_multiprocessing(
     def apply_joblib_dataframe(
         df: pd.DataFrame,
         func: Callable,
-        expand: str = None,
+        expand: Optional[str] = None,
         progress_bar: bool = False,
         **kwargs: Any,
     ) -> pd.DataFrame:
@@ -260,7 +260,7 @@ def apply_joblib_dataframe(
     def apply_joblib_series(
         s: pd.Series,
         func: Callable,
-        expand: str = None,
+        expand: Optional[str] = None,
         progress_bar: bool = False,
         **kwargs: Any,
     ) -> pd.DataFrame:
@@ -280,7 +280,7 @@ def apply_joblib_series(
 
         return result
 
-    def add_fields(self, left: pd.DataFrame, right: pd.DataFrame, fields: List[str] = None) -> pd.DataFrame:
+    def add_fields(self, left: pd.DataFrame, right: pd.DataFrame, fields: Optional[List[str]] = None) -> pd.DataFrame:
         """
         Method to add fields form the right object to the left object.
 
@@ -302,7 +302,7 @@ def add_fields(self, left: pd.DataFrame, right: pd.DataFrame, fields: List[str]
 
         return left
 
-    def copy(self, data: pd.DataFrame, fields: List[str] = None) -> pd.DataFrame:
+    def copy(self, data: pd.DataFrame, fields: Optional[List[str]] = None) -> pd.DataFrame:
         """
         Method to make a copy of the dataset.