From 73d0e84007fc76cc9d27a13d1e589cd20ad69a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Fri, 16 Aug 2024 15:27:52 +0200 Subject: [PATCH 1/2] replaced all python linters and formatters with ruff --- .flake8 | 10 -- .pre-commit-config.yaml | 34 +--- .pylintrc | 390 ---------------------------------------- pyproject.toml | 88 +++++++-- 4 files changed, 85 insertions(+), 437 deletions(-) delete mode 100644 .flake8 delete mode 100644 .pylintrc diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 5b686167..00000000 --- a/.flake8 +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -extend-ignore = E501,W503 -extend-select = E504 -exclude = - .git, - __pycache__, - fact_extractor/bin, -per-file-ignores = - fact_extractor/plugins/*: E402 - fact_extractor/install/install.py: E402 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da5ba7be..c5a708cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,7 @@ default_stages: [commit, push] -exclude: ^src/bin/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + rev: v4.0.1 hooks: - id: check-added-large-files args: [--maxkb=10000] @@ -13,9 +12,6 @@ repos: - id: check-yaml - - id: end-of-file-fixer - types: [python] - - id: fix-encoding-pragma args: [--remove] @@ -29,28 +25,14 @@ repos: - id: pretty-format-json args: [--autofix] - - id: trailing-whitespace - types: [python] - -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - args: [--select=W504, --ignore=E501,W503] - -- repo: https://github.com/pre-commit/mirrors-pylint - rev: v2.2.2 - hooks: - - id: pylint - language: system - args: [--rcfile=.pylintrc] - -- repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 2.1.5 +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 hooks: - id: shellcheck -- repo: https://github.com/pryorda/dockerfilelint-precommit-hooks - rev: v0.1.0 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.5.6' hooks: - - id: dockerfilelint + - id: ruff + args: [ --fix, --exit-non-zero-on-fix ] + - id: ruff-format diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index b8dcba64..00000000 --- a/.pylintrc +++ /dev/null @@ -1,390 +0,0 @@ -[MASTER] - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns=test/,src/bin - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint.extensions.bad_builtin,pylint.extensions.mccabe - -# Use multiple processes to speed up Pylint. -jobs=2 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. This option is deprecated -# and it will be removed in Pylint 2.0. -optimize-ast=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s) -disable=missing-docstring,locally-disabled,logging-fstring-interpolation,wrong-import-order - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, html -output-format=colorized - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -msg-template="{path}:{line}: [{symbol}:{obj}] {msg}" - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -ignored-classes= - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=200 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,50}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,50}$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{1,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{1,30}$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,33}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,33}$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{1,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{1,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{1,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{1,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[ELIF] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[DESIGN] - -# McCabe complexity cyclomatic threshold -max-complexity=7 - -# Maximum number of arguments for function / method -max-args=7 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=0 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=40 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/pyproject.toml b/pyproject.toml index decee8b0..748a318a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,78 @@ -[tool.black] +[tool.ruff] +extend-exclude = [ + "bin", + "fact_extractor/plugins/unpacking/*/test/data/*", +] line-length = 120 -skip-string-normalization = true -target-version = ['py38'] - -[tool.isort] -line_length=120 -default_section = "THIRDPARTY" -known_first_party = ["analysis", "compare", "helperFunctions", "install", "intercom", "objects", "plugins", "scheduler", - "statistic", "storage", "test", "unpacker", "version", "web_interface"] -known_third_party = "docker" -profile = "black" +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle Error + "W", # pycodestyle Warning + "C90", # mccabe + "I", # isort + "N", # pep8-naming + "UP", # pyupgrade + "B", # flake8-bugbear + "A", # flake8-builtins + "C4", # flake8-comprehensions + "EXE", # flake8-executable + "FA", # flake8-future-annotations + "ISC", # flake8-implicit-str-concat + "PIE", # flake8-pie + "T20", # flake8-print + "PT", # flake8-pytest-style + "Q", # flake8-quotes + "RET", # flake8-return + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "ARG", # flake8-unused-arguments + "PTH", # flake8-use-pathlib + "ERA", # eradicate + "PL", # Pylint + "PERF", # Perflint + "RUF", # Ruff-specific rules +] +ignore = [ + "A003", + "PERF203", + "PERF401", + "RUF001", + "RUF002", + "RUF003", + "RUF015", + # pydantic only supports these from python>=3.9 + "UP006", + "UP007", + # rules may cause conflicts when used with the formatter + "ISC001", + "Q001", +] +fixable = ["ALL"] + +[tool.ruff.lint.per-file-ignores] +"test*.py" = ["ARG002"] +"conftest.py" = ["ARG002"] + +[tool.ruff.lint.isort] +known-first-party = ["config", "helperFunctions", "install", "plugins", "test", "unpacker", "version"] + +[tool.ruff.lint.pylint] +max-args=7 +max-public-methods = 40 + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false + +[tool.ruff.lint.flake8-quotes] +inline-quotes = "single" +multiline-quotes = "single" +docstring-quotes = "double" + +[tool.ruff.format] +quote-style = "single" From 79393f8884434a0b56698fd39d7ae2016019ae3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Fri, 16 Aug 2024 15:28:09 +0200 Subject: [PATCH 2/2] replaced flake8 github action with pre-commit action --- .github/workflows/Lintly-Flake8.yml | 15 --------------- .github/workflows/pre-commit.yml | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/Lintly-Flake8.yml create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/Lintly-Flake8.yml b/.github/workflows/Lintly-Flake8.yml deleted file mode 100644 index d1b8f7da..00000000 --- a/.github/workflows/Lintly-Flake8.yml +++ /dev/null @@ -1,15 +0,0 @@ -on: [pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: grantmcconnaughey/lintly-flake8-github-action@d9db4fd0be9fb1cd19206a48ec0773bd93b82cbd - with: - # The GitHub API token to create reviews with - token: ${{ secrets.GITHUB_TOKEN }} - # Fail if "new" violations detected or "any", default "new" - failIf: new - # Additional arguments to pass to flake8, default "." (current directory) - args: "--config .flake8 fact_extractor" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..df887254 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,16 @@ +name: pre-commit + +on: + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}