Respect ignore configuration options when --recursive=y. (#6528)

* Ignore specified files/directories in recursive mode

Co-authored-by: Pierre Sassoulas <[email protected]>
Co-authored-by: Daniël van Noord <[email protected]>

Author: 3 people

ChangeLog

@@ -21,6 +21,10 @@ What's New in Pylint 2.13.9?
 Release date: TBA
 
 
+* Respect ignore configuration options with ``--recursive=y``.
+
+  Closes #6471
+
 * Fix false positives for ``no-name-in-module`` and ``import-error`` for ``numpy.distutils`` and ``pydantic``.
 
   Closes #6497

pylint/lint/expand_modules.py

@@ -43,6 +43,20 @@ def _is_in_ignore_list_re(element: str, ignore_list_re: List[Pattern]) -> bool:
     return any(file_pattern.match(element) for file_pattern in ignore_list_re)
 
 
+def _is_ignored_file(
+    element: str,
+    ignore_list: list[str],
+    ignore_list_re: list[Pattern[str]],
+    ignore_list_paths_re: list[Pattern[str]],
+) -> bool:
+    basename = os.path.basename(element)
+    return (
+        basename in ignore_list
+        or _is_in_ignore_list_re(basename, ignore_list_re)
+        or _is_in_ignore_list_re(element, ignore_list_paths_re)
+    )
+
+
 def expand_modules(
     files_or_modules: List[str],
     ignore_list: List[str],
@@ -58,10 +72,8 @@ def expand_modules(
 
     for something in files_or_modules:
         basename = os.path.basename(something)
-        if (
-            basename in ignore_list
-            or _is_in_ignore_list_re(os.path.basename(something), ignore_list_re)
-            or _is_in_ignore_list_re(something, ignore_list_paths_re)
+        if _is_ignored_file(
+            something, ignore_list, ignore_list_re, ignore_list_paths_re
         ):
             continue
         module_path = get_python_path(something)

pylint/lint/pylinter.py

@@ -40,7 +40,7 @@
     MSG_TYPES_LONG,
     MSG_TYPES_STATUS,
 )
-from pylint.lint.expand_modules import expand_modules
+from pylint.lint.expand_modules import _is_ignored_file, expand_modules
 from pylint.lint.parallel import check_parallel
 from pylint.lint.report_functions import (
     report_messages_by_module_stats,
@@ -1013,9 +1013,8 @@ def initialize(self):
             if not msg.may_be_emitted():
                 self._msgs_state[msg.msgid] = False
 
-    @staticmethod
-    def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]:
-        """Discover python modules and packages in subdirectory.
+    def _discover_files(self, files_or_modules: Sequence[str]) -> Iterator[str]:
+        """Discover python modules and packages in sub-directory.
 
         Returns iterator of paths to discovered modules and packages.
         """
@@ -1028,6 +1027,16 @@ def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]:
                     if any(root.startswith(s) for s in skip_subtrees):
                         # Skip subtree of already discovered package.
                         continue
+
+                    if _is_ignored_file(
+                        root,
+                        self.config.ignore,
+                        self.config.ignore_patterns,
+                        self.config.ignore_paths,
+                    ):
+                        skip_subtrees.append(root)
+                        continue
+
                     if "__init__.py" in files:
                         skip_subtrees.append(root)
                         yield root

tests/lint/unittest_lint.py

@@ -13,6 +13,7 @@
 from io import StringIO
 from os import chdir, getcwd
 from os.path import abspath, basename, dirname, isdir, join, sep
+from pathlib import Path
 from shutil import rmtree
 from typing import Iterable, Iterator, List, Optional, Tuple
 
@@ -29,7 +30,13 @@
     OLD_DEFAULT_PYLINT_HOME,
 )
 from pylint.exceptions import InvalidMessageError
-from pylint.lint import ArgumentPreprocessingError, PyLinter, Run, preprocess_options
+from pylint.lint import (
+    ArgumentPreprocessingError,
+    PyLinter,
+    Run,
+    fix_import_path,
+    preprocess_options,
+)
 from pylint.message import Message
 from pylint.reporters import text
 from pylint.typing import MessageLocationTuple
@@ -868,3 +875,67 @@ def test_by_module_statement_value(initialized_linter: PyLinter) -> None:
         # Check that the by_module "statement" is equal to the global "statement"
         # computed for that module
         assert module_stats["statement"] == linter2.stats.statement
+
+
+@pytest.mark.parametrize(
+    "ignore_parameter,ignore_parameter_value",
+    [
+        ("--ignore", "failing.py"),
+        ("--ignore", "ignored_subdirectory"),
+        ("--ignore-patterns", "failing.*"),
+        ("--ignore-patterns", "ignored_*"),
+        ("--ignore-paths", ".*directory/ignored.*"),
+        ("--ignore-paths", ".*ignored.*/failing.*"),
+    ],
+)
+def test_recursive_ignore(ignore_parameter, ignore_parameter_value) -> None:
+    run = Run(
+        [
+            "--recursive",
+            "y",
+            ignore_parameter,
+            ignore_parameter_value,
+            join(REGRTEST_DATA_DIR, "directory"),
+        ],
+        exit=False,
+    )
+
+    linted_files = run.linter._iterate_file_descrs(
+        tuple(run.linter._discover_files([join(REGRTEST_DATA_DIR, "directory")]))
+    )
+    linted_file_paths = [file_item.filepath for file_item in linted_files]
+
+    ignored_file = os.path.abspath(
+        join(REGRTEST_DATA_DIR, "directory", "ignored_subdirectory", "failing.py")
+    )
+    assert ignored_file not in linted_file_paths
+
+    for regrtest_data_module in (
+        ("directory", "subdirectory", "subsubdirectory", "module.py"),
+        ("directory", "subdirectory", "module.py"),
+        ("directory", "package", "module.py"),
+        ("directory", "package", "subpackage", "module.py"),
+    ):
+        module = os.path.abspath(join(REGRTEST_DATA_DIR, *regrtest_data_module))
+    assert module in linted_file_paths
+
+
+def test_import_sibling_module_from_namespace(initialized_linter: PyLinter) -> None:
+    """If the parent directory above `namespace` is on sys.path, ensure that
+    modules under `namespace` can import each other without raising `import-error`."""
+    linter = initialized_linter
+    with tempdir() as tmpdir:
+        create_files(["namespace/submodule1.py", "namespace/submodule2.py"])
+        second_path = Path("namespace/submodule2.py")
+        with open(second_path, "w", encoding="utf-8") as f:
+            f.write(
+                """\"\"\"This module imports submodule1.\"\"\"
+import submodule1
+print(submodule1)
+"""
+            )
+        os.chdir("namespace")
+        # Add the parent directory to sys.path
+        with fix_import_path([tmpdir]):
+            linter.check(["submodule2.py"])
+    assert not linter.stats.by_msg

tests/regrtest_data/directory/ignored_subdirectory/failing.py

@@ -0,0 +1 @@
+import re

tests/test_self.py

@@ -1276,17 +1276,91 @@ def test_max_inferred_for_complicated_class_hierarchy() -> None:
         assert not ex.value.code % 2
 
     def test_regression_recursive(self):
+        """Tests if error is raised when linter is executed over directory not using --recursive=y"""
         self._test_output(
             [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=n"],
             expected_output="No such file or directory",
         )
 
     def test_recursive(self):
+        """Tests if running linter over directory using --recursive=y"""
         self._runtest(
             [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"],
             code=0,
         )
 
+    def test_ignore_recursive(self):
+        """Tests recursive run of linter ignoring directory using --ignore parameter.
+
+        Ignored directory contains files yielding lint errors. If directory is not ignored
+        test would fail due these errors.
+        """
+        self._runtest(
+            [
+                join(HERE, "regrtest_data", "directory"),
+                "--recursive=y",
+                "--ignore=ignored_subdirectory",
+            ],
+            code=0,
+        )
+
+        self._runtest(
+            [
+                join(HERE, "regrtest_data", "directory"),
+                "--recursive=y",
+                "--ignore=failing.py",
+            ],
+            code=0,
+        )
+
+    def test_ignore_pattern_recursive(self):
+        """Tests recursive run of linter ignoring directory using --ignore-parameter parameter.
+
+        Ignored directory contains files yielding lint errors. If directory is not ignored
+        test would fail due these errors.
+        """
+        self._runtest(
+            [
+                join(HERE, "regrtest_data", "directory"),
+                "--recursive=y",
+                "--ignore-pattern=ignored_.*",
+            ],
+            code=0,
+        )
+
+        self._runtest(
+            [
+                join(HERE, "regrtest_data", "directory"),
+                "--recursive=y",
+                "--ignore-pattern=failing.*",
+            ],
+            code=0,
+        )
+
+    def test_ignore_path_recursive(self):
+        """Tests recursive run of linter ignoring directory using --ignore-path parameter.
+
+        Ignored directory contains files yielding lint errors. If directory is not ignored
+        test would fail due these errors.
+        """
+        self._runtest(
+            [
+                join(HERE, "regrtest_data", "directory"),
+                "--recursive=y",
+                "--ignore-path=.*ignored.*",
+            ],
+            code=0,
+        )
+
+        self._runtest(
+            [
+                join(HERE, "regrtest_data", "directory"),
+                "--recursive=y",
+                "--ignore-path=.*failing.*",
+            ],
+            code=0,
+        )
+
     def test_recursive_current_dir(self):
         with _test_sys_path():
             # pytest is including directory HERE/regrtest_data to sys.path which causes
@@ -1297,7 +1371,7 @@ def test_recursive_current_dir(self):
                 if not os.path.basename(path) == "regrtest_data"
             ]
             with _test_cwd():
-                os.chdir(join(HERE, "regrtest_data", "directory"))
+                os.chdir(join(HERE, "regrtest_data", "directory", "subdirectory"))
                 self._runtest(
                     [".", "--recursive=y"],
                     code=0,