From b86c39466549ab9b4f8136b4c60a363656cfb5e6 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 13 May 2022 08:06:04 +0200 Subject: [PATCH] Respect ignore configuration options when --recursive=y. (#6528) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Ignore specified files/directories in recursive mode Co-authored-by: Pierre Sassoulas Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- ChangeLog | 4 + pylint/lint/expand_modules.py | 20 ++++- pylint/lint/pylinter.py | 17 ++++- tests/lint/unittest_lint.py | 64 ++++++++++++++++ .../directory/ignored_subdirectory/failing.py | 1 + tests/test_self.py | 76 ++++++++++++++++++- 6 files changed, 173 insertions(+), 9 deletions(-) create mode 100644 tests/regrtest_data/directory/ignored_subdirectory/failing.py diff --git a/ChangeLog b/ChangeLog index ccce125a1bc..a49bb9e300f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,10 @@ What's New in Pylint 2.13.9? Release date: TBA +* Respect ignore configuration options with ``--recursive=y``. + + Closes #6471 + * Fix false positives for ``no-name-in-module`` and ``import-error`` for ``numpy.distutils`` and ``pydantic``. Closes #6497 diff --git a/pylint/lint/expand_modules.py b/pylint/lint/expand_modules.py index 184316e9bd7..07b92895a3c 100644 --- a/pylint/lint/expand_modules.py +++ b/pylint/lint/expand_modules.py @@ -43,6 +43,20 @@ def _is_in_ignore_list_re(element: str, ignore_list_re: List[Pattern]) -> bool: return any(file_pattern.match(element) for file_pattern in ignore_list_re) +def _is_ignored_file( + element: str, + ignore_list: list[str], + ignore_list_re: list[Pattern[str]], + ignore_list_paths_re: list[Pattern[str]], +) -> bool: + basename = os.path.basename(element) + return ( + basename in ignore_list + or _is_in_ignore_list_re(basename, ignore_list_re) + or _is_in_ignore_list_re(element, ignore_list_paths_re) + ) + + def expand_modules( files_or_modules: List[str], ignore_list: List[str], @@ -58,10 +72,8 @@ def expand_modules( for something in files_or_modules: basename = os.path.basename(something) - if ( - basename in ignore_list - or _is_in_ignore_list_re(os.path.basename(something), ignore_list_re) - or _is_in_ignore_list_re(something, ignore_list_paths_re) + if _is_ignored_file( + something, ignore_list, ignore_list_re, ignore_list_paths_re ): continue module_path = get_python_path(something) diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 0bf8961c1ee..878c1190eba 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -40,7 +40,7 @@ MSG_TYPES_LONG, MSG_TYPES_STATUS, ) -from pylint.lint.expand_modules import expand_modules +from pylint.lint.expand_modules import _is_ignored_file, expand_modules from pylint.lint.parallel import check_parallel from pylint.lint.report_functions import ( report_messages_by_module_stats, @@ -1013,9 +1013,8 @@ def initialize(self): if not msg.may_be_emitted(): self._msgs_state[msg.msgid] = False - @staticmethod - def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]: - """Discover python modules and packages in subdirectory. + def _discover_files(self, files_or_modules: Sequence[str]) -> Iterator[str]: + """Discover python modules and packages in sub-directory. Returns iterator of paths to discovered modules and packages. """ @@ -1028,6 +1027,16 @@ def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]: if any(root.startswith(s) for s in skip_subtrees): # Skip subtree of already discovered package. continue + + if _is_ignored_file( + root, + self.config.ignore, + self.config.ignore_patterns, + self.config.ignore_paths, + ): + skip_subtrees.append(root) + continue + if "__init__.py" in files: skip_subtrees.append(root) yield root diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py index 91df7dc1f6b..78ac99ff866 100644 --- a/tests/lint/unittest_lint.py +++ b/tests/lint/unittest_lint.py @@ -868,3 +868,67 @@ def test_by_module_statement_value(initialized_linter: PyLinter) -> None: # Check that the by_module "statement" is equal to the global "statement" # computed for that module assert module_stats["statement"] == linter2.stats.statement + + +@pytest.mark.parametrize( + "ignore_parameter,ignore_parameter_value", + [ + ("--ignore", "failing.py"), + ("--ignore", "ignored_subdirectory"), + ("--ignore-patterns", "failing.*"), + ("--ignore-patterns", "ignored_*"), + ("--ignore-paths", ".*directory/ignored.*"), + ("--ignore-paths", ".*ignored.*/failing.*"), + ], +) +def test_recursive_ignore(ignore_parameter, ignore_parameter_value) -> None: + run = Run( + [ + "--recursive", + "y", + ignore_parameter, + ignore_parameter_value, + join(REGRTEST_DATA_DIR, "directory"), + ], + exit=False, + ) + + linted_files = run.linter._iterate_file_descrs( + tuple(run.linter._discover_files([join(REGRTEST_DATA_DIR, "directory")])) + ) + linted_file_paths = [file_item.filepath for file_item in linted_files] + + ignored_file = os.path.abspath( + join(REGRTEST_DATA_DIR, "directory", "ignored_subdirectory", "failing.py") + ) + assert ignored_file not in linted_file_paths + + for regrtest_data_module in ( + ("directory", "subdirectory", "subsubdirectory", "module.py"), + ("directory", "subdirectory", "module.py"), + ("directory", "package", "module.py"), + ("directory", "package", "subpackage", "module.py"), + ): + module = os.path.abspath(join(REGRTEST_DATA_DIR, *regrtest_data_module)) + assert module in linted_file_paths + + +def test_import_sibling_module_from_namespace(initialized_linter: PyLinter) -> None: + """If the parent directory above `namespace` is on sys.path, ensure that + modules under `namespace` can import each other without raising `import-error`.""" + linter = initialized_linter + with tempdir() as tmpdir: + create_files(["namespace/submodule1.py", "namespace/submodule2.py"]) + second_path = Path("namespace/submodule2.py") + with open(second_path, "w", encoding="utf-8") as f: + f.write( + """\"\"\"This module imports submodule1.\"\"\" +import submodule1 +print(submodule1) +""" + ) + os.chdir("namespace") + # Add the parent directory to sys.path + with fix_import_path([tmpdir]): + linter.check(["submodule2.py"]) + assert not linter.stats.by_msg diff --git a/tests/regrtest_data/directory/ignored_subdirectory/failing.py b/tests/regrtest_data/directory/ignored_subdirectory/failing.py new file mode 100644 index 00000000000..b199df5420b --- /dev/null +++ b/tests/regrtest_data/directory/ignored_subdirectory/failing.py @@ -0,0 +1 @@ +import re diff --git a/tests/test_self.py b/tests/test_self.py index 36084b717a1..27b1e7bede5 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1276,17 +1276,91 @@ def test_max_inferred_for_complicated_class_hierarchy() -> None: assert not ex.value.code % 2 def test_regression_recursive(self): + """Tests if error is raised when linter is executed over directory not using --recursive=y""" self._test_output( [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=n"], expected_output="No such file or directory", ) def test_recursive(self): + """Tests if running linter over directory using --recursive=y""" self._runtest( [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"], code=0, ) + def test_ignore_recursive(self): + """Tests recursive run of linter ignoring directory using --ignore parameter. + + Ignored directory contains files yielding lint errors. If directory is not ignored + test would fail due these errors. + """ + self._runtest( + [ + join(HERE, "regrtest_data", "directory"), + "--recursive=y", + "--ignore=ignored_subdirectory", + ], + code=0, + ) + + self._runtest( + [ + join(HERE, "regrtest_data", "directory"), + "--recursive=y", + "--ignore=failing.py", + ], + code=0, + ) + + def test_ignore_pattern_recursive(self): + """Tests recursive run of linter ignoring directory using --ignore-parameter parameter. + + Ignored directory contains files yielding lint errors. If directory is not ignored + test would fail due these errors. + """ + self._runtest( + [ + join(HERE, "regrtest_data", "directory"), + "--recursive=y", + "--ignore-pattern=ignored_.*", + ], + code=0, + ) + + self._runtest( + [ + join(HERE, "regrtest_data", "directory"), + "--recursive=y", + "--ignore-pattern=failing.*", + ], + code=0, + ) + + def test_ignore_path_recursive(self): + """Tests recursive run of linter ignoring directory using --ignore-path parameter. + + Ignored directory contains files yielding lint errors. If directory is not ignored + test would fail due these errors. + """ + self._runtest( + [ + join(HERE, "regrtest_data", "directory"), + "--recursive=y", + "--ignore-path=.*ignored.*", + ], + code=0, + ) + + self._runtest( + [ + join(HERE, "regrtest_data", "directory"), + "--recursive=y", + "--ignore-path=.*failing.*", + ], + code=0, + ) + def test_recursive_current_dir(self): with _test_sys_path(): # pytest is including directory HERE/regrtest_data to sys.path which causes @@ -1297,7 +1371,7 @@ def test_recursive_current_dir(self): if not os.path.basename(path) == "regrtest_data" ] with _test_cwd(): - os.chdir(join(HERE, "regrtest_data", "directory")) + os.chdir(join(HERE, "regrtest_data", "directory", "subdirectory")) self._runtest( [".", "--recursive=y"], code=0,